From 294d8ca1c201aeaa7aa83d693e2e0dd856f59ff3 Mon Sep 17 00:00:00 2001 From: Dotta Date: Fri, 30 Oct 2026 15:53:26 -0500 Subject: [PATCH 01/70] perf(heartbeat): schedule retry promotion from durable deadlines Co-Authored-By: Paperclip --- doc/plans/2026-10-30-run-dispatch-sleep.md | 34 ++++ .../heartbeat-run-dispatch-work.test.ts | 146 ++++++++++++++++++ server/src/index.ts | 8 +- .../agent-lifecycle/adapters/records.ts | 13 +- .../modules/run-dispatch/adapters/postgres.ts | 9 ++ .../modules/run-dispatch/application/ports.ts | 1 + .../application/use-cases.test.ts | 1 + server/src/modules/run-dispatch/index.ts | 1 + .../modules/wake-queue/adapters/postgres.ts | 3 + server/src/services/agent-configuration.ts | 2 + server/src/services/agent-lifecycle.ts | 6 +- .../chat-run-milestone-notifications.ts | 2 + server/src/services/companies.ts | 3 + .../services/delivery-work-notifications.ts | 2 + .../execution-control-notifications.ts | 3 + server/src/services/heartbeat.ts | 9 ++ .../services/heartbeat/dispatch-work.test.ts | 105 +++++++++++++ .../src/services/heartbeat/dispatch-work.ts | 39 +++++ server/src/services/heartbeat/queue.ts | 2 + server/src/services/heartbeat/retries.test.ts | 1 + server/src/services/heartbeat/retries.ts | 10 +- .../src/services/heartbeat/run-lifecycle.ts | 2 + server/src/services/instance-settings.ts | 2 + .../issue-review-state-notifications.ts | 2 + server/src/services/issues.ts | 8 + .../native-provider-capacity-retry.ts | 2 + server/src/services/recovery/service.ts | 3 + .../services/run-dispatch-notifications.ts | 6 + ...eduled-retry-cancellation-notifications.ts | 2 + 29 files changed, 415 insertions(+), 12 deletions(-) create mode 100644 doc/plans/2026-10-30-run-dispatch-sleep.md create mode 100644 server/src/__tests__/heartbeat-run-dispatch-work.test.ts create mode 100644 server/src/services/heartbeat/dispatch-work.test.ts create mode 100644 server/src/services/heartbeat/dispatch-work.ts create mode 100644 server/src/services/run-dispatch-notifications.ts diff --git a/doc/plans/2026-10-30-run-dispatch-sleep.md b/doc/plans/2026-10-30-run-dispatch-sleep.md new file mode 100644 index 00000000000..82dd8692043 --- /dev/null +++ b/doc/plans/2026-10-30-run-dispatch-sleep.md @@ -0,0 +1,34 @@ +# Run dispatch without idle retry scans + +The run-dispatch module owns retry admission, its durable `scheduled_retry_at` +deadline, and atomic promotion. The heartbeat queue owns concurrency, current +execution authority, and dispatch. The shared coordinator owns notification +coalescing, nonoverlap, transaction uncertainty, drain/resume and shutdown. +The heartbeat facade and `index.ts` only connect these owners. + +A scheduled retry is a future obligation. Startup reconstructs its next stored +deadline through the run-dispatch adapter. Writes that create, move, cancel or +invalidate retries notify the retry worker in their transaction. Promotion uses +the existing gate and compare-and-set, and its committed write wakes ordinary +queued dispatch. A queued run behind occupied capacity waits for the owning +run's lifecycle transition. Configuration and authority changes also wake it. +An idle queue has no recurring read or timer-driven dispatch attempt. + +The supported writer contract is service writes through the shared database +connection scope, including caller-owned and nested transactions. Existing +explicit domain notification boundaries cover lifecycle, issue review state, +agent configuration/status, company lifecycle and scheduling settings. Direct +SQL writers and independent replicas must provide an explicit ingress signal or +restart reconstruction; a process-local signal does not observe their commits. +This change adds no permanent database listener or write interception. + +The remaining periodic recovery chain still owns deferred comment receipts, +execution-wait cancellation recovery, orphan/resource recovery, stranded issue +assignments, dependency wake repair, task watchdogs, silent active runs and stale +locks. Those duties retain their backstop until their separate producer/deadline +migrations are qualified. This slice removes only the periodic scheduled-retry +promotion call. Ordinary queued dispatch can still be reached by that backstop; +the existing atomic claim and per-agent start lock prevent duplicate execution. + +These workers use internal process deadlines. They do not register Cloud wake +alarms and do not relax the durable idle-sleep safety proof. diff --git a/server/src/__tests__/heartbeat-run-dispatch-work.test.ts b/server/src/__tests__/heartbeat-run-dispatch-work.test.ts new file mode 100644 index 00000000000..527309051fa --- /dev/null +++ b/server/src/__tests__/heartbeat-run-dispatch-work.test.ts @@ -0,0 +1,146 @@ +import { randomUUID } from "node:crypto"; +import { and, eq, sql } from "drizzle-orm"; +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; +import { agents, companies, createDb, heartbeatRuns } from "@paperclipai/db"; +import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; +import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js"; +import { heartbeatService } from "../services/heartbeat.js"; +import { createDeliveryWorkCoordinator } from "../services/delivery-work-coordinator.js"; +import { notifyRunDispatch } from "../services/run-dispatch-notifications.js"; + +const external = resolveExternalTestDatabaseUrl("PAPERCLIP_RUN_DISPATCH_TEST_DATABASE_URL"); +const support = external ? { supported: true } : await getEmbeddedPostgresTestSupport(); + +describe.skipIf(!support.supported)("run dispatch production wiring (controlled local process adapter)", () => { + let database: Awaited> | undefined; + let db: ReturnType; + let heartbeat: ReturnType; + let coordinator: ReturnType | undefined; + let worker: ReturnType["registerRunDispatchWork"]> | undefined; + let reads: ReturnType; + let draining = false; + const errors: unknown[] = []; + beforeAll(async () => { + if (!external) database = await startEmbeddedPostgresTestDatabase("run-dispatch-work-"); + db = createDb(external ?? database!.connectionString, { maxConnections: 6, idleTimeoutSeconds: 0, maxLifetimeSeconds: 0 }); + }, 90_000); + beforeEach(async () => { + await db.execute(sql`TRUNCATE companies CASCADE`); + heartbeat = heartbeatService(db); draining = false; errors.length = 0; + reads = vi.spyOn(db, "select"); + vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"], shouldAdvanceTime: true }); + }); + afterEach(async () => { + await worker?.stop(); await coordinator?.stop(); + await heartbeat.drainActiveRunExecutions(); + worker = undefined; coordinator = undefined; + vi.useRealTimers(); vi.restoreAllMocks(); + expect(errors).toEqual([]); + }); + afterAll(async () => { await db?.$client.end({ timeout: 1 }); await database?.cleanup(); }); + + async function start() { + coordinator = createDeliveryWorkCoordinator({ owner: db, canRun: () => !draining, + canReconcile: () => true, onError: error => errors.push(error) }); + worker = heartbeat.registerRunDispatchWork(coordinator, () => !draining); + await worker.ready; + } + async function fixture() { + const companyId = randomUUID(); + await db.insert(companies).values({ id: companyId, name: "Dispatch", issuePrefix: `D${companyId.slice(0, 7)}`, + requireBoardApprovalForNewAgents: false, defaultResponsibleUserId: "dispatch-user" }); + const [agent] = await db.insert(agents).values({ companyId, name: "Controlled process", role: "engineer", status: "idle", + adapterType: "process", adapterConfig: { command: process.execPath, args: ["-e", "console.log('dispatch-proof')"], cwd: process.cwd() }, + runtimeConfig: { heartbeat: { enabled: true, intervalSec: 0, wakeOnDemand: true } }, + }).returning(); + const [run] = await db.insert(heartbeatRuns).values({ companyId, agentId: agent.id, status: "failed", + responsibleUserId: "dispatch-user", errorCode: "codex_transient_upstream", finishedAt: new Date(), + resultJson: { executionRecovery: { kind: "bootstrap", providerWorkStarted: false } }, contextSnapshot: {}, + }).returning(); + return { companyId, agentId: agent.id, runId: run.id }; + } + async function quiet() { + await heartbeat.drainActiveRunExecutions(); + await vi.advanceTimersByTimeAsync(1); + await vi.waitFor(() => expect(coordinator!.nextWakeAt()).toBeNull(), { timeout: 5_000 }); + const count = reads.mock.calls.length; + await vi.advanceTimersByTimeAsync(20 * 60_000); + expect(reads).toHaveBeenCalledTimes(count); + } + async function status(id: string) { + return db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, id)).then(rows => rows[0]); + } + async function completed(id: string) { + await vi.waitFor(async () => expect((await status(id))?.status).toBe("succeeded"), { timeout: 10_000 }); + await heartbeat.drainActiveRunExecutions(); + } + it("keeps fresh empty, configured idle and retained terminal history quiet", async () => { + await start(); await quiet(); + await fixture(); await worker!.stop(); await coordinator!.stop(); + await start(); await quiet(); + }); + it("ordinary retry creation wakes the worker, respects the exact deadline, dispatches and settles", async () => { + const f = await fixture(); await start(); await quiet(); + const result = await heartbeat.scheduleBoundedRetry(f.runId, { delayMs: 60_000, random: () => 0 }); + expect(result.outcome).toBe("scheduled"); + if (result.outcome !== "scheduled") throw new Error("Expected retry creation"); + await vi.advanceTimersByTimeAsync(1); + await vi.waitFor(() => expect(coordinator!.nextWakeAt()).toBe(result.dueAt.getTime())); + expect((await status(result.run.id)).status).toBe("scheduled_retry"); + await vi.advanceTimersByTimeAsync(result.dueAt.getTime() - Date.now() - 1); + expect((await status(result.run.id)).status).toBe("scheduled_retry"); + await vi.advanceTimersByTimeAsync(1); + await completed(result.run.id); + expect(await db.select().from(heartbeatRuns).where(and(eq(heartbeatRuns.companyId, f.companyId), eq(heartbeatRuns.retryOfRunId, f.runId)))).toHaveLength(1); + await quiet(); + }); + it("reconstructs an overdue retry after process restart without a periodic tick", async () => { + const f = await fixture(); + const scheduled = await heartbeat.scheduleBoundedRetry(f.runId, { now: new Date(Date.now() - 120_000), delayMs: 60_000 }); + if (scheduled.outcome !== "scheduled") throw new Error("Expected retry creation"); + heartbeat = heartbeatService(db); + await start(); await completed(scheduled.run.id); await quiet(); + }); + it("cancels a future retry and removes its old deadline", async () => { + const f = await fixture(); await start(); + const scheduled = await heartbeat.scheduleBoundedRetry(f.runId, { delayMs: 600_000 }); + if (scheduled.outcome !== "scheduled") throw new Error("Expected retry creation"); + await vi.advanceTimersByTimeAsync(1); + await vi.waitFor(() => expect(coordinator!.nextWakeAt()).toBe(scheduled.dueAt.getTime())); + await heartbeat.cancelRun(scheduled.run.id, "operator stop", { suppressImmediateRecovery: true }); + await quiet(); expect((await status(scheduled.run.id)).status).toBe("cancelled"); + }); + it("retains the first retry written during drain and catches it up after resume", async () => { + const f = await fixture(); await start(); await quiet(); draining = true; + const scheduled = await heartbeat.scheduleBoundedRetry(f.runId, { delayMs: 10_000 }); + if (scheduled.outcome !== "scheduled") throw new Error("Expected retry creation"); + await vi.advanceTimersByTimeAsync(60_000); + expect((await status(scheduled.run.id)).status).toBe("scheduled_retry"); + draining = false; worker!.wake(); await vi.advanceTimersByTimeAsync(1); + await completed(scheduled.run.id); await quiet(); + }); + it("holds nested producer work until outer commit and rolls back without a successor", async () => { + const f = await fixture(); await start(); await quiet(); + let release!: () => void; const held = new Promise(resolve => { release = resolve; }); + let queuedId = ""; + const writing = db.transaction(async tx => { + await tx.transaction(async nested => { + await notifyRunDispatch(nested); + const [row] = await nested.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: f.agentId, + responsibleUserId: "dispatch-user", status: "queued", contextSnapshot: {} }).returning(); + queuedId = row.id; + }); + await held; + }); + await vi.waitFor(() => expect(queuedId).not.toBe("")); + await vi.advanceTimersByTimeAsync(1_000); + expect(await status(queuedId)).toBeUndefined(); + release(); await writing; await vi.advanceTimersByTimeAsync(1); await completed(queuedId); await quiet(); + await expect(db.transaction(async tx => { + await tx.transaction(async nested => { await notifyRunDispatch(nested); await nested.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: f.agentId, status: "queued" }); }); + throw new Error("rollback outer"); + })).rejects.toThrow("rollback outer"); + await vi.advanceTimersByTimeAsync(1); await quiet(); + expect(await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.companyId, f.companyId))).toHaveLength(2); + }); +}); diff --git a/server/src/index.ts b/server/src/index.ts index ae229f03518..f0eeddc6c91 100644 --- a/server/src/index.ts +++ b/server/src/index.ts @@ -1842,6 +1842,8 @@ async function startServerWithDatabaseTeardown( await startupHeartbeatRecovery; } + if (config.heartbeatSchedulerEnabled) await heartbeat.registerRunDispatchWork(deliveryWork, + () => !isIdleTaskDrainActive() && !isWarmStandby() && !heartbeatSchedulerStopped).ready; if (config.heartbeatSchedulerEnabled) await heartbeat.registerScheduledRetryCancellationWork(deliveryWork, () => !isIdleTaskDrainActive() && !heartbeatSchedulerStopped && !isWarmStandby()).ready; @@ -2044,12 +2046,10 @@ async function startServerWithDatabaseTeardown( // persisted queued work is still being driven forward. runPeriodicRecovery(() => heartbeat .reapOrphanedRuns({ staleThresholdMs: 5 * 60 * 1000, skipScheduledRetryCancellation: true, skipCapturedInstructionCommit: true, skipInstructionDirectoryCleanup: true, skipStoppedInstructionCollection: true, skipCancelledWorkspacePreservation: true, skipObsoletePolicyReviews: true, skipAutomaticCompletionReviews: true, skipUnsafeWorkspaceExports: true, skipNativePendingEffects: true, skipNativeResponsePresentation: true, skipNativeReviewResponse: true, skipNativeStatusReassessment: true, skipRetainedNativeSessionCleanup: true, skipNativeWorkspaceFinalization: true, skipNativeResultCommit: true, skipNativeRunProjection: true, skipNativeRecoveryNotices: true, skipExternalConversationSettlement: true, skipExternalAgentSettlement: true, skipNativeEnvironmentSettlement: true }) - .then(() => heartbeat.promoteDueScheduledRetries()) - .then(async (promotion) => { + .then(async () => { await heartbeat.resumeQueuedRuns(); const reconciled = await heartbeat.reconcileStrandedAssignedIssues({ skipExternalConversationSettlement: true }); if ( - promotion.promoted > 0 || reconciled.assignmentDispatched > 0 || reconciled.dispatchRequeued > 0 || reconciled.continuationRequeued > 0 || @@ -2058,7 +2058,7 @@ async function startServerWithDatabaseTeardown( reconciled.escalated > 0 ) { logger.warn( - { promotedScheduledRetries: promotion.promoted, promotedScheduledRetryRunIds: promotion.runIds, ...reconciled }, + { ...reconciled }, "periodic heartbeat recovery changed assigned issue state", ); } diff --git a/server/src/modules/agent-lifecycle/adapters/records.ts b/server/src/modules/agent-lifecycle/adapters/records.ts index 1c59758dd91..db842941836 100644 --- a/server/src/modules/agent-lifecycle/adapters/records.ts +++ b/server/src/modules/agent-lifecycle/adapters/records.ts @@ -80,18 +80,19 @@ export function agentRecords(db: Db, effects: LifecycleEffects) { throw conflict("Only agents in error status can have their error cleared"); } - const updated = await db - .update(agents) - .set({ + const updated = await effects.transaction(db, existing.companyId, async tx => { + await effects.beforeStatusChange(tx, existing.companyId, id, "error", "idle"); + return tx.update(agents).set({ status: "idle", pauseReason: null, pausedAt: null, errorReason: null, updatedAt: new Date(), }) - .where(and(eq(agents.id, id), eq(agents.status, "error"), eq(agents.lifecycleState, "ready"))) - .returning() - .then((rows) => rows[0] ?? null); + .where(and(eq(agents.id, id), eq(agents.status, "error"), eq(agents.lifecycleState, "ready"))) + .returning() + .then((rows) => rows[0] ?? null); + }); if (!updated) { throw conflict("Only agents in error status can have their error cleared"); diff --git a/server/src/modules/run-dispatch/adapters/postgres.ts b/server/src/modules/run-dispatch/adapters/postgres.ts index cf9946bafe9..63559e7701d 100644 --- a/server/src/modules/run-dispatch/adapters/postgres.ts +++ b/server/src/modules/run-dispatch/adapters/postgres.ts @@ -511,6 +511,14 @@ export function createPostgresRunDispatchAdapter( return decideScheduledRetryGate(factsResult.facts, input.now); } + async function nextScheduledRetryAt(input: { cutoff: Date | null }): Promise { + const [row] = await db.select({ at: heartbeatRuns.scheduledRetryAt }).from(heartbeatRuns) + .where(and(eq(heartbeatRuns.status, "scheduled_retry"), + input.cutoff ? gte(heartbeatRuns.createdAt, input.cutoff) : undefined)) + .orderBy(asc(heartbeatRuns.scheduledRetryAt)).limit(1); + return row?.at?.getTime() ?? null; + } + async function listDueRetries(input: ListDueRetriesInput): Promise { const rows = await db .select() @@ -1414,6 +1422,7 @@ export function createPostgresRunDispatchAdapter( return { evaluateScheduledRetryGate, + nextScheduledRetryAt, listDueRetries, cancelStaleQueuedRun, dispatchResolvedInteractionIfCurrent, diff --git a/server/src/modules/run-dispatch/application/ports.ts b/server/src/modules/run-dispatch/application/ports.ts index 83e41fd3da6..be6faf39ca7 100644 --- a/server/src/modules/run-dispatch/application/ports.ts +++ b/server/src/modules/run-dispatch/application/ports.ts @@ -24,6 +24,7 @@ export type EvaluateScheduledRetryGateInput = { /** Read-only scheduled-retry operations exposed to application use cases. */ export interface ScheduledRetryReader { + nextScheduledRetryAt(input: { cutoff: Date | null }): Promise; evaluateScheduledRetryGate(input: EvaluateScheduledRetryGateInput): Promise; listDueRetries(input: ListDueRetriesInput): Promise; } diff --git a/server/src/modules/run-dispatch/application/use-cases.test.ts b/server/src/modules/run-dispatch/application/use-cases.test.ts index 58e5ca6f8df..1cce6637ba0 100644 --- a/server/src/modules/run-dispatch/application/use-cases.test.ts +++ b/server/src/modules/run-dispatch/application/use-cases.test.ts @@ -12,6 +12,7 @@ function fakeReader(dueRuns: DueRetryRun[] = []): ScheduledRetryReader & { evalu const evaluateCalls: unknown[] = []; return { evaluateCalls, + async nextScheduledRetryAt() { return null; }, async evaluateScheduledRetryGate(input) { evaluateCalls.push(input); return { allowed: true }; diff --git a/server/src/modules/run-dispatch/index.ts b/server/src/modules/run-dispatch/index.ts index 5293ca0ae64..d44d559618a 100644 --- a/server/src/modules/run-dispatch/index.ts +++ b/server/src/modules/run-dispatch/index.ts @@ -67,6 +67,7 @@ export function createRunDispatch(db: Db, deps: RunDispatchDeps = {}) { }); return { + nextScheduledRetryAt: (input: { cutoff: Date | null }) => adapter.nextScheduledRetryAt(input), evaluateScheduledRetryGate: createEvaluateScheduledRetryGate({ reader: adapter }), promoteScheduledRetry, promoteDueScheduledRetries: createPromoteDueScheduledRetries({ diff --git a/server/src/modules/wake-queue/adapters/postgres.ts b/server/src/modules/wake-queue/adapters/postgres.ts index b888ee51fb3..c773b903dec 100644 --- a/server/src/modules/wake-queue/adapters/postgres.ts +++ b/server/src/modules/wake-queue/adapters/postgres.ts @@ -1,3 +1,4 @@ +import { notifyRunDispatch } from "../../../services/run-dispatch-notifications.js"; import { notifyExternalConversationSettlement } from "../../../services/external-conversation-settlement-notifications.js"; import { notifyNativeStatusReassessment } from "../../../services/native-status-reassessment-notifications.js"; import { notifyUnsafeWorkspaceExportRecovery } from "../../../services/unsafe-workspace-export-notifications.js"; @@ -637,6 +638,7 @@ function buildTransaction(tx: Db, deps: WakeQueuePostgresAdapterDeps, db: Db, ru .returning() .then((rows) => rows[0]); + await notifyRunDispatch(tx); const queuedRun = await tx .insert(heartbeatRuns) .values({ @@ -715,6 +717,7 @@ function buildTransaction(tx: Db, deps: WakeQueuePostgresAdapterDeps, db: Db, ru .returning() .then((rows) => rows[0]); + await notifyRunDispatch(tx); const queuedRun = await tx .insert(heartbeatRuns) .values({ diff --git a/server/src/services/agent-configuration.ts b/server/src/services/agent-configuration.ts index 4139dc6dfe0..88a2c636bdb 100644 --- a/server/src/services/agent-configuration.ts +++ b/server/src/services/agent-configuration.ts @@ -1,3 +1,4 @@ +import { notifyRunDispatch } from "./run-dispatch-notifications.js"; import { notifySkillVersionPinChange } from "./skill-version-retention-notifications.js"; import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js"; import { notifyExternalConversationSettlement } from "./external-conversation-settlement-notifications.js"; @@ -83,6 +84,7 @@ export async function prepareAgentConfiguration(db: Db, existing: AgentConfigura existing.adapterConfig, ); } + await notifyRunDispatch(db); if (normalizedPatch.runtimeConfig !== undefined || normalizedPatch.reportsTo !== undefined) { await notifyDeliveryWork(db, DELIVERY_QUEUES.agentTimers); } diff --git a/server/src/services/agent-lifecycle.ts b/server/src/services/agent-lifecycle.ts index 0a6dcaf0fc9..700c6ba8ffa 100644 --- a/server/src/services/agent-lifecycle.ts +++ b/server/src/services/agent-lifecycle.ts @@ -1,3 +1,4 @@ +import { notifyRunDispatch } from "./run-dispatch-notifications.js"; import { notifyNativeReviewResponse } from "./native-review-response-notifications.js"; import { isAgentStatusInvokable } from "@paperclipai/shared"; import { notifyNativeResponsePresentation } from "./native-response-presentation-notifications.js"; @@ -38,7 +39,10 @@ export function createAgentLifecycleEffects(_hooks: BudgetServiceHooks = {}): li }, notifyWork: tx => notifyDeliveryWork(tx, DELIVERY_QUEUES.agentLifecycle), async beforeStatusChange(tx, companyId, agentId, before, after) { - if (before !== after) await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers); + if (before !== after) { + await notifyRunDispatch(tx); + await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers); + } if (!isAgentStatusInvokable(before) && isAgentStatusInvokable(after)) { await notifyNativeResponsePresentation(tx, { companyId, agentId }); await notifyNativeReviewResponse(tx, { companyId, agentId }); diff --git a/server/src/services/chat-run-milestone-notifications.ts b/server/src/services/chat-run-milestone-notifications.ts index 73535f5434a..159055c10d3 100644 --- a/server/src/services/chat-run-milestone-notifications.ts +++ b/server/src/services/chat-run-milestone-notifications.ts @@ -1,3 +1,4 @@ +import { notifyRunDispatch } from "./run-dispatch-notifications.js"; import { notifyIssueReviewState } from "./issue-review-state-notifications.js"; import { notifyNativeResponsePresentation } from "./native-response-presentation-notifications.js"; import { notifyChatResponseAvailability } from "./chat-response-availability.js"; @@ -11,6 +12,7 @@ import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notificatio export type ChatRunMilestoneScope = { companyId: string; issueId?: string; runId?: string }; export async function notifyChatRunMilestones(tx: object, scope: ChatRunMilestoneScope) { + if (!scope.issueId && !scope.runId) await notifyRunDispatch(tx); if (scope.issueId || scope.runId) { await notifyIssueReviewState(tx, { companyId: scope.companyId, issueId: scope.issueId, runId: scope.issueId ? undefined : scope.runId }); diff --git a/server/src/services/companies.ts b/server/src/services/companies.ts index 0f2ed547a79..cc10ffa3bb8 100644 --- a/server/src/services/companies.ts +++ b/server/src/services/companies.ts @@ -1,3 +1,4 @@ +import { notifyRunDispatch } from "./run-dispatch-notifications.js"; import { notifyChatInboundWakeNotices } from "./chat-inbound-wake-notice-notifications.js"; import { notifyCompanyRetentionWork } from "./retention-work-notifications.js"; import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js"; @@ -332,6 +333,7 @@ export function companyService(db: Db, _budgetHooks: BudgetServiceHooks = {}) { const renamedPrefix = await resolveRenamedIssuePrefix(tx, id, companyPatch); if (companyPatch.status !== undefined && companyPatch.status !== existing.status) { + await notifyRunDispatch(tx); await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers); await notifyDeliveryWork(tx, DELIVERY_QUEUES.skillSourceSync); await notifyDeliveryWork(tx, DELIVERY_QUEUES.routineSchedules); @@ -488,6 +490,7 @@ export function companyService(db: Db, _budgetHooks: BudgetServiceHooks = {}) { const wasAlreadyArchived = existing.status === "archived"; if (!wasAlreadyArchived) { + await notifyRunDispatch(tx); await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers); await notifyDeliveryWork(tx, DELIVERY_QUEUES.skillSourceSync); await notifyDeliveryWork(tx, DELIVERY_QUEUES.routineSchedules); diff --git a/server/src/services/delivery-work-notifications.ts b/server/src/services/delivery-work-notifications.ts index 9195f1fcb12..7388bdb8861 100644 --- a/server/src/services/delivery-work-notifications.ts +++ b/server/src/services/delivery-work-notifications.ts @@ -2,6 +2,8 @@ import { signalDatabaseWork, subscribeDatabaseWork } from "@paperclipai/db"; /** Topics for existing durable queues; register intent before their writes. */ export const DELIVERY_QUEUES = { + scheduledRetryDispatch: "scheduled-retry-dispatch", + queuedRunDispatch: "queued-run-dispatch", runEvidenceMigration: "run-evidence-migration", providerTraceRetention: "provider-trace-retention", nativeRunnerEvidenceRetention: "native-runner-evidence-retention", diff --git a/server/src/services/execution-control-notifications.ts b/server/src/services/execution-control-notifications.ts index 22c8913bbcd..6fef650fe42 100644 --- a/server/src/services/execution-control-notifications.ts +++ b/server/src/services/execution-control-notifications.ts @@ -1,7 +1,9 @@ +import { notifyRunDispatch } from "./run-dispatch-notifications.js"; import type { Db } from "@paperclipai/db"; import { DELIVERY_QUEUES, notifyDeliveryWork, type DeliveryQueue } from "./delivery-work-notifications.js"; export async function notifyExecutionRunWork(transaction: object) { + await notifyRunDispatch(transaction); for (const queue of [DELIVERY_QUEUES.executionStatus, DELIVERY_QUEUES.executionFinalization, DELIVERY_QUEUES.executionReplacement, DELIVERY_QUEUES.executionDisposition, DELIVERY_QUEUES.issueMonitors]) { await notifyDeliveryWork(transaction, queue); @@ -14,6 +16,7 @@ export function writeExecutionControl( write: (tx: Db) => PromiseLike, ): Promise { return db.transaction(async transaction => { + await notifyRunDispatch(transaction); await notifyDeliveryWork(transaction, DELIVERY_QUEUES.issueMonitors); for (const topic of typeof queue === "string" ? [queue] : queue) await notifyDeliveryWork(transaction, topic); return write(transaction as unknown as Db); diff --git a/server/src/services/heartbeat.ts b/server/src/services/heartbeat.ts index 670e4a2755b..9e3348ddace 100644 --- a/server/src/services/heartbeat.ts +++ b/server/src/services/heartbeat.ts @@ -1,3 +1,4 @@ +import { registerRunDispatchWork } from "./heartbeat/dispatch-work.js"; import { externalConversationSettlementWork } from "./external-conversation-settlement-work.js"; import { nativeRecoveryNoticeWork } from "./native-runtime/native-recovery-notice-work.js"; import { nativeRunProjectionWork } from "./native-runtime/native-run-projection-work.js"; @@ -999,6 +1000,7 @@ export function heartbeatService( finalizeAiConnectionBusyDeferral, finalizeWorkspaceBusyDeferral, promoteDueScheduledRetries, + nextScheduledRetryAt, retryScheduledRetryNow, scheduleBoundedRetry, } = createHeartbeatRetries(db, { @@ -4246,6 +4248,13 @@ export function heartbeatService( computeTaskDrain, applyTaskDrain, + registerRunDispatchWork: (coordinator: Parameters[0], canRun: () => boolean) => + registerRunDispatchWork(coordinator, { + promoteDueScheduledRetries, + nextScheduledRetryAt, + resumeQueuedRunsAfterDrain, + schedulingSuppressed: async () => (await getSchedulingSuppression(true)).suppressed, + }, () => canRun() && readTaskDrain(new Date()) === null), promoteDueScheduledRetries, retryScheduledRetryNow, diff --git a/server/src/services/heartbeat/dispatch-work.test.ts b/server/src/services/heartbeat/dispatch-work.test.ts new file mode 100644 index 00000000000..2ba36f38ef2 --- /dev/null +++ b/server/src/services/heartbeat/dispatch-work.test.ts @@ -0,0 +1,105 @@ +import { PgDialect } from "drizzle-orm/pg-core"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { installDatabaseWorkSignals } from "../../../../packages/db/src/work-signals.js"; +import { createDeliveryWorkCoordinator } from "../delivery-work-coordinator.js"; +import { notifyRunDispatch } from "../run-dispatch-notifications.js"; +import { registerRunDispatchWork } from "./dispatch-work.js"; + +const cleanup: Array<() => Promise> = []; +afterEach(async () => { + for (const stop of cleanup.splice(0)) await stop(); + vi.useRealTimers(); +}); +function fixture(input: { due?: number; suppressed?: boolean; draining?: boolean } = {}) { + vi.useFakeTimers(); + vi.setSystemTime(1_000_000); + const dialect = new PgDialect(); + const execute = vi.fn(async (query: any) => { + if (dialect.sqlToQuery(query).sql.includes("pg_current_xact_id")) return [{ xid: "42" }]; + throw new Error("unexpected database query"); + }); + const db = installDatabaseWorkSignals({ execute, async transaction(callback: (tx: any) => Promise) { + return callback({ execute, transaction: db.transaction }); + } }); + const state = { due: input.due ?? null as number | null, suppressed: input.suppressed ?? false, + draining: input.draining ?? false, standby: false, fail: false }; + const promote = vi.fn(async () => { + if (state.fail) throw new Error("database unavailable"); + if (state.due !== null && state.due <= Date.now()) state.due = null; + }); + const next = vi.fn(async () => state.due); + const dispatch = vi.fn(async () => {}); + const coordinator = createDeliveryWorkCoordinator({ owner: db, + canRun: () => !state.draining && !state.standby, canReconcile: () => !state.standby, onError: vi.fn() }); + const worker = registerRunDispatchWork(coordinator, { + promoteDueScheduledRetries: promote, nextScheduledRetryAt: next, + resumeQueuedRunsAfterDrain: dispatch, schedulingSuppressed: async () => state.suppressed, + }, () => !state.draining && !state.standby); + cleanup.push(() => coordinator.stop()); + return { db, state, promote, next, dispatch, coordinator, worker, + notify: () => db.transaction(tx => notifyRunDispatch(tx)), + settle: () => vi.advanceTimersByTimeAsync(0) }; +} + +describe("run dispatch obligations", () => { + it.each([false, true])("settles empty/feature-suppressed state and stops all owner work (suppressed=%s)", async suppressed => { + const f = fixture({ suppressed }); await f.worker.ready; + const calls = [f.promote.mock.calls.length, f.next.mock.calls.length, f.dispatch.mock.calls.length]; + expect(f.coordinator.nextWakeAt()).toBeNull(); + await vi.advanceTimersByTimeAsync(20 * 60_000); + expect([f.promote.mock.calls.length, f.next.mock.calls.length, f.dispatch.mock.calls.length]).toEqual(calls); + expect(f.db.execute).not.toHaveBeenCalled(); + }); + it("restores an exact deadline and does no earlier recovery work", async () => { + const f = fixture({ due: 1_010_000 }); await f.worker.ready; + expect(f.coordinator.nextWakeAt()).toBe(1_010_000); + await vi.advanceTimersByTimeAsync(9_999); expect(f.promote).toHaveBeenCalledOnce(); + await vi.advanceTimersByTimeAsync(1); expect(f.promote).toHaveBeenCalledTimes(2); + expect(f.state.due).toBeNull(); expect(f.coordinator.nextWakeAt()).toBeNull(); + expect(f.dispatch).toHaveBeenCalledOnce(); + }); + it("rearms earlier work and cancels a future deadline on a committed producer change", async () => { + const f = fixture({ due: 1_600_000 }); await f.worker.ready; + f.state.due = 1_005_000; await f.notify(); await f.settle(); + expect(f.coordinator.nextWakeAt()).toBe(1_005_000); + f.state.due = null; await f.notify(); await f.settle(); + expect(f.coordinator.nextWakeAt()).toBeNull(); + const before = f.next.mock.calls.length; + await vi.advanceTimersByTimeAsync(20 * 60_000); expect(f.next).toHaveBeenCalledTimes(before); + }); + it("remembers the first producer arriving during drain without owner SQL", async () => { + const f = fixture(); await f.worker.ready; f.state.draining = true; + await f.notify(); await f.settle(); await vi.advanceTimersByTimeAsync(90_000); + expect(f.promote).toHaveBeenCalledOnce(); expect(f.next).toHaveBeenCalledOnce(); + f.state.draining = false; await vi.advanceTimersByTimeAsync(30_000); + expect(f.promote).toHaveBeenCalledTimes(2); expect(f.dispatch).toHaveBeenCalledTimes(2); + expect(f.coordinator.nextWakeAt()).toBeNull(); + }); + it("retains failed work across drain and settles after recovery", async () => { + const f = fixture(); f.state.fail = true; await f.worker.ready; + f.state.draining = true; await vi.advanceTimersByTimeAsync(90_000); + expect(f.promote).toHaveBeenCalledOnce(); expect(f.next).not.toHaveBeenCalled(); + f.state.fail = false; f.state.draining = false; await vi.advanceTimersByTimeAsync(30_000); + expect(f.promote).toHaveBeenCalledTimes(2); expect(f.coordinator.nextWakeAt()).toBeNull(); + }); + it("preserves a notification during a nonoverlapping pass", async () => { + const f = fixture(); await f.worker.ready; + let release!: () => void; + const held = new Promise(resolve => { release = resolve; }); + f.promote.mockImplementationOnce(() => held); + await f.notify(); await f.settle(); + await f.notify(); await f.settle(); + expect(f.promote).toHaveBeenCalledTimes(2); + release(); await f.settle(); + await vi.advanceTimersByTimeAsync(30_000); + expect(f.promote).toHaveBeenCalledTimes(3); expect(f.coordinator.nextWakeAt()).toBeNull(); + }); + it("does not dispatch on warm standby and cancels deadlines at shutdown", async () => { + const f = fixture({ due: 1_010_000 }); f.state.standby = true; await f.worker.ready; + await vi.advanceTimersByTimeAsync(90_000); expect(f.promote).not.toHaveBeenCalled(); + f.state.standby = false; await vi.advanceTimersByTimeAsync(30_000); + expect(f.promote).toHaveBeenCalledOnce(); expect(f.state.due).toBeNull(); + f.state.due = Date.now() + 60_000; await f.notify(); await f.settle(); await f.worker.stop(); + await vi.advanceTimersByTimeAsync(90_000); expect(f.promote).toHaveBeenCalledTimes(2); + }); +}); diff --git a/server/src/services/heartbeat/dispatch-work.ts b/server/src/services/heartbeat/dispatch-work.ts new file mode 100644 index 00000000000..3291008dd9a --- /dev/null +++ b/server/src/services/heartbeat/dispatch-work.ts @@ -0,0 +1,39 @@ +import type { createDeliveryWorkCoordinator } from "../delivery-work-coordinator.js"; +import { registerDeadlineWork } from "../deadline-work.js"; +import { DELIVERY_QUEUES } from "../delivery-work-notifications.js"; + +export function registerRunDispatchWork( + coordinator: ReturnType, + owner: { + promoteDueScheduledRetries(): Promise; + nextScheduledRetryAt(): Promise; + resumeQueuedRunsAfterDrain(): Promise; + schedulingSuppressed(): Promise; + }, + canRun: () => boolean, +) { + let suppressed = false; + const retries = registerDeadlineWork(coordinator, DELIVERY_QUEUES.scheduledRetryDispatch, { + retryMs: 30_000, + continuationMs: 1_000, + canRun, + run: async signal => { + suppressed = await owner.schedulingSuppressed(); + if (!signal.aborted && !suppressed && canRun()) await owner.promoteDueScheduledRetries(); + }, + next: () => suppressed ? Promise.resolve(null) : owner.nextScheduledRetryAt(), + }); + const queue = registerDeadlineWork(coordinator, DELIVERY_QUEUES.queuedRunDispatch, { + retryMs: 30_000, + canRun, + run: async signal => { + if (!signal.aborted && canRun()) await owner.resumeQueuedRunsAfterDrain(); + }, + next: async () => null, + }); + return { + ready: Promise.all([retries.ready, queue.ready]), + wake() { retries.wake(); queue.wake(); }, + async stop() { await Promise.all([retries.stop(), queue.stop()]); }, + }; +} diff --git a/server/src/services/heartbeat/queue.ts b/server/src/services/heartbeat/queue.ts index a41b486b647..376530b2f0a 100644 --- a/server/src/services/heartbeat/queue.ts +++ b/server/src/services/heartbeat/queue.ts @@ -1,3 +1,4 @@ +import { notifyRunDispatch } from "../run-dispatch-notifications.js"; import { DELIVERY_QUEUES, notifyDeliveryWork } from "../delivery-work-notifications.js"; import { notifyExternalAgentSettlement } from "../../modules/external-agents/index.js"; import { notifyNativeStatusReassessment } from "../native-status-reassessment-notifications.js"; @@ -4632,6 +4633,7 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende } const queueOutcome = await db.transaction(async (tx) => { + await notifyRunDispatch(tx); await tx.execute( sql`select id from agents where id = ${agentId} and company_id = ${agent.companyId} for update`, ); diff --git a/server/src/services/heartbeat/retries.test.ts b/server/src/services/heartbeat/retries.test.ts index a557871c553..a6382081167 100644 --- a/server/src/services/heartbeat/retries.test.ts +++ b/server/src/services/heartbeat/retries.test.ts @@ -35,6 +35,7 @@ function callbacks() { getWorktreeExecutionCutoff: vi.fn(async (): Promise => null), applyRunDispatchPostCommitEffects: vi.fn<(effects: PostCommitEffect[]) => void>(), runDispatch: { + nextScheduledRetryAt: vi.fn(async () => null), evaluateScheduledRetryGate: vi.fn(async () => ({ allowed: true })), promoteDueScheduledRetries: vi.fn(async () => ({ promoted: 0, runIds: [], postCommitEffects: [] })), promoteScheduledRetry: vi.fn(async () => ({ outcome: "not_promoted" })), diff --git a/server/src/services/heartbeat/retries.ts b/server/src/services/heartbeat/retries.ts index 946f5e309f8..5f4df5ed74e 100644 --- a/server/src/services/heartbeat/retries.ts +++ b/server/src/services/heartbeat/retries.ts @@ -1,3 +1,4 @@ +import { notifyRunDispatch } from "../run-dispatch-notifications.js"; import { deriveTaskKeyWithHeartbeatFallback } from "./run-state.js"; import { infrastructureRetryDelayMs } from "../system-recovery-state.js"; import { @@ -77,7 +78,7 @@ type PlanApprovalResumeFailure = { export interface HeartbeatRetryDependencies { hasCaseHandoffForRun: (run: Pick) => Promise; runDispatch: Pick, - "evaluateScheduledRetryGate" | "promoteDueScheduledRetries" | "promoteScheduledRetry">; + "evaluateScheduledRetryGate" | "promoteDueScheduledRetries" | "promoteScheduledRetry" | "nextScheduledRetryAt">; getAgent: (agentId: string) => Promise; getRun: ( runId: string, @@ -997,6 +998,7 @@ export function createHeartbeatRetries(db: Db, dependencies: HeartbeatRetryDepen retryContextSnapshot.previousRunId = continuation.previousRunId; } + await notifyRunDispatch(tx); const wakeupRequest = await tx .insert(agentWakeupRequests) .values({ @@ -1643,6 +1645,10 @@ export function createHeartbeatRetries(db: Db, dependencies: HeartbeatRetryDepen }); } + async function nextScheduledRetryAt() { + return runDispatch.nextScheduledRetryAt({ cutoff: await getWorktreeExecutionCutoff() }); + } + async function promoteDueScheduledRetries(now = new Date()) { const cutoff = await getWorktreeExecutionCutoff(); const result = await runDispatch.promoteDueScheduledRetries({ @@ -1750,6 +1756,7 @@ export function createHeartbeatRetries(db: Db, dependencies: HeartbeatRetryDepen }; const updated = await db.transaction(async (tx) => { + await notifyRunDispatch(tx); const row = await tx .update(heartbeatRuns) .set({ @@ -1901,6 +1908,7 @@ export function createHeartbeatRetries(db: Db, dependencies: HeartbeatRetryDepen finalizeAiConnectionBusyDeferral, finalizeWorkspaceBusyDeferral, promoteDueScheduledRetries, + nextScheduledRetryAt, retryScheduledRetryNow, scheduleBoundedRetry: async ( diff --git a/server/src/services/heartbeat/run-lifecycle.ts b/server/src/services/heartbeat/run-lifecycle.ts index 00a27c1faf2..07bb54dfdf7 100644 --- a/server/src/services/heartbeat/run-lifecycle.ts +++ b/server/src/services/heartbeat/run-lifecycle.ts @@ -1,3 +1,4 @@ +import { notifyRunDispatch } from "../run-dispatch-notifications.js"; import { preserveAcceptedRunUsageProjectionSql } from "../usage-receipts.js"; import { notifyNativeEnvironmentSettlement } from "../native-environment-settlement-notifications.js"; import { notifyExternalAgentSettlement } from "../../modules/external-agents/index.js"; @@ -2447,6 +2448,7 @@ export function createHeartbeatLifecycle(db: Db, dependencies: HeartbeatLifecycl .returning() .then((rows) => rows[0]); + await notifyRunDispatch(tx); const queuedRun = await tx .insert(heartbeatRuns) .values({ diff --git a/server/src/services/instance-settings.ts b/server/src/services/instance-settings.ts index 0208653a86f..174c39d6125 100644 --- a/server/src/services/instance-settings.ts +++ b/server/src/services/instance-settings.ts @@ -1,3 +1,4 @@ +import { notifyRunDispatch } from "./run-dispatch-notifications.js"; import { notifySkillVersionRetentionReconstruction } from "./skill-version-retention-notifications.js"; import { notifyNativeEnvironmentSettlement } from "./native-environment-settlement-notifications.js"; import { notifyExternalAgentSettlement } from "../modules/external-agents/index.js"; @@ -630,6 +631,7 @@ export function instanceSettingsService(db: Db, options: InstanceSettingsService await notifyDeliveryWork(tx, DELIVERY_QUEUES.grokDispatch); if (patch.enableWorktreeRunExecution !== undefined) await notifyScheduledRetryCancellation(tx); if (patch.enableWorktreeRunExecution !== undefined) { + await notifyRunDispatch(tx); await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers); await notifyExternalAgentSettlement(tx); await notifyInstructionCommit(tx); diff --git a/server/src/services/issue-review-state-notifications.ts b/server/src/services/issue-review-state-notifications.ts index 679b7e37475..b769be64056 100644 --- a/server/src/services/issue-review-state-notifications.ts +++ b/server/src/services/issue-review-state-notifications.ts @@ -1,9 +1,11 @@ +import { notifyRunDispatch } from "./run-dispatch-notifications.js"; import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js"; import { notifyExternalConversationSettlement } from "./external-conversation-settlement-notifications.js"; import { notifyNativeReviewResponse } from "./native-review-response-notifications.js"; import { notifyNativeStatusReassessment } from "./native-status-reassessment-notifications.js"; export async function notifyIssueReviewState(tx: object, scope: { companyId: string; issueId?: string; runId?: string; agentId?: string }) { + await notifyRunDispatch(tx); await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors); await notifyExternalConversationSettlement(tx, scope); await notifyNativeReviewResponse(tx, scope); diff --git a/server/src/services/issues.ts b/server/src/services/issues.ts index 42c65bb5d02..a7c8bb55023 100644 --- a/server/src/services/issues.ts +++ b/server/src/services/issues.ts @@ -1,3 +1,4 @@ +import { notifyRunDispatch } from "./run-dispatch-notifications.js"; import { notifyExternalConversationSettlement } from "./external-conversation-settlement-notifications.js"; import { notifyNativeRecoveryNotices } from "./native-recovery-notice-notifications.js"; import { notifyNativeRunProjection } from "./native-run-projection-notifications.js"; @@ -10605,6 +10606,7 @@ export function issueService(db: Db) { : values.parentId ? `issue:${values.parentId}` : undefined); } if (values.monitorNextCheckAt) await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors); + await notifyRunDispatch(tx); await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers); const [issue] = await tx.insert(issues).values(values).returning(); await attachOwnedDraftImages(tx, issue, issue.description, { userId: issueData.createdByUserId, agentId: issueData.createdByAgentId }); @@ -11493,6 +11495,7 @@ export function issueService(db: Db) { } if (patch.status !== undefined || patch.assigneeAgentId !== undefined || patch.assigneeUserId !== undefined || patch.hiddenAt !== undefined || patch.conversationAgentId !== undefined) { + await notifyRunDispatch(tx); await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers); } if (patch.monitorNextCheckAt !== undefined || (receiptExisting.monitorNextCheckAt && @@ -11957,6 +11960,7 @@ export function issueService(db: Db) { } await notifyDecisionTargetIssueRemoval(tx, id); await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors); + await notifyRunDispatch(tx); await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers); removedIssue = await tx .delete(issues) @@ -12085,6 +12089,7 @@ export function issueService(db: Db) { : isNull(issues.executionRunId); const updated = await db.transaction(async tx => { await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors); + await notifyRunDispatch(tx); await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers); return tx .update(issues) @@ -12214,6 +12219,7 @@ export function issueService(db: Db) { const staleExecutionRunId = current.executionRunId; const adopted = await db.transaction(async tx => { await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors); + await notifyRunDispatch(tx); await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers); return tx .update(issues) @@ -12469,6 +12475,7 @@ export function issueService(db: Db) { await notifyTerminalWorkspaceWork(tx, `issue:${id}`); await notifyNativeStatusReassessment(tx, { companyId: existing.companyId, issueId: id }); await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors); + await notifyRunDispatch(tx); await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers); const updated = await tx .update(issues) @@ -12529,6 +12536,7 @@ export function issueService(db: Db) { await notifyUnsafeWorkspaceExportRecovery(tx, { companyId: existing.companyId, issueId: id }); } await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors); + await notifyRunDispatch(tx); await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers); const updated = await tx .update(issues) diff --git a/server/src/services/native-runtime/native-provider-capacity-retry.ts b/server/src/services/native-runtime/native-provider-capacity-retry.ts index 5b54bd0cdfc..7d007c383dd 100644 --- a/server/src/services/native-runtime/native-provider-capacity-retry.ts +++ b/server/src/services/native-runtime/native-provider-capacity-retry.ts @@ -1,3 +1,4 @@ +import { notifyRunDispatch } from "../run-dispatch-notifications.js"; import { randomUUID } from "node:crypto"; import { and, eq } from "drizzle-orm"; import { agentWakeupRequests, heartbeatRuns, type Db } from "@paperclipai/db"; @@ -49,6 +50,7 @@ export async function materializeNativeProviderCapacityRetry(input: { scheduledRetryAt: dueAt.toISOString(), executionRetryAccounting: accountingForScheduledRetry(run, NATIVE_PROVIDER_OVERLOADED_CODE, attempt), }; + await notifyRunDispatch(input.tx); const [wake] = await input.tx.insert(agentWakeupRequests).values({ companyId: run.companyId, agentId: run.agentId, source: "automation", triggerDetail: "system", reason: NATIVE_PROVIDER_OVERLOADED_CODE, status: "queued", runId: retryId, diff --git a/server/src/services/recovery/service.ts b/server/src/services/recovery/service.ts index 991fa8f63b5..4393e0ad5cf 100644 --- a/server/src/services/recovery/service.ts +++ b/server/src/services/recovery/service.ts @@ -1,3 +1,4 @@ +import { notifyRunDispatch } from "../run-dispatch-notifications.js"; import { notifyNativeEnvironmentSettlement } from "../native-environment-settlement-notifications.js"; import { notifyExternalAgentSettlement } from "../../modules/external-agents/index.js"; import { notifyNativeStatusReassessment } from "../native-status-reassessment-notifications.js"; @@ -2716,6 +2717,7 @@ export function recoveryService( const now = new Date(); const retryAt = readProviderQuotaRetryAt(input.latestRun, now); return database.transaction(async (tx) => { + await notifyRunDispatch(tx); const wakeup = await tx .insert(agentWakeupRequests) .values({ @@ -3316,6 +3318,7 @@ export function recoveryService( created = Boolean(enqueuedRun); } else { scheduledRun = await db.transaction(async (tx) => { + await notifyRunDispatch(tx); const wakeup = await tx .insert(agentWakeupRequests) .values({ diff --git a/server/src/services/run-dispatch-notifications.ts b/server/src/services/run-dispatch-notifications.ts new file mode 100644 index 00000000000..9b1bb175894 --- /dev/null +++ b/server/src/services/run-dispatch-notifications.ts @@ -0,0 +1,6 @@ +import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js"; + +export async function notifyRunDispatch(transaction: object) { + await notifyDeliveryWork(transaction, DELIVERY_QUEUES.scheduledRetryDispatch); + await notifyDeliveryWork(transaction, DELIVERY_QUEUES.queuedRunDispatch); +} diff --git a/server/src/services/scheduled-retry-cancellation-notifications.ts b/server/src/services/scheduled-retry-cancellation-notifications.ts index 8a8b4fe1a2f..c5548996607 100644 --- a/server/src/services/scheduled-retry-cancellation-notifications.ts +++ b/server/src/services/scheduled-retry-cancellation-notifications.ts @@ -1,7 +1,9 @@ +import { notifyRunDispatch } from "./run-dispatch-notifications.js"; import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js"; export type ScheduledRetryCancellationScope = { companyId: string; runId?: string }; export async function notifyScheduledRetryCancellation(transaction: object, scope?: ScheduledRetryCancellationScope) { + await notifyRunDispatch(transaction); await notifyDeliveryWork(transaction, DELIVERY_QUEUES.scheduledRetryCancellation, scope ? JSON.stringify(scope) : "*"); } From dd49318d554252ed0cd52134f60caa571aa2cf96 Mon Sep 17 00:00:00 2001 From: Dotta Date: Fri, 30 Oct 2026 16:01:08 -0500 Subject: [PATCH 02/70] fix(server): settle routine status from committed work Co-Authored-By: Paperclip --- doc/plans/2026-10-30-routine-status-sleep.md | 54 +++++ .../routine-execution-status-work.test.ts | 206 ++++++++++++++++++ .../routine-execution-status-worker.test.ts | 86 ++++++++ .../routine-status-reconciliation.test.ts | 6 +- server/src/index.ts | 17 +- .../services/delivery-work-notifications.ts | 1 + .../execution-control-notifications.ts | 5 +- .../src/services/heartbeat/run-lifecycle.ts | 3 +- .../issue-review-state-notifications.ts | 2 + server/src/services/issues.ts | 11 + .../native-routine-status.test.ts | 6 +- .../routine-execution-status-notifications.ts | 5 + .../services/routine-execution-status-work.ts | 18 ++ server/src/services/routine-run-status.ts | 68 +++++- server/src/services/routines.ts | 41 +--- 15 files changed, 475 insertions(+), 54 deletions(-) create mode 100644 doc/plans/2026-10-30-routine-status-sleep.md create mode 100644 server/src/__tests__/routine-execution-status-work.test.ts create mode 100644 server/src/__tests__/routine-execution-status-worker.test.ts create mode 100644 server/src/services/routine-execution-status-notifications.ts create mode 100644 server/src/services/routine-execution-status-work.ts diff --git a/doc/plans/2026-10-30-routine-status-sleep.md b/doc/plans/2026-10-30-routine-status-sleep.md new file mode 100644 index 00000000000..ea6f9364999 --- /dev/null +++ b/doc/plans/2026-10-30-routine-status-sleep.md @@ -0,0 +1,54 @@ +# Routine execution status settlement + +The Cloud routine status worker repairs a routine invocation when its linked +execution task and saved routine status disagree. The previous heartbeat callback +scanned every five minutes, even with no routines or only completed history. + +## Ownership and trigger + +`routine-run-status.ts` owns the projection, custody checks and repair transaction. +The worker loads actual mismatches at startup. It accepts issue hints from issue +creation, status/origin changes, checkout/release, routine issue association, +execution control writes and review state writes. It uses the existing +transaction-aware work notification protocol. An outer transaction owns the hint; +a savepoint release does not wake the repair worker before commit. + +A standard execution must name the same routine, invocation, company and linked +task. Plugin-managed executions may use their declared custom origin. Repair +locks the routine and task and rechecks the linked invocation before writing. +It preserves skipped/coalesced history and unrelated dispatch failures. A task +that resumes clears an execution-task transient failure through the same owner +used by native completion and issue routes. + +`activeIssue` remains derived from open tasks and live heartbeat runs. A terminal +heartbeat with an open task does not create a completion obligation. The worker +does not retry merely because a run is active or a routine exists. + +## Lifecycle and writer contract + +`registerScopedDeadlineWork` owns coalescing, fair batches, failure backoff, drain +and shutdown. A failed owner retries with bounded backoff. Successful settlement +removes that owner. Settled configuration, healthy active tasks and completed +history leave no recurring worker SQL. The existing Cloud and heartbeat scheduler +enablement gates remain in place. Ordinary routine scheduling is unchanged. + +Supported app writers register `notifyRoutineExecutionStatus(tx, issueId)` before +their mutation. Common execution writers without an issue hint request debt +reconstruction. Backups and independent SQL import/repair tools must restore +before startup or call that same transaction contract; process-local hints do +not observe independent SQL or another process. No permanent listener or periodic +configured-routine scan is added. + +The in-process retry deadline does not establish external Cloud wake coverage. +This change preserves the existing host sleep safety checks and does not claim +measured host or database suspension. + +## Acceptance + +Tests exercise ordinary issue writes and a real heartbeat lifecycle transition +against PostgreSQL with the production coordinator. Provider wakeup is simulated. +They inspect durable routine rows and the normal routine detail read separately. +Coverage includes empty/configured/history quiet periods, first producer, +terminal outcomes and reopen, legacy mismatch reconstruction, skipped/coalesced +custody, outer commit/rollback, retry/restart, drain/resume and standby/disable. +Focused worker tests exercise in-flight invalidations, retry isolation and stop. diff --git a/server/src/__tests__/routine-execution-status-work.test.ts b/server/src/__tests__/routine-execution-status-work.test.ts new file mode 100644 index 00000000000..7415b63bfd5 --- /dev/null +++ b/server/src/__tests__/routine-execution-status-work.test.ts @@ -0,0 +1,206 @@ +import { randomUUID } from "node:crypto"; +import { eq, sql } from "drizzle-orm"; +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; +import { agents, companies, createDb, heartbeatRuns, issues, pluginManagedResources, plugins, routineRuns, type Db } from "@paperclipai/db"; +import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; +import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js"; +import { createDeliveryWorkCoordinator } from "../services/delivery-work-coordinator.js"; +import { notifyRoutineExecutionStatus } from "../services/routine-execution-status-notifications.js"; +import { registerRoutineExecutionStatusWork } from "../services/routine-execution-status-work.js"; +import { routineExecutionStatusService } from "../services/routine-run-status.js"; +import { routineService } from "../services/routines.js"; +import { issueService } from "../services/issues.js"; +import { idleWorkSnapshot } from "../services/task-admission.js"; +import { createHeartbeatLifecycle, type HeartbeatLifecycleDependencies } from "../services/heartbeat/run-lifecycle.js"; +import { createHeartbeatRunState } from "../services/heartbeat/run-state.js"; +import { createHeartbeatRunPreparation } from "../services/heartbeat/run-preparation.js"; + +const external = resolveExternalTestDatabaseUrl("PAPERCLIP_ROUTINE_STATUS_WORK_TEST_DATABASE_URL"); +const support = external ? { supported: true } : await getEmbeddedPostgresTestSupport(); +(support.supported ? describe : describe.skip)("routine execution status work with PostgreSQL", () => { + let db: Db, temporary: Awaited> | undefined; + let coordinator: ReturnType | undefined; + let enabled = true, draining = false, standby = false; + const errors: unknown[] = []; + beforeAll(async () => { + if (!external) temporary = await startEmbeddedPostgresTestDatabase("routine-status-work-"); + db = createDb(external ?? temporary!.connectionString, { maxConnections: 6 }); + }, 60_000); + beforeEach(() => { + enabled = true; draining = false; standby = false; errors.length = 0; + vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"], shouldAdvanceTime: true }); + }); + afterEach(async () => { + await coordinator?.stop(); coordinator = undefined; + vi.useRealTimers(); vi.restoreAllMocks(); + await db.execute(sql`SET client_min_messages = warning; TRUNCATE companies CASCADE`); + }); + afterAll(async () => { await db?.$client.end(); await temporary?.cleanup(); }); + const canRun = () => enabled && !draining && !standby; + async function start(service = routineExecutionStatusService(db)) { + coordinator = createDeliveryWorkCoordinator({ owner: db, canRun, canReconcile: () => !standby, + onError: error => errors.push(error) }); + const worker = registerRoutineExecutionStatusWork(coordinator, service, canRun); + await worker.ready; + return worker; + } + async function settled() { await vi.waitFor(() => expect(idleWorkSnapshot().active).toBe(0)); } + async function quiet() { + await settled(); + const select = vi.spyOn(db, "select"), execute = vi.spyOn(db, "execute"), transaction = vi.spyOn(db, "transaction"); + try { + await vi.advanceTimersByTimeAsync(20 * 60_000); + expect(select).not.toHaveBeenCalled(); expect(execute).not.toHaveBeenCalled(); expect(transaction).not.toHaveBeenCalled(); + } finally { select.mockRestore(); execute.mockRestore(); transaction.mockRestore(); } + } + async function fixture() { + const companyId = randomUUID(), agentId = randomUUID(); + await db.insert(companies).values({ id: companyId, name: "Routine status work", issuePrefix: `R${companyId.slice(0, 6)}`, + defaultResponsibleUserId: randomUUID(), requireBoardApprovalForNewAgents: false }); + await db.insert(agents).values({ id: agentId, companyId, name: "Routine worker", status: "active", lifecycleState: "ready", adapterType: "codex_local" }); + const service = routineService(db, { heartbeat: { wakeup: async () => null }, runtimeEnv: {} }); + const routine = await service.create(companyId, { title: "Status projection", assigneeAgentId: agentId, + priority: "medium", status: "active", concurrencyPolicy: "always_enqueue", catchUpPolicy: "skip_missed" }, {}); + return { companyId, agentId, routine, service }; + } + async function dispatch(f: Awaited>) { + return f.service.runRoutine(f.routine.id, { source: "manual", idempotencyKey: randomUUID() }); + } + async function saved(id: string) { return (await db.select().from(routineRuns).where(eq(routineRuns.id, id)))[0]; } + async function projected(id: string, status: string) { + await vi.waitFor(async () => expect((await saved(id)).status).toBe(status)); + await settled(); + } + + it("settles empty startup, configured idle, healthy active and retained terminal history without recurring SQL", async () => { + await start(); await quiet(); + const f = await fixture(); await quiet(); + const run = await dispatch(f); await quiet(); + await issueService(db).update(run.linkedIssueId!, { status: "done" }); + await projected(run.id, "completed"); + const before = await saved(run.id); await quiet(); + expect(await saved(run.id)).toEqual(before); expect(errors).toEqual([]); + }); + it.each(["done", "blocked", "cancelled"])("reactivates from the first ordinary %s transition and preserves input", async status => { + await start(); await quiet(); + const f = await fixture(), run = await dispatch(f); + await issueService(db).update(run.linkedIssueId!, { status }); + await projected(run.id, status === "done" ? "completed" : "failed"); + expect((await saved(run.id)).completedAt).toBeInstanceOf(Date); + await quiet(); expect(errors).toEqual([]); + if (status !== "done") { + await issueService(db).update(run.linkedIssueId!, { status: "in_progress" }); + await projected(run.id, "issue_created"); + expect(await saved(run.id)).toMatchObject({ completedAt: null, failureReason: null }); + await quiet(); + } + }); + it("reconstructs legacy terminal debt and reopened transient failures but leaves skipped and coalesced history unchanged", async () => { + const f = await fixture(), first = await dispatch(f), second = await dispatch(f); + await db.update(issues).set({ status: "done" }).where(eq(issues.id, first.linkedIssueId!)); + await db.update(routineRuns).set({ status: "failed", failureReason: "Execution issue moved to blocked", completedAt: new Date() }).where(eq(routineRuns.id, second.id)); + const terminal = await db.insert(routineRuns).values(["skipped", "coalesced"].map(status => ({ companyId: f.companyId, + routineId: f.routine.id, source: "manual", linkedIssueId: first.linkedIssueId, status, completedAt: new Date() }))).returning(); + await start(); await projected(first.id, "completed"); await projected(second.id, "issue_created"); + for (const row of terminal) expect(await saved(row.id)).toEqual(row); + await quiet(); expect(errors).toEqual([]); + }); + it("settles the declared custom origin of a real plugin-managed dispatch and preserves other companies", async () => { + const f = await fixture(), other = await fixture(); + const pluginKey = `routine-status-${randomUUID()}`, originId = `operation:${randomUUID()}`; + const [plugin] = await db.insert(plugins).values({ pluginKey, packageName: pluginKey, version: "0.1.0", + manifestJson: { id: pluginKey, apiVersion: 1, version: "0.1.0", displayName: "Routine test", description: "Routine test", + author: "Paperclip", categories: ["automation"], capabilities: ["routines.managed"], entrypoints: { worker: "./worker.js" } } }).returning(); + await db.insert(pluginManagedResources).values({ companyId: f.companyId, pluginId: plugin.id, pluginKey, + resourceKind: "routine", resourceKey: "status", resourceId: f.routine.id, defaultsJson: { issueTemplate: { originId } } }); + const run = await dispatch(f), otherRun = await dispatch(other); + expect((await db.select().from(issues).where(eq(issues.id, run.linkedIssueId!)))[0].originId).toBe(originId); + await start(); await quiet(); + await issueService(db).update(run.linkedIssueId!, { status: "done" }); await projected(run.id, "completed"); + const before = await saved(run.id); + await db.transaction(async tx => { await notifyRoutineExecutionStatus(tx, run.linkedIssueId!); await notifyRoutineExecutionStatus(tx, run.linkedIssueId!); }); + await settled(); await quiet(); + expect(await saved(run.id)).toEqual(before); expect((await saved(otherRun.id)).status).toBe("issue_created"); expect(errors).toEqual([]); + }); + it("waits for the real outer transaction and forgets a rolled-back nested status write", async () => { + const f = await fixture(), run = await dispatch(f); await start(); await quiet(); + let release!: () => void, entered!: () => void; + const gate = new Promise(resolve => { release = resolve; }); + const written = new Promise(resolve => { entered = resolve; }); + const writing = db.transaction(async tx => { + await issueService(db).update(run.linkedIssueId!, { status: "done" }, tx); + entered(); await gate; + }); + try { await written; expect((await saved(run.id)).status).toBe("issue_created"); } + finally { release(); await writing; } + await projected(run.id, "completed"); await quiet(); + const other = await dispatch(f); + await expect(db.transaction(async tx => { + await issueService(db).update(other.linkedIssueId!, { status: "cancelled" }, tx); + throw new Error("outer rollback"); + })).rejects.toThrow("outer rollback"); + await settled(); expect((await saved(other.id)).status).toBe("issue_created"); await quiet(); expect(errors).toEqual([]); + }); + it("keeps the first write during drain and reconstructs on resume without passive SQL", async () => { + const f = await fixture(), run = await dispatch(f); await start(); await quiet(); + draining = true; + await issueService(db).update(run.linkedIssueId!, { status: "cancelled" }); + await settled(); await quiet(); expect((await saved(run.id)).status).toBe("issue_created"); + draining = false; await vi.advanceTimersByTimeAsync(30_000); await projected(run.id, "failed"); + await quiet(); expect(errors).toEqual([]); + }); + it.each(["standby", "disabled"])("does no owner SQL while %s and reconstructs when enabled", async mode => { + const f = await fixture(), run = await dispatch(f); + await issueService(db).update(run.linkedIssueId!, { status: "done" }); + if (mode === "standby") standby = true; else enabled = false; + await start(); await quiet(); expect((await saved(run.id)).status).toBe("issue_created"); + standby = false; enabled = true; await vi.advanceTimersByTimeAsync(30_000); + await projected(run.id, "completed"); await quiet(); expect(errors).toEqual([]); + }); + it("retries failed exact owners, retains the retry across drain and reconstructs after restart", async () => { + const f = await fixture(), run = await dispatch(f), service = routineExecutionStatusService(db); + const settle = vi.spyOn(service, "settle").mockRejectedValueOnce(new Error("projection unavailable")); + await start(service); + await issueService(db).update(run.linkedIssueId!, { status: "done" }); + await vi.waitFor(() => expect(errors).toHaveLength(1)); + expect((await saved(run.id)).status).toBe("issue_created"); + await vi.advanceTimersByTimeAsync(29_000); expect(settle).toHaveBeenCalledTimes(1); + draining = true; await vi.advanceTimersByTimeAsync(60_000); await quiet(); + draining = false; await vi.advanceTimersByTimeAsync(30_000); await projected(run.id, "completed"); await quiet(); + await coordinator!.stop(); coordinator = undefined; + const other = await dispatch(f); await issueService(db).update(other.linkedIssueId!, { status: "blocked" }); + await start(); await projected(other.id, "failed"); await quiet(); + }); + it("preserves notifications during a sweep, coalesces duplicate hints and never overlaps", async () => { + const f = await fixture(), first = await dispatch(f), second = await dispatch(f), service = routineExecutionStatusService(db); + let entered!: () => void, release!: () => void; + const enteredGate = new Promise(resolve => { entered = resolve; }); + const releaseGate = new Promise(resolve => { release = resolve; }); + const load = service.load; + vi.spyOn(service, "load").mockImplementationOnce(async issueId => { const rows = await load(issueId); entered(); await releaseGate; return rows; }); + const starting = start(service); await enteredGate; + await issueService(db).update(second.linkedIssueId!, { status: "done" }); + await db.transaction(async tx => { await notifyRoutineExecutionStatus(tx, second.linkedIssueId!); await notifyRoutineExecutionStatus(tx, second.linkedIssueId!); }); + release(); await starting; + await projected(second.id, "completed"); expect((await saved(first.id)).status).toBe("issue_created"); + await quiet(); expect(errors).toEqual([]); + }); + it.each(["succeeded", "failed", "cancelled", "interrupted", "timed_out"])("uses real %s run lifecycle changes for activeIssue without an open-issue retry", async status => { + const f = await fixture(), run = await dispatch(f); await start(); + const [execution] = await db.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: f.agentId, + status: "running", contextSnapshot: { issueId: run.linkedIssueId } }).returning(); + await db.update(issues).set({ executionRunId: execution.id }).where(eq(issues.id, run.linkedIssueId!)); + expect((await f.service.getDetail(f.routine.id))?.activeIssue?.id).toBe(run.linkedIssueId); + const deps = { + ...createHeartbeatRunState(db), ...createHeartbeatRunPreparation(db), + issuesSvc: { addComment: vi.fn(), listReviewAttention: vi.fn() }, + recovery: { escalateStrandedAssignedIssue: vi.fn() }, companySkills: { completeTestRunForIssue: vi.fn() }, + budgets: { getInvocationBlock: vi.fn(async () => null) }, treeControlSvc: { getActivePauseHoldGate: vi.fn(async () => null) }, + enqueueWakeup: vi.fn(), getCurrentUserRedactionOptions: vi.fn(async () => ({})), + getAgentInvokability: vi.fn(async () => ({ invokable: true })), emitTerminalAgentTaskRun: vi.fn(), budgetHooks: {}, + } satisfies HeartbeatLifecycleDependencies; + await createHeartbeatLifecycle(db, deps).setRunStatusIfRunning(execution.id, status, { finishedAt: new Date() }); + expect((await f.service.getDetail(f.routine.id))?.activeIssue).toBeNull(); + expect((await saved(run.id)).status).toBe("issue_created"); await quiet(); expect(errors).toEqual([]); + }); +}); diff --git a/server/src/__tests__/routine-execution-status-worker.test.ts b/server/src/__tests__/routine-execution-status-worker.test.ts new file mode 100644 index 00000000000..b88babbbd98 --- /dev/null +++ b/server/src/__tests__/routine-execution-status-worker.test.ts @@ -0,0 +1,86 @@ +import { installDatabaseWorkSignals } from "../../../packages/db/src/work-signals.js"; +import { afterEach, beforeEach, expect, it, vi } from "vitest"; +import { createDeliveryWorkCoordinator } from "../services/delivery-work-coordinator.js"; +import { registerRoutineExecutionStatusWork } from "../services/routine-execution-status-work.js"; + +let coordinator: ReturnType; +let canRun = true; +const errors: unknown[] = []; +const entry = { id: "run", companyId: "company", routineId: "routine", issueId: "issue" }; +beforeEach(() => { + vi.useFakeTimers(); canRun = true; errors.length = 0; + const owner = installDatabaseWorkSignals({ execute: vi.fn(async () => []) }); + coordinator = createDeliveryWorkCoordinator({ owner, canRun: () => canRun, canReconcile: () => true, + onError: error => errors.push(error) }); +}); +afterEach(async () => { await coordinator.stop(); vi.useRealTimers(); }); + +it("loads actual debt once and leaves no recurring owner work", async () => { + const service = { load: vi.fn(async () => []), settle: vi.fn(async () => true) }; + await registerRoutineExecutionStatusWork(coordinator, service, () => canRun).ready; + expect(service.load).toHaveBeenCalledExactlyOnceWith(undefined); + expect(coordinator.nextWakeAt()).toBeNull(); + await vi.advanceTimersByTimeAsync(20 * 60_000); + expect(service.load).toHaveBeenCalledOnce(); expect(service.settle).not.toHaveBeenCalled(); expect(errors).toEqual([]); +}); + +it("reconstructs only an invalidated issue and coalesces duplicate requests", async () => { + const service = { load: vi.fn(async (_issueId?: string) => [] as typeof entry[]), settle: vi.fn(async () => true) }; + const worker = registerRoutineExecutionStatusWork(coordinator, service, () => canRun); await worker.ready; + service.load.mockResolvedValueOnce([entry]); + worker.invalidate("issue"); worker.invalidate("issue"); + await vi.advanceTimersByTimeAsync(1000); + expect(service.load).toHaveBeenLastCalledWith("issue"); expect(service.load).toHaveBeenCalledTimes(2); + expect(service.settle).toHaveBeenCalledExactlyOnceWith(expect.objectContaining(entry)); + expect(coordinator.nextWakeAt()).toBeNull(); expect(errors).toEqual([]); +}); + +it("retries only the failed owner at backoff and disarms after success", async () => { + const service = { load: vi.fn(async () => [entry]), settle: vi.fn(async () => true).mockRejectedValueOnce(new Error("locked")) }; + await registerRoutineExecutionStatusWork(coordinator, service, () => canRun).ready; + expect(errors).toHaveLength(1); + const retryAt = coordinator.nextWakeAt(); expect(retryAt).toBe(Date.now() + 30_000); + await vi.advanceTimersByTimeAsync(29_999); expect(service.settle).toHaveBeenCalledOnce(); + await vi.advanceTimersByTimeAsync(1); expect(service.settle).toHaveBeenCalledTimes(2); + expect(service.load).toHaveBeenCalledOnce(); expect(coordinator.nextWakeAt()).toBeNull(); + await vi.advanceTimersByTimeAsync(20 * 60_000); expect(service.settle).toHaveBeenCalledTimes(2); +}); + +it("retains startup and first work during drain without passive owner calls", async () => { + canRun = false; + const service = { load: vi.fn(async () => [entry]), settle: vi.fn(async () => true) }; + const worker = registerRoutineExecutionStatusWork(coordinator, service, () => canRun); await worker.ready; + worker.invalidate("issue"); + await vi.advanceTimersByTimeAsync(20 * 60_000); + expect(service.load).not.toHaveBeenCalled(); expect(service.settle).not.toHaveBeenCalled(); + canRun = true; await vi.advanceTimersByTimeAsync(30_000); + expect(service.settle).toHaveBeenCalledOnce(); expect(coordinator.nextWakeAt()).toBeNull(); expect(errors).toEqual([]); +}); + +it("keeps a failed owner across drain and retries after resume", async () => { + const service = { load: vi.fn(async () => [entry]), settle: vi.fn(async () => true).mockRejectedValueOnce(new Error("write failed")) }; + await registerRoutineExecutionStatusWork(coordinator, service, () => canRun).ready; + canRun = false; await vi.advanceTimersByTimeAsync(20 * 60_000); + expect(service.settle).toHaveBeenCalledOnce(); expect(service.load).toHaveBeenCalledOnce(); + canRun = true; await vi.advanceTimersByTimeAsync(30_000); + expect(service.settle).toHaveBeenCalledTimes(2); expect(coordinator.nextWakeAt()).toBeNull(); +}); + +it("retains a notification that arrives while the startup reconstruction is in flight", async () => { + let release!: () => void; + const gate = new Promise(resolve => { release = resolve; }); + const service = { load: vi.fn(async (_issueId?: string) => [] as typeof entry[]), settle: vi.fn(async () => true) }; + service.load.mockImplementationOnce(async () => { await gate; return []; }); + const worker = registerRoutineExecutionStatusWork(coordinator, service, () => canRun); + await vi.waitFor(() => expect(service.load).toHaveBeenCalledOnce()); + worker.invalidate("issue"); service.load.mockResolvedValueOnce([entry]); release(); await worker.ready; + await vi.advanceTimersByTimeAsync(1000); + expect(service.settle).toHaveBeenCalledOnce(); expect(coordinator.nextWakeAt()).toBeNull(); expect(errors).toEqual([]); +}); + +it("cancels retries and subscriptions at shutdown", async () => { + const service = { load: vi.fn(async () => [entry]), settle: vi.fn(async () => true).mockRejectedValue(new Error("offline")) }; + const worker = registerRoutineExecutionStatusWork(coordinator, service, () => canRun); await worker.ready; + await worker.stop(); worker.invalidate("issue"); await vi.advanceTimersByTimeAsync(20 * 60_000); + expect(service.settle).toHaveBeenCalledOnce(); expect(coordinator.nextWakeAt()).toBeNull(); +}); diff --git a/server/src/__tests__/routine-status-reconciliation.test.ts b/server/src/__tests__/routine-status-reconciliation.test.ts index eba0718ce25..964e43d6035 100644 --- a/server/src/__tests__/routine-status-reconciliation.test.ts +++ b/server/src/__tests__/routine-status-reconciliation.test.ts @@ -1,3 +1,4 @@ +import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js"; import { randomUUID } from "node:crypto"; import { eq, sql } from "drizzle-orm"; import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest"; @@ -5,11 +6,12 @@ import { companies, createDb, issues, routineRuns, routines } from "@paperclipai import { routineService } from "../services/routines.js"; import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; -const support = await getEmbeddedPostgresTestSupport(); +const external = resolveExternalTestDatabaseUrl("PAPERCLIP_ROUTINE_STATUS_TEST_DATABASE_URL"); +const support = external ? { supported: true, reason: undefined } : await getEmbeddedPostgresTestSupport(); if (!support.supported) console.warn(`Skipping routine status database tests: ${support.reason}`); (support.supported ? describe : describe.skip)("routine status reconciliation", () => { let db: ReturnType, database: Awaited>; - beforeAll(async () => { database = await startEmbeddedPostgresTestDatabase("routine-status-"); db = createDb(database.connectionString); }, 60_000); + beforeAll(async () => { if (!external) database = await startEmbeddedPostgresTestDatabase("routine-status-"); db = createDb(external ?? database.connectionString, { maxConnections: 6 }); }, 60_000); afterEach(async () => { vi.restoreAllMocks(); await db.execute(sql`SET client_min_messages = warning; TRUNCATE companies CASCADE`); }); afterAll(async () => { await db?.$client.end(); await database?.cleanup(); }); async function seed(status = "blocked") { diff --git a/server/src/index.ts b/server/src/index.ts index ae229f03518..8484b58054f 100644 --- a/server/src/index.ts +++ b/server/src/index.ts @@ -1,3 +1,5 @@ +import { registerRoutineExecutionStatusWork } from "./services/routine-execution-status-work.js"; +import { routineExecutionStatusService } from "./services/routine-run-status.js"; import { registerProviderTraceRetentionWork } from "./services/provider-trace-retention-work.js"; import { registerRunEvidenceMigrationWork } from "./services/run-evidence-migration-work.js"; import { registerSkillVersionRetentionWork } from "./services/skill-version-retention-work.js"; @@ -1560,15 +1562,6 @@ async function startServerWithDatabaseTeardown( heartbeat.drainActiveRunExecutions(); prepareHotRestartShutdown = heartbeat.prepareHotRestartShutdown; const routines = routineService(db as any, { pluginWorkerManager }); - let nextRoutineStatusReconcile = 0; - const scheduleRoutineStatusReconcile = () => { - if (!isCloudManagedInstance() || isWarmStandby() || isIdleTaskDrainActive() || heartbeatSchedulerStopped || - Date.now() < nextRoutineStatusReconcile) return; - nextRoutineStatusReconcile = Date.now() + 5 * 60_000; - trackHeartbeatSchedulerWork(routines.reconcileExecutionRunStatuses().then(result => { - if (result.settled > 0) logger.info(result, "reconciled routine execution issue status"); - }).catch(() => logger.warn("routine execution status reconciliation deferred"))); - }; const statusCards = statusCardService(db as any); const issues = issueService(db as any); const mergedPullRequestConfirmations = issueThreadInteractionService(db as any, { @@ -1989,7 +1982,6 @@ async function startServerWithDatabaseTeardown( // captures the authoritative set of running heartbeat rows. trackHeartbeatSchedulerWork((async () => { if (isIdleTaskDrainActive() || heartbeatSchedulerStopped) return; - scheduleRoutineStatusReconcile(); trackHeartbeatSchedulerWork(runRetentionSweep().catch((err: unknown) => { logger.error({ err }, "decision retention sweep failed"); })); @@ -2097,6 +2089,11 @@ async function startServerWithDatabaseTeardown( })); }; const startHeartbeatScheduler = () => { + if (isCloudManagedInstance()) { + const status = registerRoutineExecutionStatusWork(deliveryWork, routineExecutionStatusService(db as any), + () => !isIdleTaskDrainActive() && !isWarmStandby() && !heartbeatSchedulerStopped); + trackHeartbeatSchedulerWork(status.ready); + } const scheduled = registerScheduledWork(deliveryWork, { retryMs: config.heartbeatSchedulerIntervalMs, tickAgentTimers: async signal => { diff --git a/server/src/services/delivery-work-notifications.ts b/server/src/services/delivery-work-notifications.ts index 9195f1fcb12..22171c435cf 100644 --- a/server/src/services/delivery-work-notifications.ts +++ b/server/src/services/delivery-work-notifications.ts @@ -114,6 +114,7 @@ export const DELIVERY_QUEUES = { pluginLifecycle: "plugin-resource-lifecycle", toolAction: "tool-action-receipts", terminalWorkspaces: "terminal-workspaces", + routineExecutionStatus: "routine-execution-status", routineSchedules: "routine-schedules", issueMonitors: "issue-monitors", agentTimers: "agent-timers", diff --git a/server/src/services/execution-control-notifications.ts b/server/src/services/execution-control-notifications.ts index 22c8913bbcd..33a99774a48 100644 --- a/server/src/services/execution-control-notifications.ts +++ b/server/src/services/execution-control-notifications.ts @@ -1,7 +1,9 @@ +import { notifyRoutineExecutionStatus } from "./routine-execution-status-notifications.js"; import type { Db } from "@paperclipai/db"; import { DELIVERY_QUEUES, notifyDeliveryWork, type DeliveryQueue } from "./delivery-work-notifications.js"; -export async function notifyExecutionRunWork(transaction: object) { +export async function notifyExecutionRunWork(transaction: object, issueId?: string) { + await notifyRoutineExecutionStatus(transaction, issueId); for (const queue of [DELIVERY_QUEUES.executionStatus, DELIVERY_QUEUES.executionFinalization, DELIVERY_QUEUES.executionReplacement, DELIVERY_QUEUES.executionDisposition, DELIVERY_QUEUES.issueMonitors]) { await notifyDeliveryWork(transaction, queue); @@ -14,6 +16,7 @@ export function writeExecutionControl( write: (tx: Db) => PromiseLike, ): Promise { return db.transaction(async transaction => { + await notifyRoutineExecutionStatus(transaction); await notifyDeliveryWork(transaction, DELIVERY_QUEUES.issueMonitors); for (const topic of typeof queue === "string" ? [queue] : queue) await notifyDeliveryWork(transaction, topic); return write(transaction as unknown as Db); diff --git a/server/src/services/heartbeat/run-lifecycle.ts b/server/src/services/heartbeat/run-lifecycle.ts index 00a27c1faf2..f796da03a5e 100644 --- a/server/src/services/heartbeat/run-lifecycle.ts +++ b/server/src/services/heartbeat/run-lifecycle.ts @@ -1044,7 +1044,8 @@ export function createHeartbeatLifecycle(db: Db, dependencies: HeartbeatLifecycl if (previous && isHeartbeatRunTerminalStatus(status)) await notifyRetainedNativeSessionCleanup(tx, { companyId: previous.companyId, runId: previous.id }); if (isHeartbeatRunTerminalStatus(status)) await notifyDeliveryWork(tx, DELIVERY_QUEUES.accounting, companyId); - await notifyExecutionRunWork(tx); + await notifyExecutionRunWork(tx, previous?.nativeIssueId ?? previous?.issueId + ?? readNonEmptyString(parseObject(previous?.contextSnapshot).issueId) ?? undefined); if (previous) await notifyChatRunMilestones(tx, { companyId: previous.companyId, runId: previous.id, issueId: previous.nativeIssueId ?? previous.issueId ?? readNonEmptyString(parseObject(previous.contextSnapshot).issueId) ?? undefined }); return write(tx); diff --git a/server/src/services/issue-review-state-notifications.ts b/server/src/services/issue-review-state-notifications.ts index 679b7e37475..15bc2a24287 100644 --- a/server/src/services/issue-review-state-notifications.ts +++ b/server/src/services/issue-review-state-notifications.ts @@ -1,9 +1,11 @@ +import { notifyRoutineExecutionStatus } from "./routine-execution-status-notifications.js"; import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js"; import { notifyExternalConversationSettlement } from "./external-conversation-settlement-notifications.js"; import { notifyNativeReviewResponse } from "./native-review-response-notifications.js"; import { notifyNativeStatusReassessment } from "./native-status-reassessment-notifications.js"; export async function notifyIssueReviewState(tx: object, scope: { companyId: string; issueId?: string; runId?: string; agentId?: string }) { + await notifyRoutineExecutionStatus(tx, scope.issueId); await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors); await notifyExternalConversationSettlement(tx, scope); await notifyNativeReviewResponse(tx, scope); diff --git a/server/src/services/issues.ts b/server/src/services/issues.ts index 42c65bb5d02..eeebf326267 100644 --- a/server/src/services/issues.ts +++ b/server/src/services/issues.ts @@ -1,3 +1,4 @@ +import { notifyRoutineExecutionStatus } from "./routine-execution-status-notifications.js"; import { notifyExternalConversationSettlement } from "./external-conversation-settlement-notifications.js"; import { notifyNativeRecoveryNotices } from "./native-recovery-notice-notifications.js"; import { notifyNativeRunProjection } from "./native-run-projection-notifications.js"; @@ -10606,6 +10607,7 @@ export function issueService(db: Db) { } if (values.monitorNextCheckAt) await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors); await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers); + if (values.originKind === "routine_execution") await notifyRoutineExecutionStatus(tx, values.id); const [issue] = await tx.insert(issues).values(values).returning(); await attachOwnedDraftImages(tx, issue, issue.description, { userId: issueData.createdByUserId, agentId: issueData.createdByAgentId }); if (issue.visibility === "private" && !inheritedPrivateSubtree && issueData.createdByAgentId) { @@ -11494,6 +11496,7 @@ export function issueService(db: Db) { if (patch.status !== undefined || patch.assigneeAgentId !== undefined || patch.assigneeUserId !== undefined || patch.hiddenAt !== undefined || patch.conversationAgentId !== undefined) { await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers); + await notifyRoutineExecutionStatus(tx, id); } if (patch.monitorNextCheckAt !== undefined || (receiptExisting.monitorNextCheckAt && (patch.status !== undefined || patch.assigneeAgentId !== undefined || patch.assigneeUserId !== undefined))) { @@ -11542,6 +11545,9 @@ export function issueService(db: Db) { await notifyNativeWorkspaceFinalization(tx, { companyId: receiptExisting.companyId, issueId: id }); await notifyNativeStatusReassessment(tx, { companyId: receiptExisting.companyId, issueId: id }); } + if (patch.originKind !== undefined || patch.originRunId !== undefined || patch.originId !== undefined) { + await notifyRoutineExecutionStatus(tx, id); + } const updated = await tx .update(issues) .set(patch) @@ -11958,6 +11964,7 @@ export function issueService(db: Db) { await notifyDecisionTargetIssueRemoval(tx, id); await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors); await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers); + await notifyRoutineExecutionStatus(tx, id); removedIssue = await tx .delete(issues) .where(eq(issues.id, id)) @@ -12086,6 +12093,7 @@ export function issueService(db: Db) { const updated = await db.transaction(async tx => { await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors); await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers); + await notifyRoutineExecutionStatus(tx, id); return tx .update(issues) .set({ @@ -12215,6 +12223,7 @@ export function issueService(db: Db) { const adopted = await db.transaction(async tx => { await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors); await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers); + await notifyRoutineExecutionStatus(tx, id); return tx .update(issues) .set(adoptionSet) @@ -12470,6 +12479,7 @@ export function issueService(db: Db) { await notifyNativeStatusReassessment(tx, { companyId: existing.companyId, issueId: id }); await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors); await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers); + await notifyRoutineExecutionStatus(tx, id); const updated = await tx .update(issues) .set({ @@ -12530,6 +12540,7 @@ export function issueService(db: Db) { } await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors); await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers); + await notifyRoutineExecutionStatus(tx, id); const updated = await tx .update(issues) .set(patch) diff --git a/server/src/services/native-runtime/native-routine-status.test.ts b/server/src/services/native-runtime/native-routine-status.test.ts index 668de83a4d5..415622ea2c3 100644 --- a/server/src/services/native-runtime/native-routine-status.test.ts +++ b/server/src/services/native-runtime/native-routine-status.test.ts @@ -1,3 +1,4 @@ +import { resolveExternalTestDatabaseUrl } from "../../__tests__/helpers/external-test-database.js"; import { randomUUID } from "node:crypto"; import { eq } from "drizzle-orm"; import { afterAll, beforeAll, describe, expect, it } from "vitest"; @@ -8,13 +9,14 @@ import { routineService } from "../routines.js"; import { commitNativeStatusDecision } from "./status-decision-committer.js"; import { NATIVE_STATUS_ARBITER_POLICY_VERSION, type NativeStatusDecision } from "./status-arbiter.js"; -const support = await getEmbeddedPostgresTestSupport(); +const external = resolveExternalTestDatabaseUrl("PAPERCLIP_NATIVE_ROUTINE_STATUS_TEST_DATABASE_URL"); +const support = external ? { supported: true, reason: undefined } : await getEmbeddedPostgresTestSupport(); if (!support.supported) console.warn(`Skipping native routine status tests: ${support.reason}`); (support.supported ? describe : describe.skip)("native routine history status projection", () => { let db: ReturnType, database: Awaited>; const companyId = randomUUID(), agentId = randomUUID(); beforeAll(async () => { - database = await startEmbeddedPostgresTestDatabase("native-routine-status-"); db = createDb(database.connectionString); + if (!external) database = await startEmbeddedPostgresTestDatabase("native-routine-status-"); db = createDb(external ?? database.connectionString, { maxConnections: 6 }); await db.insert(companies).values({ id: companyId, name: "Routine status fixture", issuePrefix: "NRS" }); await db.insert(agents).values({ id: agentId, companyId, name: "Routine worker", adapterType: "paperclip_runner", status: "running" }); }, 60_000); diff --git a/server/src/services/routine-execution-status-notifications.ts b/server/src/services/routine-execution-status-notifications.ts new file mode 100644 index 00000000000..cdd2be826c0 --- /dev/null +++ b/server/src/services/routine-execution-status-notifications.ts @@ -0,0 +1,5 @@ +import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js"; + +export function notifyRoutineExecutionStatus(tx: object, issueId?: string) { + return notifyDeliveryWork(tx, DELIVERY_QUEUES.routineExecutionStatus, issueId ?? "*"); +} diff --git a/server/src/services/routine-execution-status-work.ts b/server/src/services/routine-execution-status-work.ts new file mode 100644 index 00000000000..81c8e566a7c --- /dev/null +++ b/server/src/services/routine-execution-status-work.ts @@ -0,0 +1,18 @@ +import type { createDeliveryWorkCoordinator } from "./delivery-work-coordinator.js"; +import { DELIVERY_QUEUES } from "./delivery-work-notifications.js"; +import { registerScopedDeadlineWork } from "./scoped-deadline-work.js"; +import type { routineExecutionStatusService } from "./routine-run-status.js"; + +export function registerRoutineExecutionStatusWork( + coordinator: ReturnType, + service: ReturnType, + canRun: () => boolean, +) { + return registerScopedDeadlineWork(coordinator, { + queue: DELIVERY_QUEUES.routineExecutionStatus, + decodeScope: hint => hint === "*" ? undefined : hint, + matches: (entry, issueId?: string) => issueId === undefined || entry.issueId === issueId, + load: async (issueId?: string) => (await service.load(issueId)).map(row => ({ ...row, dueAt: Date.now() })), + check: async entry => { await service.settle(entry); return null; }, + }, canRun); +} diff --git a/server/src/services/routine-run-status.ts b/server/src/services/routine-run-status.ts index 77341f9cba4..79c1ff9a231 100644 --- a/server/src/services/routine-run-status.ts +++ b/server/src/services/routine-run-status.ts @@ -1,6 +1,7 @@ -import { and, eq } from "drizzle-orm"; +import { notifyRoutineExecutionStatus } from "./routine-execution-status-notifications.js"; +import { and, eq, inArray, or, sql } from "drizzle-orm"; import type { Db } from "@paperclipai/db"; -import { issues, routineRuns, routines } from "@paperclipai/db"; +import { issues, pluginManagedResources, routineRuns, routines } from "@paperclipai/db"; const EXECUTION_ISSUE_TRANSIENT_FAILURE_CODE = "execution_issue_status"; const EXECUTION_ISSUE_TRANSIENT_FAILURE_STATUSES = ["blocked", "cancelled"] as const; @@ -37,6 +38,7 @@ function legacyExecutionIssueTransientFailureStatus( } export async function finalizeRoutineRun(executor: Db, runId: string, patch: Partial) { + if (patch.linkedIssueId) await notifyRoutineExecutionStatus(executor, patch.linkedIssueId); return executor .update(routineRuns) .set({ @@ -67,14 +69,16 @@ export async function syncRoutineRunStatusForIssue(executor: Db, issueId: string status: routineRuns.status, failureReason: routineRuns.failureReason, triggerPayload: routineRuns.triggerPayload, + completedAt: routineRuns.completedAt, }) .from(routineRuns) .innerJoin(routines, and(eq(routines.id, routineRuns.routineId), eq(routines.companyId, issue.companyId))) .where(and(eq(routineRuns.id, issue.originRunId), eq(routineRuns.companyId, issue.companyId), eq(routineRuns.linkedIssueId, issue.id))) .then((rows) => rows[0] ?? null); - if (!run) return null; + if (!run || ["received", "skipped", "coalesced"].includes(run.status)) return null; if (issue.status === "done") { + if (run.status === "completed" && run.failureReason === null && run.completedAt) return null; const transientFailureStatus = executionIssueTransientFailureStatusFromPayload(run.triggerPayload) ?? legacyExecutionIssueTransientFailureStatus(run.failureReason); const transientFailureClearedAt = executionIssueTransientFailureClearedAtFromPayload(run.triggerPayload); @@ -99,6 +103,7 @@ export async function syncRoutineRunStatusForIssue(executor: Db, issueId: string } if (issue.status === "blocked" || issue.status === "cancelled") { const failureReason = executionIssueTransientFailureReason(issue.status); + if (run.status === "failed" && run.failureReason === failureReason && run.completedAt) return null; return finalizeRoutineRun(executor, issue.originRunId, { status: "failed", failureReason, @@ -135,3 +140,60 @@ export async function syncRoutineRunStatusForIssue(executor: Db, issueId: string return null; } + +export function routineExecutionOriginCondition() { + return or(sql`${issues.originId} = ${routineRuns.routineId}::text`, sql`exists ( + select 1 from ${pluginManagedResources} resource + where resource.company_id = ${routineRuns.companyId} + and resource.resource_kind = 'routine' and resource.resource_id = ${routineRuns.routineId} + and resource.defaults_json -> 'issueTemplate' ->> 'originId' = ${issues.originId} + )`); +} + +export function routineExecutionStatusDebtCondition() { + const transientFailure = sql`(${routineRuns.triggerPayload} -> 'transientFailure' ->> 'code' = ${EXECUTION_ISSUE_TRANSIENT_FAILURE_CODE} + or ${routineRuns.failureReason} in ('Execution issue moved to blocked', 'Execution issue moved to cancelled'))`; + return or( + and(eq(routineRuns.status, "issue_created"), inArray(issues.status, ["done", "blocked", "cancelled"])), + and(eq(routineRuns.status, "completed"), or(inArray(issues.status, ["blocked", "cancelled"]), + and(eq(issues.status, "done"), sql`(${routineRuns.completedAt} is null or ${routineRuns.failureReason} is not null)`))), + and(eq(routineRuns.status, "failed"), transientFailure, + sql`(${issues.status} not in ('blocked', 'cancelled') + or ${routineRuns.completedAt} is null + or ${routineRuns.failureReason} is distinct from 'Execution issue moved to ' || ${issues.status})`), + ); +} + +export function routineExecutionStatusService(db: Db) { + async function load(issueId?: string) { + return db.select({ id: routineRuns.id, companyId: routineRuns.companyId, + issueId: issues.id, routineId: routineRuns.routineId }) + .from(routineRuns) + .innerJoin(routines, and(eq(routines.id, routineRuns.routineId), eq(routines.companyId, routineRuns.companyId))) + .innerJoin(issues, and(eq(issues.id, routineRuns.linkedIssueId), eq(issues.companyId, routineRuns.companyId), + eq(issues.originKind, "routine_execution"), sql`${issues.originRunId} = ${routineRuns.id}::text`, routineExecutionOriginCondition())) + .where(and(routineExecutionStatusDebtCondition(), ...(issueId ? [eq(issues.id, issueId)] : []))) + .orderBy(routineRuns.id); + } + async function settle(candidate: Awaited>[number]) { + return db.transaction(async tx => { + await tx.execute(sql`SET LOCAL lock_timeout = '500ms'`); + await tx.execute(sql`SET LOCAL statement_timeout = '5s'`); + const [routine] = await tx.select({ id: routines.id }).from(routines).where(and( + eq(routines.id, candidate.routineId), eq(routines.companyId, candidate.companyId))).for("share"); + if (!routine) return false; + const [issue] = await tx.select().from(issues).where(and(eq(issues.id, candidate.issueId), + eq(issues.companyId, candidate.companyId), eq(issues.originKind, "routine_execution"), + eq(issues.originRunId, candidate.id))).for("update"); + if (!issue) return false; + const [run] = await tx.select({ id: routineRuns.id }).from(routineRuns) + .innerJoin(issues, eq(issues.id, routineRuns.linkedIssueId)) + .where(and(eq(routineRuns.id, candidate.id), eq(routineRuns.companyId, candidate.companyId), + eq(routineRuns.routineId, candidate.routineId), eq(routineRuns.linkedIssueId, issue.id), + routineExecutionOriginCondition(), routineExecutionStatusDebtCondition())).for("update", { of: routineRuns }); + if (!run) return false; + return (await syncRoutineRunStatusForIssue(tx as unknown as Db, issue.id)) !== null; + }); + } + return { load, settle }; +} diff --git a/server/src/services/routines.ts b/server/src/services/routines.ts index 23787b45249..c391207950b 100644 --- a/server/src/services/routines.ts +++ b/server/src/services/routines.ts @@ -73,7 +73,7 @@ import { secretService } from "./secrets.js"; import { getSecretProvider } from "../secrets/provider-registry.js"; import { parseCron, validateCron } from "./cron.js"; import { heartbeatService } from "./heartbeat.js"; -import { finalizeRoutineRun, syncRoutineRunStatusForIssue } from "./routine-run-status.js"; +import { finalizeRoutineRun, routineExecutionStatusService, syncRoutineRunStatusForIssue } from "./routine-run-status.js"; import { instanceSettingsService, isTruthyRuntimeEnvValue, @@ -2001,7 +2001,6 @@ export function routineService( return run; } - let statusReconcileCursor: string | null = null; async function syncRunStatusForIssue(issueId: string, executor: Db = db) { return syncRoutineRunStatusForIssue(executor, issueId); @@ -3245,41 +3244,13 @@ export function routineService( return { triggered }; }, - // Repair a missed issue-status hook through the same status transition as - // normal issue writes. Never discard queued input or infer run completion - // from age. Lock both owners and re-read the issue before applying repair. reconcileExecutionRunStatuses: async (): Promise<{ scanned: number; settled: number; deferred: number }> => { - const candidates = await db.select({ id: routineRuns.id, companyId: routineRuns.companyId, - issueId: issues.id, routineId: routineRuns.routineId }) - .from(routineRuns).innerJoin(routines, and(eq(routines.id, routineRuns.routineId), eq(routines.companyId, routineRuns.companyId))) - .innerJoin(issues, and(eq(issues.id, routineRuns.linkedIssueId), - eq(issues.companyId, routineRuns.companyId), eq(issues.originKind, "routine_execution"), - sql`${issues.originRunId} = ${routineRuns.id}::text`, sql`${issues.originId} = ${routineRuns.routineId}::text`)) - .where(and(eq(routineRuns.status, "issue_created"), inArray(issues.status, ["done", "cancelled", "blocked"]), - ...(statusReconcileCursor ? [sql`${routineRuns.id}::text > ${statusReconcileCursor}`] : []))) - .orderBy(asc(routineRuns.id)).limit(50); - statusReconcileCursor = candidates.length === 50 ? candidates.at(-1)!.id : null; - let settled = 0; - let deferred = 0; + const service = routineExecutionStatusService(db); + const candidates = await service.load(); + let settled = 0, deferred = 0; for (const candidate of candidates) { try { - const repaired = await db.transaction(async tx => { - await tx.execute(sql`SET LOCAL lock_timeout = '500ms'`); - await tx.execute(sql`SET LOCAL statement_timeout = '5s'`); - const [routine] = await tx.select({ id: routines.id }).from(routines).where(and( - eq(routines.id, candidate.routineId), eq(routines.companyId, candidate.companyId))).for("share"); - if (!routine) return false; - const [issue] = await tx.select().from(issues).where(and(eq(issues.id, candidate.issueId), - eq(issues.companyId, candidate.companyId), eq(issues.originKind, "routine_execution"), - eq(issues.originRunId, candidate.id), eq(issues.originId, candidate.routineId))).for("update"); - if (!issue || !["done", "cancelled", "blocked"].includes(issue.status)) return false; - const [run] = await tx.select().from(routineRuns).where(and(eq(routineRuns.id, candidate.id), - eq(routineRuns.companyId, candidate.companyId), eq(routineRuns.routineId, candidate.routineId), - eq(routineRuns.linkedIssueId, issue.id), eq(routineRuns.status, "issue_created"))).for("update"); - if (!run) return false; - return (await syncRunStatusForIssue(issue.id, tx as unknown as Db)) !== null; - }); - if (repaired) settled += 1; + if (await service.settle(candidate)) settled++; } catch (error) { let current: unknown = error; let retryable = false; @@ -3289,7 +3260,7 @@ export function routineService( current = cause.cause; } if (!retryable) throw error; - deferred += 1; + deferred++; } } return { scanned: candidates.length, settled, deferred }; From acaa86e88c0a93d39e98e685efe1158919aa25dd Mon Sep 17 00:00:00 2001 From: Dotta Date: Fri, 30 Oct 2026 16:10:20 -0500 Subject: [PATCH 03/70] fix: release settled deadline work holds during drain Preserve dirty generations with an in-memory resume deadline while allowing settled transaction holds to end during drain. Keep unresolved database intent fenced by the coordinator. Co-Authored-By: Paperclip --- server/src/__tests__/deadline-work.test.ts | 85 +++++++++++++++++++++- server/src/services/deadline-work.ts | 13 ++-- 2 files changed, 92 insertions(+), 6 deletions(-) diff --git a/server/src/__tests__/deadline-work.test.ts b/server/src/__tests__/deadline-work.test.ts index 7b852977345..7657a7e9d99 100644 --- a/server/src/__tests__/deadline-work.test.ts +++ b/server/src/__tests__/deadline-work.test.ts @@ -1,4 +1,5 @@ -import { installDatabaseWorkSignals } from "../../../packages/db/src/work-signals.js"; +import { idleWorkSnapshot } from "../services/task-admission.js"; +import { installDatabaseWorkSignals, signalDatabaseWork, databaseWorkPending } from "../../../packages/db/src/work-signals.js"; import { afterEach, expect, it, vi } from "vitest"; import { createDeliveryWorkCoordinator } from "../services/delivery-work-coordinator.js"; import { registerDeadlineWork } from "../services/deadline-work.js"; @@ -39,3 +40,85 @@ it.each([ expect(run).toHaveBeenCalledTimes(2); expect(onError).not.toHaveBeenCalled(); }); + +it("releases a settled first-write hold during drain and retains a probe-free resume wake", async () => { + vi.useFakeTimers(); + let draining = false; + const owner = installDatabaseWorkSignals({ + execute: vi.fn(async () => [{ status: "committed" }]), + transaction: async (callback: (tx: object) => Promise) => callback({ execute: async () => [{ xid: "1" }] }), + }); + const run = vi.fn(async () => {}), next = vi.fn(async () => null); + coordinator = createDeliveryWorkCoordinator({ owner, canRun: () => !draining, canReconcile: () => true, onError: vi.fn() }); + const worker = registerDeadlineWork(coordinator, DELIVERY_QUEUES.agentTimers, { run, next, canRun: () => !draining, retryMs: 30_000 }); + await worker.ready; + draining = true; + const before = idleWorkSnapshot().active; + await owner.transaction(async tx => { await signalDatabaseWork(tx, DELIVERY_QUEUES.agentTimers); }); + await vi.advanceTimersByTimeAsync(0); + expect(idleWorkSnapshot().active).toBe(before); + expect(coordinator.nextWakeAt()).not.toBeNull(); + await vi.advanceTimersByTimeAsync(3 * 30_000); + expect(run).toHaveBeenCalledOnce(); expect(next).toHaveBeenCalledOnce(); expect(owner.execute).not.toHaveBeenCalled(); + draining = false; + await vi.advanceTimersByTimeAsync(30_000); + expect(run).toHaveBeenCalledTimes(2); expect(coordinator.nextWakeAt()).toBeNull(); +}); + +it("retains failed work through drain and preserves a future domain deadline", async () => { + vi.useFakeTimers(); + let draining = false; + const owner = installDatabaseWorkSignals({ execute: vi.fn(async () => []) }); + const run = vi.fn(async () => { if (run.mock.calls.length === 1) { draining = true; throw new Error("retry debt"); } }); + const deadline = Date.now() + 600_000; + const next = vi.fn(async () => deadline); + const onError = vi.fn(); + coordinator = createDeliveryWorkCoordinator({ owner, canRun: () => !draining, canReconcile: () => true, onError }); + const worker = registerDeadlineWork(coordinator, DELIVERY_QUEUES.agentTimers, { run, next, canRun: () => !draining, retryMs: 30_000 }); + await worker.ready; + await vi.advanceTimersByTimeAsync(90_000); + expect(run).toHaveBeenCalledOnce(); expect(next).not.toHaveBeenCalled(); expect(owner.execute).not.toHaveBeenCalled(); + draining = false; await vi.advanceTimersByTimeAsync(30_000); + expect(run).toHaveBeenCalledTimes(2); expect(coordinator.nextWakeAt()).toBe(deadline); + draining = true; await vi.advanceTimersByTimeAsync(30_000); + expect(coordinator.nextWakeAt()).toBe(deadline); expect(onError).toHaveBeenCalledOnce(); +}); + +it("does no domain probes in warm standby and reconstructs after admission opens", async () => { + vi.useFakeTimers(); + let standby = true; + const owner = installDatabaseWorkSignals({ execute: vi.fn(async () => []) }); + const run = vi.fn(async () => {}), next = vi.fn(async () => null); + coordinator = createDeliveryWorkCoordinator({ owner, canRun: () => !standby, canReconcile: () => !standby, onError: vi.fn() }); + const worker = registerDeadlineWork(coordinator, DELIVERY_QUEUES.agentTimers, { run, next, canRun: () => !standby, retryMs: 30_000 }); + await worker.ready; await vi.advanceTimersByTimeAsync(90_000); + expect(run).not.toHaveBeenCalled(); expect(next).not.toHaveBeenCalled(); expect(owner.execute).not.toHaveBeenCalled(); + standby = false; await vi.advanceTimersByTimeAsync(30_000); + expect(run).toHaveBeenCalledOnce(); expect(coordinator.nextWakeAt()).toBeNull(); +}); + +it("does not release an ambiguous database intent while dispatch is drained", async () => { + vi.useFakeTimers(); + let draining = false, committed = false; + const owner = installDatabaseWorkSignals({ + execute: vi.fn(async () => [{ status: committed ? "committed" : "in progress" }]), + transaction: async (callback: (tx: object) => Promise) => { + await callback({ execute: async () => [{ xid: "2" }] }); + throw new Error("lost acknowledgement"); + }, + }); + coordinator = createDeliveryWorkCoordinator({ owner, canRun: () => !draining, canReconcile: () => true, onError: vi.fn() }); + const run = vi.fn(async () => {}); + const worker = registerDeadlineWork(coordinator, DELIVERY_QUEUES.agentTimers, { run, next: async () => null, canRun: () => !draining, retryMs: 30_000 }); + await worker.ready; draining = true; + const before = idleWorkSnapshot().active; + await expect(owner.transaction(async tx => { await signalDatabaseWork(tx, DELIVERY_QUEUES.agentTimers); })).rejects.toThrow("lost acknowledgement"); + await vi.advanceTimersByTimeAsync(0); + expect(idleWorkSnapshot().active).toBe(before + 1); + expect(databaseWorkPending(owner, DELIVERY_QUEUES.agentTimers)).toBe(true); + committed = true; await vi.advanceTimersByTimeAsync(30_000); + expect(databaseWorkPending(owner, DELIVERY_QUEUES.agentTimers)).toBe(false); + expect(idleWorkSnapshot().active).toBe(before); + draining = false; await vi.advanceTimersByTimeAsync(30_000); + expect(run).toHaveBeenCalledTimes(2); expect(coordinator.nextWakeAt()).toBeNull(); +}); diff --git a/server/src/services/deadline-work.ts b/server/src/services/deadline-work.ts index 306f340d9fc..15ba76abdae 100644 --- a/server/src/services/deadline-work.ts +++ b/server/src/services/deadline-work.ts @@ -26,7 +26,7 @@ export function registerDeadlineWork( pausePassiveMaintenance: true, onSettled: hints => { requested++; task.onSettled?.(hints); }, run: async signal => { - if (!task.canRun()) { requested++; next = null; return; } + if (!task.canRun()) { requested++; return; } if (Date.now() < earliest) { next = earliest; return; } if (Date.now() < coalesceUntil && (next === null || next > Date.now())) { next = next === null ? coalesceUntil : Math.min(next, coalesceUntil); @@ -39,8 +39,6 @@ export function registerDeadlineWork( if (signal.aborted || !task.canRun()) { next = null; return; } const deadline = await task.next(); - - if (!task.canRun()) { next = null; return; } earliest = deadline === null || deadline > Date.now() ? (task.minimumSpacingMs ? Date.now() + task.minimumSpacingMs : 0) @@ -49,8 +47,13 @@ export function registerDeadlineWork( coalesceUntil = Date.now() + (task.notificationCoalesceMs ?? 0); reconciled = generation; }, - hasPending: async () => requested !== reconciled, - nextRunAt: () => next, + // A settled producer must release its admission hold during drain. Keep + // the unreconciled generation and an in-memory resume wake, without asking + // the domain to probe its database while dispatch is closed. + hasPending: async () => task.canRun() && requested !== reconciled, + nextRunAt: () => !task.canRun() && requested !== reconciled + ? Math.min(next ?? Infinity, Date.now() + task.retryMs) + : next, }); return { ...worker, wake: () => { requested++; worker.wake(); } }; } From e55bc2df65d2a73125d1ed4c02fff69bc67c078a Mon Sep 17 00:00:00 2001 From: Dotta Date: Fri, 30 Oct 2026 16:19:27 -0500 Subject: [PATCH 04/70] fix(server): schedule only outstanding merge confirmations Co-Authored-By: Paperclip --- ...026-10-30-merged-pr-confirmations-sleep.md | 83 ++++++ .../merged-pr-confirmation-sweep.test.ts | 11 +- .../merged-pr-confirmation-work.test.ts | 278 ++++++++++++++++++ .../server-startup-feedback-export.test.ts | 5 +- server/src/index.ts | 17 +- .../chat-confirmation-notifications.ts | 2 + .../services/delivery-work-notifications.ts | 1 + .../src/services/issue-thread-interactions.ts | 101 ++++--- server/src/services/issues.ts | 4 + .../merged-pr-confirmation-notifications.ts | 7 + .../services/merged-pr-confirmation-work.ts | 30 ++ 11 files changed, 474 insertions(+), 65 deletions(-) create mode 100644 doc/plans/2026-10-30-merged-pr-confirmations-sleep.md create mode 100644 server/src/__tests__/merged-pr-confirmation-work.test.ts create mode 100644 server/src/services/merged-pr-confirmation-notifications.ts create mode 100644 server/src/services/merged-pr-confirmation-work.ts diff --git a/doc/plans/2026-10-30-merged-pr-confirmations-sleep.md b/doc/plans/2026-10-30-merged-pr-confirmations-sleep.md new file mode 100644 index 00000000000..fb8b091f841 --- /dev/null +++ b/doc/plans/2026-10-30-merged-pr-confirmations-sleep.md @@ -0,0 +1,83 @@ +# Merged pull request confirmation scheduling + +The merged-PR confirmation duty previously ran from every heartbeat maintenance +callback, including when there were no merge requests. It loaded all pending +confirmation cards and then applied the merge-only parser in memory. + +The interaction service still owns merge-only eligibility, provider access, +automatic acceptance, current resolver authority, audit, and continuation. +The worker registers the existing scoped-deadline helper with a dedicated delivery +topic. Shared scheduling receives interaction/company identities and deadlines; +it receives no GitHub payloads. No database migration is required. + +## Durable truth and producers + +Pending `request_confirmation` interaction rows remain the durable work source. +The candidate query excludes terminal and hidden issues, linked tool actions, +and text without merge intent or a PR reference. The existing fail-closed parser +then excludes additional actions, governed actions, and document/attachment targets. +No accepted or cancelled history keeps a deadline alive. + +Ordinary creation registers the topic before inserting a confirmation in its +transaction. Existing confirmation-resolution notifications invalidate its issue +before acceptance, rejection, withdrawal, supersession, and administrative expiry. +Issue status and visibility changes invalidate that issue before their update. +The signal helper joins caller-owned outer transactions; nested savepoint release +is not a commit. Startup reconstructs existing candidates. Direct tool, secret, +and OAuth interaction inserts do not create eligible merge-only work. Company +portability does not import pending interaction rows. + +The supported writer model is the instance's ordinary service/API process. +Independent SQL writers must use the same transaction-aware notification contract +or explicitly invalidate/restart the owning worker. Process-local notifications +do not establish a cross-replica ingress guarantee. No permanent database listener +or empty recovery scan is added. + +## External observation and retries + +The existing normalized `applyPullRequestEvent` to +`sweepMergedPullRequestConfirmations` seam remains available. A normalized merge +hint checks matching references in its company; unrelated tenants and PRs do not +cause provider reads. Existing provider/binding authorization remains unchanged. + +A pending candidate can reference a PR whose external changes have no dependable +webhook coverage. Each such candidate retains a five-minute observation deadline. +Resolution, withdrawal, supersession, hiding, or terminal issue state removes the +obligation. Provider/read failures and unknown states retain the candidate, using +the existing scoped helper's 30-second exponential backoff capped at five minutes. +These retries do not scan completed history. Removing candidate reconciliation +requires a proven durable delivery/replay guarantee for every referenced PR, +including the first event after idle and supported non-Cloud installations. + +This is candidate-backed external observation. It does not eliminate all polling, +register a Cloud wake alarm, or prove that the host/database suspends. Existing +sleep-safety blockers remain in force. + +## Acceptance evidence + +Focused tests use real PostgreSQL, the actual service producers, transaction +signals, coordinator, scheduler, acceptance transaction, and persisted audit and +chat continuation outbox. GitHub state and the direct heartbeat wake callback are +controlled simulation boundaries. The tests do not claim live GitHub delivery or +real agent execution. + +Cases cover empty/configured idle and retained terminal history with no recurring +SQL/provider work across old cadence windows; the first ordinary creation; +outer commit/rollback; exact candidate and failure deadlines; withdrawal; +provider failure across drain; standby/resume; restart; tenant-scoped and duplicate +merge hints; latest/superseded cards during provider observation; hidden/terminal +issues; URL targets; no-continuation policy; and shutdown during an in-flight read. +Existing startup and normalized GitHub event tests exercise the changed consumers. + +## Existing limitations outside this migration + +Direct non-chat continuation still uses the existing postcommit `wakeup` call. +If that call fails after acceptance commits, the previous periodic sweep also +skipped the accepted row and did not reconstruct the missing wake. This migration +preserves that behavior; a general durable interaction-continuation contract is +a separate follow-up. Real chat bindings retain the existing transactional +`interaction_wakeup` outbox, verified through ordinary creation and acceptance. + +The separate review-authority issue tracked in public issue #17704 and PR #17711 +remains owned there. The worker isolates candidate failures and preserves current +review authorization; it does not grant a system actor new review authority. diff --git a/server/src/__tests__/merged-pr-confirmation-sweep.test.ts b/server/src/__tests__/merged-pr-confirmation-sweep.test.ts index 1e4d56436e7..d9cf9880357 100644 --- a/server/src/__tests__/merged-pr-confirmation-sweep.test.ts +++ b/server/src/__tests__/merged-pr-confirmation-sweep.test.ts @@ -1,3 +1,4 @@ +import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js"; import { randomUUID } from "node:crypto"; import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest"; import { inArray } from "drizzle-orm"; @@ -131,7 +132,8 @@ describe("merged pull-request confirmation extraction", () => { }); }); -const embeddedPostgresSupport = await getEmbeddedPostgresTestSupport(); +const external = resolveExternalTestDatabaseUrl("PAPERCLIP_MERGE_SWEEP_TEST_DATABASE_URL"); +const embeddedPostgresSupport = external ? { supported: true } : await getEmbeddedPostgresTestSupport(); const describeEmbeddedPostgres = embeddedPostgresSupport.supported ? describe : describe.skip; describeEmbeddedPostgres("merged pull-request confirmation sweep", () => { @@ -139,8 +141,8 @@ describeEmbeddedPostgres("merged pull-request confirmation sweep", () => { let tempDb: Awaited> | null = null; beforeAll(async () => { - tempDb = await startEmbeddedPostgresTestDatabase("paperclip-merged-pr-confirmations-"); - db = createDb(tempDb.connectionString); + if (!external) tempDb = await startEmbeddedPostgresTestDatabase("paperclip-merged-pr-confirmations-"); + db = createDb(external ?? tempDb!.connectionString, { maxConnections: 6 }); }, 20_000); afterEach(async () => { @@ -153,6 +155,7 @@ describeEmbeddedPostgres("merged pull-request confirmation sweep", () => { }); afterAll(async () => { + await db?.$client.end({ timeout: 1 }); await tempDb?.cleanup(); }); @@ -276,7 +279,7 @@ describeEmbeddedPostgres("merged pull-request confirmation sweep", () => { repo: "paperclip", number: 39, }])).resolves.toEqual({ - checked: 6, + checked: 3, candidates: 3, accepted: 2, woken: 2, diff --git a/server/src/__tests__/merged-pr-confirmation-work.test.ts b/server/src/__tests__/merged-pr-confirmation-work.test.ts new file mode 100644 index 00000000000..51195e6157b --- /dev/null +++ b/server/src/__tests__/merged-pr-confirmation-work.test.ts @@ -0,0 +1,278 @@ +import { randomUUID } from "node:crypto"; +import { setTimeout as delay } from "node:timers/promises"; +import { eq, sql } from "drizzle-orm"; +import { activityLog, agents, chatActions, chatConversations, chatEndpoints, chatPublications, companies, createDb, issues, issueThreadInteractions, toolApplications, toolConnections, type Db } from "@paperclipai/db"; +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; +import { createDeliveryWorkCoordinator } from "../services/delivery-work-coordinator.js"; +import { issueThreadInteractionService } from "../services/issue-thread-interactions.js"; +import { issueService } from "../services/issues.js"; +import { registerMergeConfirmationWork, MERGE_CONFIRMATION_RECONCILE_MS } from "../services/merged-pr-confirmation-work.js"; +import { idleWorkSnapshot } from "../services/task-admission.js"; +import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; +import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js"; + +const external = resolveExternalTestDatabaseUrl("PAPERCLIP_MERGE_WORK_TEST_DATABASE_URL"); +const support = external ? { supported: true } : await getEmbeddedPostgresTestSupport(); +(support.supported ? describe : describe.skip)("merge confirmation work with PostgreSQL and simulated GitHub", () => { + let temporary: Awaited> | undefined; + let db: ReturnType; + let companyId: string, issueId: string, agentId: string; + let coordinator: ReturnType | undefined; + let draining = false, standby = false; + const errors: unknown[] = []; + const provider = vi.fn(async (_companyId: string, _reference: { number: number }): Promise<"open" | "merged" | "unknown"> => "merged"); + const wakeup = vi.fn(async () => ({ id: "simulated-durable-wake" })); + beforeAll(async () => { + if (!external) temporary = await startEmbeddedPostgresTestDatabase("merged-pr-work-"); + db = createDb(external ?? temporary!.connectionString, { maxConnections: 6, idleTimeoutSeconds: 0, maxLifetimeSeconds: 0 }); + }, 30_000); + beforeEach(async () => { + errors.length = 0; draining = false; standby = false; + provider.mockReset().mockResolvedValue("merged"); wakeup.mockClear(); + companyId = randomUUID(); agentId = randomUUID(); issueId = randomUUID(); + await db.insert(companies).values({ id: companyId, name: "Merge worker", issuePrefix: `M${companyId.slice(0, 6)}` }); + await db.insert(agents).values({ id: agentId, companyId, name: "Coder", role: "engineer", status: "idle", adapterType: "codex_local" }); + await db.insert(issues).values({ id: issueId, companyId, title: "Merge task", status: "todo", assigneeAgentId: agentId }); + await Promise.all(Array.from({ length: 6 }, () => db.execute(sql`select pg_sleep(0.01)`))); + vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"], shouldAdvanceTime: false }); + }); + afterEach(async () => { + await coordinator?.stop(); coordinator = undefined; + vi.useRealTimers(); vi.restoreAllMocks(); + await db.delete(chatActions); await db.delete(chatPublications); await db.delete(chatConversations); await db.delete(chatEndpoints); + await db.delete(toolConnections); await db.delete(toolApplications); + await db.delete(activityLog); await db.delete(issueThreadInteractions); await db.delete(issues); + await db.delete(agents); await db.delete(companies); + }); + afterAll(async () => { await db?.$client.end({ timeout: 1 }); await temporary?.cleanup(); }); + const service = (database: Db = db) => issueThreadInteractionService(database, { resolvePullRequestState: provider, wakeup }); + const input = (number = 39) => ({ kind: "request_confirmation" as const, title: `Merge paperclipai/paperclip#${number}?`, + resolverPolicy: "human_only" as const, continuationPolicy: "wake_assignee_on_accept" as const, + payload: { version: 1 as const, prompt: "Merge the PR?" } }); + const create = (database: Db = db, number = 39) => service(database).create({ id: issueId, companyId }, input(number), { agentId }); + const stored = (id: string) => db.select().from(issueThreadInteractions).where(eq(issueThreadInteractions.id, id)).then(rows => rows[0]); + async function start() { + coordinator = createDeliveryWorkCoordinator({ owner: db, canRun: () => !draining && !standby, + canReconcile: () => !standby, onError: error => errors.push(error) }); + const worker = registerMergeConfirmationWork(coordinator, service(), () => !draining && !standby); + await worker.ready; + return worker; + } + async function waitFor(check: () => Promise | void) { + for (let attempt = 0; attempt < 100; attempt++) { + await vi.advanceTimersByTimeAsync(1000); + await delay(10); + try { await check(); return; } catch (error) { if (attempt === 99) throw error; } + } + } + const settle = () => waitFor(() => expect(idleWorkSnapshot().active).toBe(0)); + async function quiet() { + await waitFor(() => { expect(idleWorkSnapshot().active).toBe(0); expect(coordinator!.nextWakeAt()).toBeNull(); }); + const reads = vi.spyOn(db, "select"), transactions = vi.spyOn(db, "transaction"), executes = vi.spyOn(db, "execute"); + const calls = provider.mock.calls.length; + await vi.advanceTimersByTimeAsync(3 * MERGE_CONFIRMATION_RECONCILE_MS); + expect(reads).not.toHaveBeenCalled(); expect(transactions).not.toHaveBeenCalled(); expect(executes).not.toHaveBeenCalled(); + expect(provider).toHaveBeenCalledTimes(calls); + reads.mockRestore(); transactions.mockRestore(); executes.mockRestore(); + } + + it("settles empty and configured idle, then ordinary creation accepts, audits, wakes once and becomes quiet with retained history", async () => { + await start(); await quiet(); expect(provider).not.toHaveBeenCalled(); + const card = await create(); + await waitFor(async () => expect((await stored(card.id)).status).toBe("accepted")); + await quiet(); + expect(wakeup).toHaveBeenCalledTimes(1); + expect(wakeup).toHaveBeenCalledWith(agentId, expect.objectContaining({ idempotencyKey: `interaction:${card.id}:accepted`, allowRunCoalescing: false })); + expect(await db.select().from(activityLog)).toEqual(expect.arrayContaining([expect.objectContaining({ + action: "issue.thread_interaction_accepted", actorId: "system:pr-merged", details: expect.objectContaining({ interactionId: card.id }), + })])); + expect(errors).toEqual([]); + }); + + it("reconciles an open candidate at its deadline, never early, then stops after merge", async () => { + provider.mockResolvedValue("open"); await start(); const card = await create(); + await waitFor(() => expect(provider).toHaveBeenCalledTimes(1)); await settle(); + const dueAt = coordinator!.nextWakeAt()!; + expect(dueAt).toBeGreaterThan(Date.now()); + await vi.advanceTimersByTimeAsync(dueAt - Date.now() - 1); expect(provider).toHaveBeenCalledTimes(1); + provider.mockResolvedValue("merged"); await vi.advanceTimersByTimeAsync(1); + await waitFor(async () => expect((await stored(card.id)).status).toBe("accepted")); await quiet(); + expect(provider).toHaveBeenCalledTimes(2); expect(wakeup).toHaveBeenCalledTimes(1); + }); + + it("manual cancellation removes the candidate deadline without provider work", async () => { + provider.mockResolvedValue("open"); await start(); const card = await create(); + await waitFor(() => expect(provider).toHaveBeenCalledTimes(1)); await settle(); + await service().withdrawInteraction({ id: issueId, companyId }, card.id, {}, { agentId }); + await quiet(); expect(provider).toHaveBeenCalledTimes(1); expect(wakeup).not.toHaveBeenCalled(); + expect((await stored(card.id)).status).toBe("cancelled"); + }); + + it("does not accept before caller-owned outer commit and forgets a rolled-back create", async () => { + await start(); await quiet(); + let release!: () => void, created!: () => void; + const barrier = new Promise(resolve => { release = resolve; }); + const inserted = new Promise(resolve => { created = resolve; }); + let id = ""; + const transaction = db.transaction(async tx => { + id = (await create(tx as unknown as Db)).id; created(); await barrier; + }); + await inserted; + await vi.advanceTimersByTimeAsync(10_000); expect(provider).not.toHaveBeenCalled(); + expect(await stored(id)).toBeUndefined(); release(); await transaction; + await waitFor(async () => expect((await stored(id)).status).toBe("accepted")); await quiet(); + const prior = provider.mock.calls.length; + await expect(db.transaction(async tx => { await create(tx as unknown as Db, 41); throw new Error("rollback"); })).rejects.toThrow("rollback"); + await quiet(); expect(provider).toHaveBeenCalledTimes(prior); expect(errors).toEqual([]); + }); + + it("retains provider failures across drain, resumes without overlap, and clears retry after success", async () => { + provider.mockRejectedValue(new Error("GitHub unavailable")); await start(); const card = await create(); + await waitFor(() => expect(errors).toHaveLength(1)); await settle(); + expect((await stored(card.id)).status).toBe("pending"); + draining = true; + const reads = vi.spyOn(db, "select"), calls = provider.mock.calls.length; + await vi.advanceTimersByTimeAsync(120_000); + expect(reads).not.toHaveBeenCalled(); expect(provider).toHaveBeenCalledTimes(calls); reads.mockRestore(); + draining = false; provider.mockResolvedValue("merged"); + await waitFor(async () => expect((await stored(card.id)).status).toBe("accepted")); await quiet(); + expect(wakeup).toHaveBeenCalledTimes(1); + }); + + it("keeps first work committed during drain and startup during standby until resume", async () => { + standby = true; await start(); const card = await create(); + await vi.advanceTimersByTimeAsync(60_000); expect(provider).not.toHaveBeenCalled(); + standby = false; draining = true; + await vi.advanceTimersByTimeAsync(60_000); expect(provider).not.toHaveBeenCalled(); + draining = false; + await waitFor(async () => expect((await stored(card.id)).status).toBe("accepted")); await quiet(); + }); + + it("reconstructs pending cards on restart and ignores terminal or hidden tasks", async () => { + provider.mockResolvedValue("open"); const card = await create(); await start(); + await waitFor(() => expect(provider).toHaveBeenCalledTimes(1)); await settle(); + await coordinator!.stop(); coordinator = undefined; provider.mockResolvedValue("merged"); + await start(); await waitFor(async () => expect((await stored(card.id)).status).toBe("accepted")); await quiet(); + const hidden = await db.transaction(async tx => { + const card = await create(tx as unknown as Db, 41); + await issueService(tx as unknown as Db).update(issueId, { hiddenAt: new Date() }); + return card; + }); + await quiet(); expect((await stored(hidden.id)).status).toBe("pending"); + await issueService(db).update(issueId, { hiddenAt: null }); + await waitFor(async () => expect((await stored(hidden.id)).status).toBe("accepted")); await quiet(); + }); + + it("preserves a real chat binding's durable continuation outbox on automatic acceptance", async () => { + const applicationId = randomUUID(), connectionId = randomUUID(), endpointId = randomUUID(); + await db.insert(toolApplications).values({ id: applicationId, companyId, applicationKey: `chat:slack:${endpointId}`, name: "Slack", type: "chat", status: "active" }); + await db.insert(toolConnections).values({ id: connectionId, companyId, applicationId, name: "Slack", uid: endpointId, + connectionPurpose: "channel", transport: "chat_sdk", status: "active", enabled: true }); + await db.insert(chatEndpoints).values({ id: endpointId, companyId, connectionId, provider: "slack", publicId: endpointId, + assignedAgentId: agentId, status: "active" }); + const [conversation] = await db.insert(chatConversations).values({ companyId, endpointId, issueId, + externalConversationId: "merge-channel", externalThreadId: "merge-thread", externalLabel: "Merge thread", state: "active" }).returning(); + await start(); await quiet(); const card = await create(); + await waitFor(async () => expect((await stored(card.id)).status).toBe("accepted")); await quiet(); + expect(await db.select().from(chatActions).where(eq(chatActions.kind, "interaction_wakeup"))).toEqual([ + expect.objectContaining({ companyId, endpointId, conversationId: conversation.id, providerActionId: `interaction_wakeup:${card.id}`, + payload: expect.objectContaining({ interactionId: card.id, issueId, agentId, interactionStatus: "accepted" }) }), + ]); + expect(await db.select().from(chatPublications)).toEqual(expect.arrayContaining([ + expect.objectContaining({ idempotencyKey: `interaction:${card.id}:${endpointId}`, state: "cancelled" }), + ])); + expect(errors).toEqual([]); + }); + + it("does not lose a newer card created during provider observation or wake the superseded card", async () => { + let release!: () => void, observed!: () => void; + const barrier = new Promise(resolve => { release = resolve; }); + const observing = new Promise(resolve => { observed = resolve; }); + provider.mockImplementationOnce(async () => { observed(); await barrier; return "merged"; }); + await start(); const old = await create(); + await vi.advanceTimersByTimeAsync(1000); await observing; + const current = await create(db, 41); + expect((await stored(old.id)).status).toBe("expired"); + release(); + await waitFor(async () => expect((await stored(current.id)).status).toBe("accepted")); await quiet(); + expect(wakeup).toHaveBeenCalledTimes(1); + expect(wakeup).toHaveBeenCalledWith(agentId, expect.objectContaining({ idempotencyKey: `interaction:${current.id}:accepted` })); + expect(errors).toEqual([]); + }); + + it("keeps unknown provider results as retryable debt and does not act before retry", async () => { + provider.mockResolvedValue("unknown"); await start(); const card = await create(); + await waitFor(() => expect(errors).toHaveLength(1)); + const next = coordinator!.nextWakeAt()!; + await vi.advanceTimersByTimeAsync(next - Date.now() - 1); expect(provider).toHaveBeenCalledTimes(1); + provider.mockResolvedValue("merged"); await vi.advanceTimersByTimeAsync(1); + await waitFor(async () => expect((await stored(card.id)).status).toBe("accepted")); await quiet(); + expect(provider).toHaveBeenCalledTimes(2); + }); + + it("honors terminal issues, pure merge URL targets and continuationPolicy none", async () => { + await start(); await quiet(); + const target = await service().create({ id: issueId, companyId }, { ...input(), title: "Merge the linked PR?", continuationPolicy: "none", + payload: { version: 1, prompt: "Merge the PR?", target: { type: "custom", key: "github-pr-39", href: "https://github.com/paperclipai/paperclip/pull/39" } } }, { agentId }); + await waitFor(async () => expect((await stored(target.id)).status).toBe("accepted")); await quiet(); + expect(wakeup).not.toHaveBeenCalled(); + provider.mockClear(); + const terminal = await db.transaction(async tx => { + const card = await create(tx as unknown as Db, 41); + await issueService(tx as unknown as Db).update(issueId, { status: "cancelled" }); + return card; + }); + await quiet(); expect((await stored(terminal.id)).status).toBe("expired"); + expect(provider).not.toHaveBeenCalled(); expect(wakeup).not.toHaveBeenCalled(); + }); + + it("rechecks hidden state under the acceptance lock when visibility changes during a provider read", async () => { + let release!: () => void, observed!: () => void; + const barrier = new Promise(resolve => { release = resolve; }); + const observing = new Promise(resolve => { observed = resolve; }); + provider.mockImplementationOnce(async () => { observed(); await barrier; return "merged"; }); + await start(); const card = await create(); await vi.advanceTimersByTimeAsync(1000); await observing; + await issueService(db).update(issueId, { hiddenAt: new Date() }); release(); await quiet(); + expect((await stored(card.id)).status).toBe("pending"); expect(wakeup).not.toHaveBeenCalled(); + await issueService(db).update(issueId, { hiddenAt: null }); + await waitFor(async () => expect((await stored(card.id)).status).toBe("accepted")); await quiet(); + expect(wakeup).toHaveBeenCalledTimes(1); expect(errors).toEqual([]); + }); + + it("unrelated pending confirmations do not retain worker SQL or provider work", async () => { + await start(); await quiet(); + await service().create({ id: issueId, companyId }, { ...input(), title: "Approve the plan?", payload: { version: 1, prompt: "Approve the plan?" } }, { agentId }); + await quiet(); expect(provider).not.toHaveBeenCalled(); expect(wakeup).not.toHaveBeenCalled(); + expect(errors).toEqual([]); + }); + + it("awaits an in-flight observation at shutdown and reconstructs its unaccepted card on restart", async () => { + let release!: () => void, observed!: () => void; + const barrier = new Promise(resolve => { release = resolve; }); + const observing = new Promise(resolve => { observed = resolve; }); + provider.mockImplementationOnce(async () => { observed(); await barrier; return "merged"; }); + await start(); const card = await create(); await vi.advanceTimersByTimeAsync(1000); await observing; + let stopped = false; + const stopping = coordinator!.stop().then(() => { stopped = true; }); + await delay(10); expect(stopped).toBe(false); release(); await stopping; coordinator = undefined; + expect((await stored(card.id)).status).toBe("pending"); expect(wakeup).not.toHaveBeenCalled(); + const reads = vi.spyOn(db, "select"); + await vi.advanceTimersByTimeAsync(2 * MERGE_CONFIRMATION_RECONCILE_MS); expect(reads).not.toHaveBeenCalled(); reads.mockRestore(); + await start(); await waitFor(async () => expect((await stored(card.id)).status).toBe("accepted")); await quiet(); + expect(wakeup).toHaveBeenCalledTimes(1); expect(errors).toEqual([]); + }); + + it("limits normalized merge hints to the owning company and matching PR and tolerates duplicates", async () => { + provider.mockResolvedValue("open"); const own = await create(); + const foreignCompany = randomUUID(), foreignIssue = randomUUID(); + await db.insert(companies).values({ id: foreignCompany, name: "Other company", issuePrefix: "OTH" }); + await db.insert(issues).values({ id: foreignIssue, companyId: foreignCompany, title: "Other", status: "todo" }); + const foreign = await service().create({ id: foreignIssue, companyId: foreignCompany }, input(), { userId: "board" }); + const local = service(); + const hint = [{ companyId, owner: "PAPERCLIPAI", repo: "paperclip", number: 39 }]; + expect((await local.sweepMergedPullRequestConfirmations(hint)).accepted).toBe(1); + expect((await stored(own.id)).status).toBe("accepted"); expect((await stored(foreign.id)).status).toBe("pending"); + expect(provider).not.toHaveBeenCalled(); + expect((await local.sweepMergedPullRequestConfirmations(hint)).accepted).toBe(0); expect(wakeup).toHaveBeenCalledTimes(1); + }); +}); diff --git a/server/src/__tests__/server-startup-feedback-export.test.ts b/server/src/__tests__/server-startup-feedback-export.test.ts index b645d7437c8..9fc7234b2e6 100644 --- a/server/src/__tests__/server-startup-feedback-export.test.ts +++ b/server/src/__tests__/server-startup-feedback-export.test.ts @@ -180,6 +180,7 @@ const { const heartbeatServiceFactoryMock = vi.fn(() => heartbeatServiceMock); const issueThreadInteractionServiceMock = { sweepSupersededPendingRequestConfirmations: vi.fn(async () => ({ expired: 0 })), + mergedPullRequestConfirmationCandidates: vi.fn(async () => []), sweepMergedPullRequestConfirmations: vi.fn(async () => ({ checked: 0, candidates: 0, @@ -1334,7 +1335,9 @@ describe("startServer feedback export wiring", () => { expect(heartbeatServiceMock.tickAgentTimers).toHaveBeenCalledTimes(1); expect(externalObjectsServiceMock.refreshDueObjectsForActiveCompanies).toHaveBeenCalledTimes(1); - expect(issueThreadInteractionServiceMock.sweepMergedPullRequestConfirmations).toHaveBeenCalledTimes(1); + expect(issueThreadInteractionServiceMock.mergedPullRequestConfirmationCandidates).toHaveBeenCalledTimes(1); + expect(issueThreadInteractionServiceMock.sweepMergedPullRequestConfirmations).not.toHaveBeenCalled(); + expect(deliveryWorkRegisterMock.mock.calls.some(([queue]) => queue === DELIVERY_QUEUES.mergedPullRequestConfirmations)).toBe(true); expect(executionWorkspaceServiceMock.sweepTerminalWorkspaces).toHaveBeenCalledTimes(1); expect(routineServiceMock.tickScheduledTriggers).toHaveBeenCalledTimes(1); const routineWorkerIndex = deliveryWorkRegisterMock.mock.calls.findIndex(([queue]) => queue === DELIVERY_QUEUES.routineSchedules); diff --git a/server/src/index.ts b/server/src/index.ts index ae229f03518..c1804998ef5 100644 --- a/server/src/index.ts +++ b/server/src/index.ts @@ -1,3 +1,4 @@ +import { registerMergeConfirmationWork } from "./services/merged-pr-confirmation-work.js"; import { registerProviderTraceRetentionWork } from "./services/provider-trace-retention-work.js"; import { registerRunEvidenceMigrationWork } from "./services/run-evidence-migration-work.js"; import { registerSkillVersionRetentionWork } from "./services/skill-version-retention-work.js"; @@ -1600,19 +1601,8 @@ async function startServerWithDatabaseTeardown( : config.workspaceCompactionDelayHours * 60 * 60 * 1000, terminalWorkspaceInspectionBackoffMs: TERMINAL_WORKSPACE_RETRY_MS, }); - const scheduleMergedPullRequestConfirmationSweep = () => { - if (isIdleTaskDrainActive() || heartbeatSchedulerStopped) return; - trackHeartbeatSchedulerWork(mergedPullRequestConfirmations - .sweepMergedPullRequestConfirmations() - .then((result) => { - if (result.accepted > 0) { - logger.info(result, "accepted merge confirmations for merged pull requests"); - } - }) - .catch((err) => { - logger.error({ err }, "merged pull-request confirmation sweep failed"); - })); - }; + await registerMergeConfirmationWork(deliveryWork, mergedPullRequestConfirmations, + () => !isIdleTaskDrainActive() && !heartbeatSchedulerStopped && !isWarmStandby()).ready; const gitRestorePointWorker = registerGitRestorePointWork(deliveryWork, gitRestorePointRetentionService(db as any, { storage: storageService, retentionDays: config.gitRestorePointRetentionDays, @@ -2005,7 +1995,6 @@ async function startServerWithDatabaseTeardown( scheduleExternalObjectRefreshSweep(new Date()); if (isIdleTaskDrainActive() || heartbeatSchedulerStopped) return; - scheduleMergedPullRequestConfirmationSweep(); scheduleGitHubConnectionEventPoll(); scheduleGitHubConnectionContinuitySweep(); diff --git a/server/src/services/chat-confirmation-notifications.ts b/server/src/services/chat-confirmation-notifications.ts index e55d8c0d7ac..02cdfbee8c8 100644 --- a/server/src/services/chat-confirmation-notifications.ts +++ b/server/src/services/chat-confirmation-notifications.ts @@ -1,3 +1,4 @@ +import { notifyMergeConfirmationWork } from "./merged-pr-confirmation-notifications.js"; import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js"; export type ChatConfirmationScope = { companyId: string; issueId?: string }; @@ -10,6 +11,7 @@ export async function notifyChatConfirmationWork(tx: object, scope: ChatConfirma /** Interaction resolution can release a callback parked while its human input * was pending. Register on the owning transaction before its resolution write. */ export async function notifyChatConfirmationResolution(tx: object, interaction: ChatConfirmationScope & { kind: string }) { + if (interaction.kind === "request_confirmation") await notifyMergeConfirmationWork(tx, interaction); if (["request_confirmation", "request_checkbox_confirmation", "suggest_tasks", "connection_intent"].includes(interaction.kind)) { await notifyChatConfirmationWork(tx, interaction); } diff --git a/server/src/services/delivery-work-notifications.ts b/server/src/services/delivery-work-notifications.ts index 9195f1fcb12..308a721cbc5 100644 --- a/server/src/services/delivery-work-notifications.ts +++ b/server/src/services/delivery-work-notifications.ts @@ -97,6 +97,7 @@ export const DELIVERY_QUEUES = { browser: "browser-use", email: "email-channels", museMaintenance: "muse-maintenance", + mergedPullRequestConfirmations: "merged-pull-request-confirmations", mcpEvents: "mcp-events", dotEvents: "dot-events", grokDispatch: "grok-dispatch", diff --git a/server/src/services/issue-thread-interactions.ts b/server/src/services/issue-thread-interactions.ts index 59c14618342..b306f637e27 100644 --- a/server/src/services/issue-thread-interactions.ts +++ b/server/src/services/issue-thread-interactions.ts @@ -1,3 +1,4 @@ +import { notifyMergeConfirmationWork, type MergeConfirmationScope } from "./merged-pr-confirmation-notifications.js"; import { notifyNativeStatusReassessment } from "./native-status-reassessment-notifications.js"; import { notifyIssueReviewState } from "./issue-review-state-notifications.js"; import { notifyChatConfirmationResolution } from "./chat-confirmation-notifications.js"; @@ -501,6 +502,7 @@ type IssueResolutionContext = { companyId: string; status: string; executionRunId: string | null; + hiddenAt?: Date | null; workMode: string; assigneeAgentId: string | null; assigneeUserId: string | null; @@ -2074,6 +2076,32 @@ export function issueThreadInteractionService( ); return states; } + async function mergedPullRequestConfirmationCandidates(scope?: MergeConfirmationScope) { + const text = sql`concat_ws(' ', ${issueThreadInteractions.title}, ${issueThreadInteractions.summary}, + ${issueThreadInteractions.payload}->>'prompt', ${issueThreadInteractions.payload}->>'acceptLabel', + ${issueThreadInteractions.payload}->>'detailsMarkdown', ${issueThreadInteractions.payload}->'target'->>'key', + ${issueThreadInteractions.payload}->'target'->>'label', ${issueThreadInteractions.payload}->'target'->>'href')`; + const rows = await db.select({ interaction: issueThreadInteractions, issue: { + id: issues.id, companyId: issues.companyId, projectId: issues.projectId, goalId: issues.goalId, + status: issues.status, assigneeAgentId: issues.assigneeAgentId, + } }).from(issueThreadInteractions).innerJoin(issues, and( + eq(issueThreadInteractions.issueId, issues.id), eq(issueThreadInteractions.companyId, issues.companyId), + )).where(and( + eq(issueThreadInteractions.kind, "request_confirmation"), eq(issueThreadInteractions.status, "pending"), + isNull(issues.hiddenAt), sql`${issues.status} NOT IN ('done', 'cancelled')`, + scope ? eq(issueThreadInteractions.companyId, scope.companyId) : undefined, + scope?.issueId ? eq(issueThreadInteractions.issueId, scope.issueId) : undefined, + scope?.interactionId ? eq(issueThreadInteractions.id, scope.interactionId) : undefined, + sql`${text} ~* '(merge|merging)'`, + sql`${text} ~* '(github\\.com/[^ /]+/[^ /]+/pull/[1-9][0-9]*|[a-z0-9_.-]+/[a-z0-9_.-]+#[1-9][0-9]*)'`, + sql`NOT EXISTS (SELECT 1 FROM ${toolActionRequests} WHERE ${toolActionRequests.interactionId} = ${issueThreadInteractions.id})`, + )); + return rows.flatMap(row => { + const references = getMergeConfirmationPullRequestReferences(row.interaction); + return references.length ? [{ ...row, references }] : []; + }); + } + async function getIdempotentInteraction(args: { issueId: string; companyId: string; @@ -2271,6 +2299,7 @@ export function issueThreadInteractionService( companyId: issues.companyId, status: issues.status, executionRunId: issues.executionRunId, + hiddenAt: issues.hiddenAt, workMode: issues.workMode, assigneeAgentId: issues.assigneeAgentId, assigneeUserId: issues.assigneeUserId, @@ -2309,6 +2338,10 @@ export function issueThreadInteractionService( "Interaction has already been resolved", ); } + if (args.actor.systemId === "system:pr-merged" && + (issueContext.hiddenAt || getMergeConfirmationPullRequestReferences(lockedCurrent).length === 0)) { + throw conflict("Merge confirmation is no longer actionable"); + } await assertRequestConfirmationResolutionAllowedUnderLock( tx as unknown as Db, issueContext, @@ -2979,6 +3012,7 @@ export function issueThreadInteractionService( await emitInteractionResolvedTelemetry(db, interaction); return interaction; }, + mergedPullRequestConfirmationCandidates, sweepMergedPullRequestConfirmations: async ( mergedHints: Array<{ companyId: string; @@ -2986,51 +3020,17 @@ export function issueThreadInteractionService( repo: string; number: number; }> = [], + scope?: MergeConfirmationScope, + signal?: AbortSignal, ) => { - const rows = await db - .select({ - interaction: issueThreadInteractions, - issue: { - id: issues.id, - companyId: issues.companyId, - projectId: issues.projectId, - goalId: issues.goalId, - status: issues.status, - assigneeAgentId: issues.assigneeAgentId, - }, - }) - .from(issueThreadInteractions) - .innerJoin(issues, eq(issueThreadInteractions.issueId, issues.id)) - .where( - and( - eq(issueThreadInteractions.kind, "request_confirmation"), - eq(issueThreadInteractions.status, "pending"), - ), - ); - - const candidates = rows.flatMap((row) => { - const references = getMergeConfirmationPullRequestReferences( - row.interaction, - ); - return references.length > 0 ? [{ ...row, references }] : []; - }); - if (candidates.length === 0) { - return { checked: rows.length, candidates: 0, accepted: 0, woken: 0 }; - } - - const candidateIds = candidates.map(({ interaction }) => interaction.id); - const linkedToolActions = await db - .select({ interactionId: toolActionRequests.interactionId }) - .from(toolActionRequests) - .where(inArray(toolActionRequests.interactionId, candidateIds)); - const toolActionInteractionIds = new Set( - linkedToolActions - .map((row) => row.interactionId) - .filter((value): value is string => Boolean(value)), - ); - const eligible = candidates.filter( - ({ interaction }) => !toolActionInteractionIds.has(interaction.id), - ); + const candidates = await mergedPullRequestConfirmationCandidates(scope ?? + (mergedHints.length === 1 ? { companyId: mergedHints[0]!.companyId } : undefined)); + const eligible = mergedHints.length ? candidates.filter(candidate => mergedHints.some(hint => + hint.companyId === candidate.issue.companyId && candidate.references.some(reference => + hint.owner.toLowerCase() === reference.owner.toLowerCase() && + hint.repo.toLowerCase() === reference.repo.toLowerCase() && hint.number === reference.number), + )) : candidates; + if (eligible.length === 0) return { checked: 0, candidates: 0, accepted: 0, woken: 0 }; const checkedAt = now().getTime(); const cacheTtlMs = opts.pullRequestCacheTtlMs ?? 5 * 60 * 1000; @@ -3065,6 +3065,9 @@ export function issueThreadInteractionService( const refreshedStates = await resolvePullRequestStates([ ...uniqueReferences.values(), ]); + if (scope && [...refreshedStates.values()].some(state => state === "unknown")) { + throw new Error("Merge confirmation provider state is unavailable"); + } for (const [key, state] of refreshedStates) { setBoundedPullRequestCacheEntry(pullRequestStateCache, key, { state, @@ -3075,6 +3078,7 @@ export function issueThreadInteractionService( let accepted = 0; let woken = 0; for (const candidate of eligible) { + if (signal?.aborted) break; const allMerged = candidate.references.every((reference) => { const key = `${candidate.issue.companyId}:${reference.owner.toLowerCase()}/${reference.repo.toLowerCase()}#${reference.number}`; return pullRequestStateCache.get(key)?.state === "merged"; @@ -3111,12 +3115,16 @@ export function issueThreadInteractionService( if (resolved.interaction.status !== "accepted") continue; accepted += 1; - const wakeIssue = resolved.continuationIssue ?? candidate.issue; + const wakeIssue = resolved.continuationIssue ?? await db.select({ + id: issues.id, assigneeAgentId: issues.assigneeAgentId, status: issues.status, + }).from(issues).where(and(eq(issues.id, candidate.issue.id), + eq(issues.companyId, candidate.issue.companyId), isNull(issues.hiddenAt))).then(rows => rows[0]); const shouldWake = resolved.interaction.continuationPolicy === "wake_assignee" || resolved.interaction.continuationPolicy === "wake_assignee_on_accept"; if ( !opts.wakeup || + !wakeIssue || !shouldWake || !wakeIssue.assigneeAgentId || isTerminalIssueStatus(wakeIssue.status) @@ -3158,7 +3166,7 @@ export function issueThreadInteractionService( } return { - checked: rows.length, + checked: candidates.length, candidates: eligible.length, accepted, woken, @@ -3909,6 +3917,7 @@ export function issueThreadInteractionService( } else { await notifyNativeStatusReassessment(tx, { companyId: issue.companyId, issueId: issue.id }); } + if (data.kind === "request_confirmation") await notifyMergeConfirmationWork(tx, { companyId: issue.companyId, issueId: issue.id }); const [row] = await tx .insert(issueThreadInteractions) .values({ diff --git a/server/src/services/issues.ts b/server/src/services/issues.ts index 42c65bb5d02..4178dc1a35c 100644 --- a/server/src/services/issues.ts +++ b/server/src/services/issues.ts @@ -1,3 +1,4 @@ +import { notifyMergeConfirmationWork } from "./merged-pr-confirmation-notifications.js"; import { notifyExternalConversationSettlement } from "./external-conversation-settlement-notifications.js"; import { notifyNativeRecoveryNotices } from "./native-recovery-notice-notifications.js"; import { notifyNativeRunProjection } from "./native-run-projection-notifications.js"; @@ -11491,6 +11492,9 @@ export function issueService(db: Db) { || issueData.parentId !== undefined || issueData.executionWorkspaceId !== undefined) { await notifyTerminalWorkspaceWork(tx, `issue:${id}`); } + if (patch.status !== undefined || patch.hiddenAt !== undefined) { + await notifyMergeConfirmationWork(tx, { companyId: receiptExisting.companyId, issueId: id }); + } if (patch.status !== undefined || patch.assigneeAgentId !== undefined || patch.assigneeUserId !== undefined || patch.hiddenAt !== undefined || patch.conversationAgentId !== undefined) { await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers); diff --git a/server/src/services/merged-pr-confirmation-notifications.ts b/server/src/services/merged-pr-confirmation-notifications.ts new file mode 100644 index 00000000000..34df4ecffb0 --- /dev/null +++ b/server/src/services/merged-pr-confirmation-notifications.ts @@ -0,0 +1,7 @@ +import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js"; + +export type MergeConfirmationScope = { companyId: string; issueId?: string; interactionId?: string }; + +export function notifyMergeConfirmationWork(tx: object, scope: MergeConfirmationScope) { + return notifyDeliveryWork(tx, DELIVERY_QUEUES.mergedPullRequestConfirmations, JSON.stringify({ companyId: scope.companyId, issueId: scope.issueId, interactionId: scope.interactionId })); +} diff --git a/server/src/services/merged-pr-confirmation-work.ts b/server/src/services/merged-pr-confirmation-work.ts new file mode 100644 index 00000000000..bfccaeea79f --- /dev/null +++ b/server/src/services/merged-pr-confirmation-work.ts @@ -0,0 +1,30 @@ +import type { createDeliveryWorkCoordinator } from "./delivery-work-coordinator.js"; +import { DELIVERY_QUEUES } from "./delivery-work-notifications.js"; +import type { issueThreadInteractionService } from "./issue-thread-interactions.js"; +import type { MergeConfirmationScope } from "./merged-pr-confirmation-notifications.js"; +import { registerScopedDeadlineWork } from "./scoped-deadline-work.js"; + +export const MERGE_CONFIRMATION_RECONCILE_MS = 5 * 60_000; + +export function registerMergeConfirmationWork( + coordinator: ReturnType, + service: Pick, + "mergedPullRequestConfirmationCandidates" | "sweepMergedPullRequestConfirmations">, + canRun: () => boolean, +) { + return registerScopedDeadlineWork(coordinator, { + queue: DELIVERY_QUEUES.mergedPullRequestConfirmations, + decodeScope: hint => hint === "*" ? undefined : JSON.parse(hint) as MergeConfirmationScope, + matches: (entry, scope?: MergeConfirmationScope) => !scope || (entry.companyId === scope.companyId && + (!scope.issueId || entry.issueId === scope.issueId) && (!scope.interactionId || entry.id === scope.interactionId)), + load: async (scope?: MergeConfirmationScope) => (await service.mergedPullRequestConfirmationCandidates(scope)) + .map(({ interaction }) => ({ id: interaction.id, companyId: interaction.companyId, issueId: interaction.issueId, dueAt: Date.now() })), + check: async (entry, signal) => { + const scope = { companyId: entry.companyId, issueId: entry.issueId, interactionId: entry.id }; + await service.sweepMergedPullRequestConfirmations([], scope, signal); + if (signal?.aborted) return entry; + const remaining = await service.mergedPullRequestConfirmationCandidates(scope); + return remaining.length ? { ...entry, dueAt: Date.now() + MERGE_CONFIRMATION_RECONCILE_MS } : null; + }, + }, canRun); +} From 50cd007005c7971612d6543da30aee0b3bf94d2b Mon Sep 17 00:00:00 2001 From: Dotta Date: Fri, 30 Oct 2026 16:24:46 -0500 Subject: [PATCH 05/70] perf(external-objects): schedule scoped refresh obligations Co-Authored-By: Paperclip --- .../2026-10-30-external-objects-sleep.md | 92 +++++ .../external-object-refresh-work.test.ts | 334 ++++++++++++++++++ .../external-objects-service.test.ts | 9 +- .../github-connection-events.test.ts | 20 +- server/src/index.ts | 17 +- server/src/services/companies.ts | 3 + server/src/services/company-deletion.ts | 2 + .../services/delivery-work-notifications.ts | 1 + .../external-object-refresh-notifications.ts | 7 + server/src/services/external-objects.ts | 183 +++++++--- .../src/services/github-connection-events.ts | 44 ++- server/src/services/instance-settings.ts | 2 + server/src/services/issues.ts | 2 + server/src/services/plugin-registry.ts | 43 ++- 14 files changed, 659 insertions(+), 100 deletions(-) create mode 100644 doc/plans/2026-10-30-external-objects-sleep.md create mode 100644 server/src/__tests__/external-object-refresh-work.test.ts create mode 100644 server/src/services/external-object-refresh-notifications.ts diff --git a/doc/plans/2026-10-30-external-objects-sleep.md b/doc/plans/2026-10-30-external-objects-sleep.md new file mode 100644 index 00000000000..639bbaea343 --- /dev/null +++ b/doc/plans/2026-10-30-external-objects-sleep.md @@ -0,0 +1,92 @@ +# External object refresh deadlines + +External-object refresh uses the delivery coordinator. The server registers the +service in both heartbeat-scheduler configurations. It no longer calls the +company-wide aggregate sweep on each heartbeat tick. + +`external-objects.ts` owns automatic eligibility, resolver selection, refresh +claims, provider results and the next deadline. `index.ts` supplies registration +and admission gates. The existing scoped deadline helper owns wake coalescing, +page reconciliation, fair company selection, retry backoff and shutdown. + +## Durable obligations and quiet states + +An automatic refresh needs an active company, a retained mention in that company, +a non-null `next_refresh_at` and an available resolver. Injected resolvers, +including custom `url/link` resolvers, remain supported. A plugin resolver needs a +ready plugin with `external.objects.read`, a matching provider/object declaration, +and no company setting that disables it. Built-in resolvers remain available +independently of optional plugin support. + +Terminal objects are passive. The existing exception remains: GitHub +`not_found` snapshots without `data.probeVerified=true` need a repository-visibility +probe before their terminal verdict is definitive. A null deadline retains its +existing never-refresh meaning. Registration does not change stored timestamps. + +The worker reconstructs eligible deadlines in pages of at most 250 rows. A +company notification reads that company only. A due callback reads only its +object. Pages select scheduling columns and retain compact deadlines in memory. +Dispatch uses fair batches of at most 50 objects. The deadline is the later of +`next_refresh_at` and the current refresh lease expiry. Lease renewals and atomic +refresh tokens continue to exclude another claimant. + +Unsupported generic links, orphan objects, disabled plugins, empty configuration +and retained terminal history do not schedule fallback discovery. Manual refresh +retains its existing `no_resolver` result. Actual provider failures retain their +saved retry deadline. Unexpected failures retain scoped coordinator backoff; +restart reconstructs the saved object and any unexpired claim. + +The supported-resolver and retained-reference approach follows the related draft +[PR #17298](https://github.com/paperclipai/paperclip/pull/17298) by +[@kaluli123123](https://github.com/kaluli123123). This change adapts that eligibility +pattern to scoped scheduling, adds dynamic plugin/company permission predicates, +and fences automatic result persistence. It does not modify that draft PR. + +## Producer and transaction audit + +| Mutation | Owner and activation | +| --- | --- | +| Issue title/description, comment and document references | Existing sync methods register company intent before mention replacement and object upsert. Routes, runtime tools, adapters and plugin paths reuse these methods. | +| Comment/document disappearance | Sync registers reconstruction before deleting obsolete mentions. | +| Last mention removal and reattachment | Mention replacement invalidates company deadlines. Retained objects stay available for reattachment. | +| Issue deletion | Issue service registers its company before cascading mention deletion. | +| Company status/archive/deletion | Company services register company reconstruction before their status or deletion writes. | +| Feature enable/disable | Instance settings registers global reconstruction before the experimental-settings write. | +| Plugin manifest/status/uninstall | Plugin registry registers reconstruction before lifecycle writes. Installed plugins acquire support through the ordinary ready transition. | +| Plugin company configuration and enabled setting | Registry writes register only the affected company. There is no plugin-readiness poll. | +| Manual and automatic resolver result | Result persistence registers object intent in the same transaction. Automatic persistence rechecks reference and resolver eligibility. Lost support releases the worker's own claim without publishing the stale result. | +| Normalized GitHub ingress | The event service registers company intent with the snapshot write. It clears the old refresh token, so an older provider response cannot overwrite that event. | +| Import and restore | No independent object-table importer exists in the source baseline. Supported reference ingestion uses the ordinary sync boundary. Startup reconstructs restored durable objects. | + +Notifications are awaited before writes. Caller-owned and nested transactions +retain the outer commit boundary through the existing database-work protocol. +Rollback does not publish the write. Notifications received during a scan or an +active resolver remain dirty until a successful reconciliation. + +The supported live writer model is one shared `createDb` signal owner, including +its transactions and child pools. These process-local signals do not cover +independent SQL clients or other replicas. Such writers must invoke the supported +service boundary or arrange explicit reconstruction/restart. Existing normalized +GitHub ingress uses the supported boundary. No permanent database listener or +periodic global safety scan is added. + +## Acceptance evidence + +The focused suite uses real PostgreSQL with a controlled simulated provider. +Ordinary sync and registry/settings writes create work; the fixture does not +pre-create the tested resolved outcome. Assertions independently read saved +objects. Tests measure owner reads and provider calls across former tick windows. + +Coverage includes empty and terminal quiet, unsupported links, orphan removal and +reattachment, feature/company eligibility, plugin readiness and permission loss +during a held provider call, exact and earlier deadlines, durable provider retry, +rollback and outer commit after nested savepoint release, startup and paginated +reconstruction, inactive companies, never-refresh rows, first work during drain, +warm standby, failed refresh after restart, lease renewal, duplicate exclusion, +and shutdown while a provider call remains active. The GitHub ingress suite also +checks a real normalized event superseding a held resolver response. + +The conservative external-object host-sleep blocker remains. An in-process +deadline cannot wake stopped compute. This change proves worker quiet and +reactivation in the tested process; it does not establish Cloud alarm coverage, +host/database suspension or deployed savings. diff --git a/server/src/__tests__/external-object-refresh-work.test.ts b/server/src/__tests__/external-object-refresh-work.test.ts new file mode 100644 index 00000000000..8d1f5cdc00c --- /dev/null +++ b/server/src/__tests__/external-object-refresh-work.test.ts @@ -0,0 +1,334 @@ +import { randomUUID } from "node:crypto"; +import { setTimeout as delay } from "node:timers/promises"; +import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest"; +import { eq, sql } from "drizzle-orm"; +import { activityLog, companies, createDb, externalObjects, issues, plugins } from "@paperclipai/db"; +import { externalObjectService, type ExternalObjectResolveResult } from "../services/external-objects.js"; +import type { PaperclipPluginManifestV1 } from "@paperclipai/shared"; +import type { PluginWorkerManager } from "../services/plugin-worker-manager.js"; +import { pluginRegistryService } from "../services/plugin-registry.js"; +import { companyService } from "../services/companies.js"; +import { instanceSettingsService } from "../services/instance-settings.js"; +import { notifyExternalObjectRefresh } from "../services/external-object-refresh-notifications.js"; +import { createDeliveryWorkCoordinator } from "../services/delivery-work-coordinator.js"; +import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js"; +import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; + +const url = resolveExternalTestDatabaseUrl("PAPERCLIP_EXTERNAL_OBJECT_REFRESH_TEST_DATABASE_URL"); +const support = url ? { supported: true } : await getEmbeddedPostgresTestSupport(); +const terminal: ExternalObjectResolveResult = { ok: true, snapshot: { + statusKey: "done", statusCategory: "succeeded", statusTone: "success", isTerminal: true, +} }; +const active: ExternalObjectResolveResult = { ok: true, snapshot: { + statusKey: "open", statusCategory: "open", statusTone: "neutral", isTerminal: false, ttlSeconds: 60, +} }; + +(support.supported ? describe : describe.skip)("external object refresh work (real PostgreSQL, simulated provider)", () => { + let db: ReturnType; + let temp: Awaited> | undefined; + let coordinator: ReturnType | undefined; + let admitted = true; + let reconcile = true; + let enabled = true; + const provider = vi.fn(async (): Promise => terminal); + const errors = vi.fn(); + const companyIds: string[] = []; + beforeAll(async () => { + if (!url) temp = await startEmbeddedPostgresTestDatabase("external-refresh-work-"); + db = createDb(url ?? temp!.connectionString, { maxConnections: 6 }); + await db.execute(sql`truncate table ${companies} cascade`); + }); + afterEach(async () => { + await coordinator?.stop(); coordinator = undefined; + vi.useRealTimers(); vi.restoreAllMocks(); + for (const id of companyIds.splice(0)) { + await db.delete(activityLog).where(eq(activityLog.companyId, id)); + await db.delete(issues).where(eq(issues.companyId, id)); + await db.delete(externalObjects).where(eq(externalObjects.companyId, id)); + await db.delete(companies).where(eq(companies.id, id)); + } + await db.delete(plugins); + admitted = true; reconcile = true; enabled = true; errors.mockClear(); + provider.mockReset().mockResolvedValue(terminal); + }); + afterAll(async () => { await db?.$client.end({ timeout: 1 }); await temp?.cleanup(); }); + + function service() { + return externalObjectService(db, { github: false, enabled: () => enabled, + resolvers: [{ providerKey: "url", resolve: provider }] }); + } + async function start(svc = service()) { + coordinator = createDeliveryWorkCoordinator({ owner: db, canRun: () => admitted, + canReconcile: () => reconcile, onError: errors }); + const worker = svc.registerRefreshWork(coordinator, () => admitted); + await worker.ready; + return worker; + } + async function fixture(status = "active") { + const id = randomUUID(); companyIds.push(id); + await db.insert(companies).values({ id, name: "Refresh test", status, + issuePrefix: `E${id.replaceAll("-", "").slice(0, 8)}` }); + const [issue] = await db.insert(issues).values({ companyId: id, title: "Reference", + description: `https://example.com/${id}`, status: "todo" }).returning(); + return { companyId: id, issueId: issue!.id }; + } + async function row(companyId: string) { + return (await db.select().from(externalObjects).where(eq(externalObjects.companyId, companyId)))[0]!; + } + async function settle(predicate: () => Promise | boolean) { + for (let i = 0; i < 200; i++) { + await vi.advanceTimersByTimeAsync(0); await delay(5); + if (await predicate()) return; + } + throw new Error("refresh worker did not settle"); + } + async function quiet() { await settle(() => coordinator!.nextWakeAt() === null); } + function clock() { vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout", "setInterval", "clearInterval"], shouldAdvanceTime: true }); } + async function change(companyId: string, patch: Partial) { + await db.transaction(async tx => { + await notifyExternalObjectRefresh(tx, companyId); + await tx.update(externalObjects).set(patch).where(eq(externalObjects.companyId, companyId)); + }); + } + + it("captures the old configured-empty aggregate SQL cost", async () => { + const svc = service(); await fixture(); + const selects = vi.spyOn(db, "select"); + for (let i = 0; i < 3; i++) await svc.refreshDueObjectsForActiveCompanies(); + expect(selects).toHaveBeenCalledTimes(6); + expect(provider).not.toHaveBeenCalled(); + }); + + it("activates saved work when the ordinary feature and company settings enable it", async () => { + clock(); const settings = instanceSettingsService(db); + await settings.updateExperimental({ enableExternalObjects: true }); + const svc = externalObjectService(db, { github: false, + enabled: async () => (await settings.getExperimental()).enableExternalObjects, + resolvers: [{ providerKey: "url", resolve: provider }] }); + const f = await fixture("paused"); await svc.syncIssue(f.issueId); + await settings.updateExperimental({ enableExternalObjects: false }); + await start(svc); await quiet(); + await companyService(db).update(f.companyId, { status: "active" }); await quiet(); + expect(provider).not.toHaveBeenCalled(); + await settings.updateExperimental({ enableExternalObjects: true }); + await settle(async () => (await row(f.companyId)).isTerminal); await quiet(); + expect(provider).toHaveBeenCalledOnce(); + }); + + it("settles empty, configured-idle and retained terminal history without recurring SQL or provider calls", async () => { + clock(); + const svc = service(); await start(svc); await quiet(); + const selects = vi.spyOn(db, "select"); + await vi.advanceTimersByTimeAsync(15 * 60_000); + expect(selects).not.toHaveBeenCalled(); expect(provider).not.toHaveBeenCalled(); + const f = await fixture(); await svc.syncIssue(f.issueId); + await settle(async () => (await row(f.companyId))?.isTerminal === true); await quiet(); + expect((await row(f.companyId)).statusKey).toBe("done"); + selects.mockClear(); provider.mockClear(); + await vi.advanceTimersByTimeAsync(15 * 60_000); + expect(selects).not.toHaveBeenCalled(); expect(provider).not.toHaveBeenCalled(); + expect(errors).not.toHaveBeenCalled(); + }); + + it("registers exact future deadlines, preempts them on earlier writes and cancels terminal work", async () => { + clock(); provider.mockResolvedValue(active); + const svc = service(); await start(svc); + const f = await fixture(); await svc.syncIssue(f.issueId); + await settle(() => provider.mock.calls.length === 1); + await settle(async () => (await row(f.companyId)).lastResolvedAt !== null); + const saved = await row(f.companyId); + await settle(() => coordinator!.nextWakeAt() === saved.nextRefreshAt!.getTime()); + const earlier = new Date(Date.now() + 1000); await change(f.companyId, { nextRefreshAt: earlier }); + await settle(() => coordinator!.nextWakeAt() === earlier.getTime()); + await vi.advanceTimersByTimeAsync(999); expect(provider).toHaveBeenCalledTimes(1); + provider.mockResolvedValue(terminal); await vi.advanceTimersByTimeAsync(1); + await settle(async () => (await row(f.companyId)).isTerminal); await quiet(); + await vi.advanceTimersByTimeAsync(120_000); expect(provider).toHaveBeenCalledTimes(2); + }); + + it("keeps disabled and paused company work passive and reconstructs on eligibility mutation", async () => { + clock(); enabled = false; + const svc = service(); await start(svc); await quiet(); + const f = await fixture("paused"); + enabled = true; await svc.syncIssue(f.issueId); await quiet(); + await vi.advanceTimersByTimeAsync(600_000); expect(provider).not.toHaveBeenCalled(); + await db.transaction(async tx => { + await notifyExternalObjectRefresh(tx, f.companyId); + await tx.update(companies).set({ status: "active" }).where(eq(companies.id, f.companyId)); + }); + await settle(async () => (await row(f.companyId)).isTerminal); await quiet(); + expect(provider).toHaveBeenCalledOnce(); + }); + + it("waits for the outer commit and leaves rolled-back ingestion quiet", async () => { + clock(); const svc = service(); await start(svc); await quiet(); + const f = await fixture(); + await expect(db.transaction(async tx => { + await svc.syncIssue(f.issueId, tx); throw new Error("rollback"); + })).rejects.toThrow("rollback"); + await quiet(); expect(await row(f.companyId)).toBeUndefined(); expect(provider).not.toHaveBeenCalled(); + let release!: () => void; const held = new Promise(resolve => { release = resolve; }); + let nestedDone = false; + const commit = db.transaction(async tx => { + await tx.transaction(async nested => { await svc.syncIssue(f.issueId, nested); }); + nestedDone = true; await held; + }); + await settle(() => nestedDone); await vi.advanceTimersByTimeAsync(60_000); + expect(provider).not.toHaveBeenCalled(); release(); await commit; + await settle(async () => (await row(f.companyId))?.isTerminal); await quiet(); + expect(provider).toHaveBeenCalledOnce(); + }); + + it("retains first work during drain and recovers a provider failure after restart", async () => { + clock(); const svc = service(); await start(svc); await quiet(); + admitted = false; const f = await fixture(); await svc.syncIssue(f.issueId); + const selects = vi.spyOn(db, "select"); + await vi.advanceTimersByTimeAsync(90_000); expect(provider).not.toHaveBeenCalled(); + selects.mockClear(); await vi.advanceTimersByTimeAsync(90_000); expect(selects).not.toHaveBeenCalled(); + admitted = true; provider.mockRejectedValueOnce(new Error("simulated transport failure")); + await vi.advanceTimersByTimeAsync(30_000); await settle(() => errors.mock.calls.length === 1); + const claimed = await row(f.companyId); expect(claimed.refreshToken).not.toBeNull(); + await coordinator!.stop(); coordinator = undefined; await start(svc); + const due = claimed.refreshStartedAt!.getTime() + 300_000; + expect(coordinator!.nextWakeAt()).toBe(due); + await vi.advanceTimersByTimeAsync(due - Date.now() - 1); expect(provider).toHaveBeenCalledTimes(1); + await vi.advanceTimersByTimeAsync(1); + await settle(async () => (await row(f.companyId)).isTerminal); await quiet(); + expect(provider).toHaveBeenCalledTimes(2); + }); + + it("leaves unsupported URLs passive and cancels orphan work until ordinary reattachment", async () => { + clock(); const unsupported = externalObjectService(db, { github: false }); + await start(unsupported); const f = await fixture(); await unsupported.syncIssue(f.issueId); await quiet(); + const selects = vi.spyOn(db, "select"); + await vi.advanceTimersByTimeAsync(900_000); expect(selects).not.toHaveBeenCalled(); + expect((await row(f.companyId)).lastResolvedAt).toBeNull(); + await coordinator!.stop(); coordinator = undefined; + provider.mockResolvedValue(active); const svc = service(); await start(svc); + await settle(async () => (await row(f.companyId)).lastResolvedAt !== null); + await db.update(issues).set({ description: null }).where(eq(issues.id, f.issueId)); + await svc.syncIssue(f.issueId); await quiet(); + selects.mockClear(); provider.mockClear(); + await vi.advanceTimersByTimeAsync(900_000); + expect(selects).not.toHaveBeenCalled(); expect(provider).not.toHaveBeenCalled(); + provider.mockResolvedValue(terminal); + await db.update(issues).set({ description: `https://example.com/${f.companyId}` }).where(eq(issues.id, f.issueId)); + await svc.syncIssue(f.issueId); + await settle(async () => (await row(f.companyId)).isTerminal); await quiet(); + expect(provider).toHaveBeenCalledOnce(); + }); + + it("activates a newly ready plugin and fences a result after company permission is withdrawn", async () => { + clock(); const registry = pluginRegistryService(db); + const manifest: PaperclipPluginManifestV1 = { + id: `paperclip.refresh-${randomUUID()}`, apiVersion: 1, version: "1.0.0", displayName: "Refresh fixture", + description: "Simulated external object provider", author: "Paperclip", categories: ["connector"], + capabilities: ["external.objects.detect", "external.objects.read"], entrypoints: { worker: "worker.js" }, + objectReferences: [{ providerKey: "tracker", displayName: "Tracker", objectTypes: ["ticket"], urlPatterns: ["https://example.com/:id"] }], + }; + const plugin = await registry.install({ packageName: "refresh-fixture" }, manifest); + let finish!: (result: ExternalObjectResolveResult) => void; + const calls = vi.fn(async (_id: string, method: string) => { + if (method === "detectExternalObjects") return { detections: [] }; + if (calls.mock.calls.filter(call => call[1] === "resolveExternalObject").length === 1) + return new Promise(resolve => { finish = resolve; }); + return terminal; + }); + const svc = externalObjectService(db, { github: false, + pluginWorkerManager: { call: calls } as unknown as PluginWorkerManager, + detectors: [{ key: "fixture", detect: ({ urls }) => urls.map(canonical => ({ canonical, + detectorKey: "fixture", providerKey: "tracker", objectType: "ticket", externalId: canonical.canonicalIdentityHash })) }], + }); + await start(svc); const f = await fixture(); await svc.syncIssue(f.issueId); await quiet(); + const selects = vi.spyOn(db, "select"); + await vi.advanceTimersByTimeAsync(900_000); expect(selects).not.toHaveBeenCalled(); expect(calls).not.toHaveBeenCalled(); + await registry.updateStatus(plugin!.id, { status: "ready" }); + await settle(() => typeof finish === "function"); + await registry.upsertCompanySettings(plugin!.id, f.companyId, { enabled: false, settingsJson: {} }); + finish(terminal); + await settle(async () => (await row(f.companyId)).refreshToken === null); await quiet(); + expect((await row(f.companyId)).isTerminal).toBe(false); + selects.mockClear(); calls.mockClear(); + await vi.advanceTimersByTimeAsync(900_000); expect(selects).not.toHaveBeenCalled(); expect(calls).not.toHaveBeenCalled(); + calls.mockImplementation(async () => terminal); + await registry.upsertCompanySettings(plugin!.id, f.companyId, { enabled: true, settingsJson: {} }); + await settle(async () => (await row(f.companyId)).isTerminal); await quiet(); + expect(calls).toHaveBeenCalledOnce(); + }); + + it("keeps warm standby startup SQL-free and resumes a first committed producer", async () => { + clock(); admitted = false; reconcile = false; + const svc = service(); const selects = vi.spyOn(db, "select"); + await start(svc); await vi.advanceTimersByTimeAsync(90_000); + expect(selects).not.toHaveBeenCalled(); + const f = await fixture(); await svc.syncIssue(f.issueId); + selects.mockClear(); await vi.advanceTimersByTimeAsync(90_000); + expect(selects).not.toHaveBeenCalled(); expect(provider).not.toHaveBeenCalled(); + admitted = true; reconcile = true; + await vi.advanceTimersByTimeAsync(30_000); + await settle(async () => (await row(f.companyId))?.isTerminal); await quiet(); + expect(provider).toHaveBeenCalledOnce(); + }); + + it("renews an active provider lease so another service cannot duplicate its action", async () => { + clock(); const svc = service(); await start(svc); + let finish!: (result: ExternalObjectResolveResult) => void; + provider.mockImplementationOnce(() => new Promise(resolve => { finish = resolve; })); + const f = await fixture(); await svc.syncIssue(f.issueId); + await settle(() => provider.mock.calls.length === 1); + const claimed = await row(f.companyId); + await vi.advanceTimersByTimeAsync(60_000); + await settle(async () => (await row(f.companyId)).refreshStartedAt!.getTime() > claimed.refreshStartedAt!.getTime()); + const competing = await service().refreshObject(claimed.id, { companyId: f.companyId, + force: true, now: new Date(claimed.refreshStartedAt!.getTime() + 301_000) }); + expect(competing.reason).toBe("refresh_in_progress"); expect(provider).toHaveBeenCalledOnce(); + finish(terminal); await settle(async () => (await row(f.companyId)).isTerminal); await quiet(); + }); + + it("uses durable provider retry deadlines and waits for in-flight shutdown without overlap", async () => { + clock(); const svc = service(); await start(svc); + provider.mockResolvedValueOnce({ ok: false, liveness: "rate_limited", errorCode: "slow_down", retryAfterSeconds: 5 }); + const f = await fixture(); await svc.syncIssue(f.issueId); + await settle(async () => (await row(f.companyId))?.lastErrorCode === "slow_down"); + const saved = await row(f.companyId); + await settle(() => coordinator!.nextWakeAt() === saved.nextRefreshAt!.getTime()); + expect(saved.refreshToken).toBeNull(); + await vi.advanceTimersByTimeAsync(saved.nextRefreshAt!.getTime() - Date.now() - 1); + expect(provider).toHaveBeenCalledOnce(); + let finish!: (result: ExternalObjectResolveResult) => void; + provider.mockImplementationOnce(() => new Promise(resolve => { finish = resolve; })); + await vi.advanceTimersByTimeAsync(1); await settle(() => provider.mock.calls.length === 2); + await svc.syncIssue(f.issueId); await vi.advanceTimersByTimeAsync(0); + expect(provider).toHaveBeenCalledTimes(2); + let stopped = false; const stop = coordinator!.stop().then(() => { stopped = true; }); + await delay(10); expect(stopped).toBe(false); + finish(terminal); await stop; coordinator = undefined; + const selects = vi.spyOn(db, "select"); + await vi.advanceTimersByTimeAsync(600_000); + expect(selects).not.toHaveBeenCalled(); expect(provider).toHaveBeenCalledTimes(2); + }); + + it("reconstructs paginated multi-company work and never invokes inactive or never-refresh objects", async () => { + clock(); const svc = service(); const a = await fixture(); const b = await fixture(); const inactive = await fixture("paused"); + for (const companyId of [b.companyId, inactive.companyId]) { + await db.insert(externalObjects).values(Array.from({ length: 255 }, (_, index) => ({ + companyId, providerKey: "url", objectType: "link", externalId: String(index), + sanitizedCanonicalUrl: `https://example.com/${companyId}/${index}`, canonicalIdentityHash: `${companyId}-${index}`, + nextRefreshAt: null, + }))); + } + await db.update(issues).set({ description: Array.from({ length: 255 }, (_, index) => + `https://example.com/${a.companyId}/${index}`).join(" ") }).where(eq(issues.id, a.issueId)); + await svc.syncIssue(a.issueId); + const future = new Date(Date.now() + 60_000); + await db.update(externalObjects).set({ nextRefreshAt: future }).where(eq(externalObjects.companyId, a.companyId)); + await svc.syncIssue(b.issueId); + await start(svc); await settle(() => provider.mock.calls.length === 1); + await settle(() => coordinator!.nextWakeAt() === future.getTime()); + expect((await row(a.companyId)).nextRefreshAt).toEqual(future); + const companiesSeen = provider.mock.calls.map(call => (call as unknown as [{ companyId: string }])[0].companyId); + expect(companiesSeen).toEqual([b.companyId]); + await change(a.companyId, { isTerminal: true }); await quiet(); + await vi.advanceTimersByTimeAsync(600_000); expect(provider).toHaveBeenCalledOnce(); + }); +}); diff --git a/server/src/__tests__/external-objects-service.test.ts b/server/src/__tests__/external-objects-service.test.ts index e9022599779..e50e298b466 100644 --- a/server/src/__tests__/external-objects-service.test.ts +++ b/server/src/__tests__/external-objects-service.test.ts @@ -11,6 +11,7 @@ import { issues, plugins, } from "@paperclipai/db"; +import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js"; import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase, @@ -27,7 +28,8 @@ import type { PluginWorkerManager } from "../services/plugin-worker-manager.js"; import { createGitHubExternalObjectProvider } from "../services/github-external-object-provider.js"; const embeddedPostgresSupport = await getEmbeddedPostgresTestSupport(); -const describeEmbeddedPostgres = embeddedPostgresSupport.supported ? describe : describe.skip; +const externalDatabaseUrl = resolveExternalTestDatabaseUrl("PAPERCLIP_EXTERNAL_OBJECTS_TEST_DATABASE_URL"); +const describeEmbeddedPostgres = externalDatabaseUrl || embeddedPostgresSupport.supported ? describe : describe.skip; if (!embeddedPostgresSupport.supported) { console.warn( @@ -709,8 +711,8 @@ describeEmbeddedPostgres("externalObjectService", () => { let tempDb: Awaited> | null = null; beforeAll(async () => { - tempDb = await startEmbeddedPostgresTestDatabase("paperclip-external-objects-"); - db = createDb(tempDb.connectionString); + if (!externalDatabaseUrl) tempDb = await startEmbeddedPostgresTestDatabase("paperclip-external-objects-"); + db = createDb(externalDatabaseUrl ?? tempDb!.connectionString, { maxConnections: 6 }); }, 20_000); afterEach(async () => { @@ -725,6 +727,7 @@ describeEmbeddedPostgres("externalObjectService", () => { }); afterAll(async () => { + await db?.$client.end({ timeout: 1 }); await tempDb?.cleanup(); }); diff --git a/server/src/__tests__/github-connection-events.test.ts b/server/src/__tests__/github-connection-events.test.ts index 7c7b7025243..0417a434718 100644 --- a/server/src/__tests__/github-connection-events.test.ts +++ b/server/src/__tests__/github-connection-events.test.ts @@ -1,3 +1,4 @@ +import { externalObjectService } from "../services/external-objects.js"; import { startToolHealthWorker } from "./helpers/tool-health-worker.js"; import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js"; import { randomUUID } from "node:crypto"; @@ -35,7 +36,7 @@ describeEmbeddedPostgres("GitHub connection event delivery", () => { beforeAll(async () => { if (!externalTestDatabaseUrl) tempDb = await startEmbeddedPostgresTestDatabase("paperclip-github-events-"); if (externalTestDatabaseUrl) await applyPendingMigrations(externalTestDatabaseUrl); - db = createDb(externalTestDatabaseUrl ?? tempDb!.connectionString); + db = createDb(externalTestDatabaseUrl ?? tempDb!.connectionString, { maxConnections: 6 }); }, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS); afterEach(async () => { @@ -171,12 +172,27 @@ describeEmbeddedPostgres("GitHub connection event delivery", () => { expect(connector.leaseEvents).toHaveBeenCalledTimes(5); } + let releaseRefresh!: () => void; + let markStarted!: () => void; + const started = new Promise(resolve => { markStarted = resolve; }); + const held = new Promise(resolve => { releaseRefresh = resolve; }); + const resolver = externalObjectService(db, { github: false, resolvers: [{ + providerKey: "github", resolve: async () => { + markStarted(); await held; + return { ok: true, snapshot: { statusKey: "open", statusCategory: "open", statusTone: "info", isTerminal: false } }; + }, + }] }); + const oldRefresh = resolver.refreshObject(externalObjectId, { companyId, force: true }); + await started; + const warning = vi.spyOn(logger, "warn").mockImplementation(() => undefined); await expect(service.pollOnce({ forSetup: activeSetup })).resolves.toMatchObject({ leased: 2, processed: 1, duplicate: 0, failed: 1 }); expect(warning.mock.calls).toEqual([[{ deliveryId: "undecryptable-delivery", reason: "inner_event_authentication_failed", }, "GitHub connector event retained for retry"]]); warning.mockRestore(); + releaseRefresh(); + await expect(oldRefresh).resolves.toMatchObject({ refreshed: false, reason: "refresh_superseded" }); const [snapshot] = await db.select().from(externalObjects).where(eq(externalObjects.id, externalObjectId)); expect(snapshot).toMatchObject({ statusKey: "merged", @@ -184,6 +200,8 @@ describeEmbeddedPostgres("GitHub connection event delivery", () => { statusCategory: "succeeded", statusTone: "success", isTerminal: true, + refreshToken: null, + refreshStartedAt: null, remoteVersion: "2026-09-04T11:59:01.000Z", data: expect.objectContaining({ marker: "preserved", merged: true, headRef: "feature", baseSha: "b".repeat(40) }), }); diff --git a/server/src/index.ts b/server/src/index.ts index c127753889c..f6b59d277cf 100644 --- a/server/src/index.ts +++ b/server/src/index.ts @@ -1323,19 +1323,6 @@ async function startServerWithDatabaseTeardown( pluginWorkerManager, enabled: async () => (await instanceSettingsService(db).getExperimental()).enableExternalObjects === true, }); - const scheduleExternalObjectRefreshSweep = (now = new Date()) => { - if (isIdleTaskDrainActive() || heartbeatSchedulerStopped) return; - trackHeartbeatSchedulerWork(externalObjects - .refreshDueObjectsForActiveCompanies(50, now) - .then((result) => { - if (result.checked > 0 || result.refreshed > 0) { - logger.info({ ...result }, "external-object scheduler tick refreshed due objects"); - } - }) - .catch((err) => { - logger.error({ err }, "external-object scheduler tick failed"); - })); - }; // The retry backstop for orphan sandboxes. An acquire that rejects a // foreign-company insert tears the provisioned sandbox down. If that teardown @@ -1362,6 +1349,8 @@ async function startServerWithDatabaseTeardown( resolveNativeQuestion: (interaction) => deliverNativeQuestionResponse(db as any, interaction), }); const deliveryWork = app.locals.deliveryWork as ReturnType; + externalObjects.registerRefreshWork(deliveryWork, + () => !heartbeatSchedulerStopped && !isIdleTaskDrainActive() && !isWarmStandby()); if (config.nativeHarnessBackupExpirationEnabled) { nativeHarnessBackupExpiration.registerCheckpointExpiration(deliveryWork); } @@ -2007,7 +1996,6 @@ async function startServerWithDatabaseTeardown( } if (isIdleTaskDrainActive() || heartbeatSchedulerStopped) return; - scheduleExternalObjectRefreshSweep(new Date()); if (isIdleTaskDrainActive() || heartbeatSchedulerStopped) return; scheduleMergedPullRequestConfirmationSweep(); @@ -2133,7 +2121,6 @@ async function startServerWithDatabaseTeardown( // as the enabled path. await startEnvironmentCleanup(); startHeartbeatSchedulerInterval(() => { - scheduleExternalObjectRefreshSweep(new Date()); scheduleGitHubConnectionEventPoll(); scheduleGitHubConnectionContinuitySweep(); }); diff --git a/server/src/services/companies.ts b/server/src/services/companies.ts index 0f2ed547a79..f429b08badf 100644 --- a/server/src/services/companies.ts +++ b/server/src/services/companies.ts @@ -1,5 +1,6 @@ import { notifyChatInboundWakeNotices } from "./chat-inbound-wake-notice-notifications.js"; import { notifyCompanyRetentionWork } from "./retention-work-notifications.js"; +import { notifyExternalObjectRefresh } from "./external-object-refresh-notifications.js"; import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js"; import { isRunnerApiResponseAsset } from "./asset-content-access.js"; import { deleteCompany } from "./company-deletion.js"; @@ -334,6 +335,7 @@ export function companyService(db: Db, _budgetHooks: BudgetServiceHooks = {}) { if (companyPatch.status !== undefined && companyPatch.status !== existing.status) { await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers); await notifyDeliveryWork(tx, DELIVERY_QUEUES.skillSourceSync); + await notifyExternalObjectRefresh(tx, id); await notifyDeliveryWork(tx, DELIVERY_QUEUES.routineSchedules); await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors); await notifyCompanyRetentionWork(tx, id); @@ -490,6 +492,7 @@ export function companyService(db: Db, _budgetHooks: BudgetServiceHooks = {}) { if (!wasAlreadyArchived) { await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers); await notifyDeliveryWork(tx, DELIVERY_QUEUES.skillSourceSync); + await notifyExternalObjectRefresh(tx, id); await notifyDeliveryWork(tx, DELIVERY_QUEUES.routineSchedules); await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors); await notifyCompanyRetentionWork(tx, id); diff --git a/server/src/services/company-deletion.ts b/server/src/services/company-deletion.ts index 7ab88aadfa9..645e17f528b 100644 --- a/server/src/services/company-deletion.ts +++ b/server/src/services/company-deletion.ts @@ -1,3 +1,4 @@ +import { notifyExternalObjectRefresh } from "./external-object-refresh-notifications.js"; import { notifyUnsafeWorkspaceExportRecovery } from "./unsafe-workspace-export-notifications.js"; import { notifyCancelledWorkspacePreservation } from "./cancelled-workspace-preservation-notifications.js"; import { notifyInstructionCollection } from "./instruction-collection-notifications.js"; @@ -71,6 +72,7 @@ export async function deleteCompany(db: Db, id: string) { const [existing] = await tx.select({ id: companies.id }).from(companies) .where(eq(companies.id, id)).for("no key update"); if (!existing) return null; + await notifyExternalObjectRefresh(tx, id); await notifyChatCompanyConfiguration(tx, id); await chatCompanyDeletion.deleteCompanyData(tx, id); await notifyDeliveryWork(tx, DELIVERY_QUEUES.skillSourceSync); diff --git a/server/src/services/delivery-work-notifications.ts b/server/src/services/delivery-work-notifications.ts index 9195f1fcb12..e9555d8ad53 100644 --- a/server/src/services/delivery-work-notifications.ts +++ b/server/src/services/delivery-work-notifications.ts @@ -6,6 +6,7 @@ export const DELIVERY_QUEUES = { providerTraceRetention: "provider-trace-retention", nativeRunnerEvidenceRetention: "native-runner-evidence-retention", skillVersionRetention: "skill-version-retention", + externalObjectRefresh: "external-object-refresh", databaseBackups: "database-backups", importSpoolCleanup: "import-spool-cleanup", externalConversationSettlement: "external-conversation-settlement", diff --git a/server/src/services/external-object-refresh-notifications.ts b/server/src/services/external-object-refresh-notifications.ts new file mode 100644 index 00000000000..95833421a8a --- /dev/null +++ b/server/src/services/external-object-refresh-notifications.ts @@ -0,0 +1,7 @@ +import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js"; + +export async function notifyExternalObjectRefresh(tx: object, companyId?: string, objectId?: string) { + await notifyDeliveryWork(tx, DELIVERY_QUEUES.externalObjectRefresh, + companyId ? JSON.stringify({ companyId, ...(objectId ? { objectId } : {}) }) : "*"); +} + diff --git a/server/src/services/external-objects.ts b/server/src/services/external-objects.ts index 56a8fb91834..c49df7ff076 100644 --- a/server/src/services/external-objects.ts +++ b/server/src/services/external-objects.ts @@ -1,7 +1,7 @@ import { randomUUID } from "node:crypto"; -import { and, asc, eq, inArray, isNull, lte, not, or, sql } from "drizzle-orm"; +import { and, asc, eq, exists, gt, inArray, isNotNull, isNull, lte, not, or, sql } from "drizzle-orm"; import type { Db } from "@paperclipai/db"; -import { companies, documents, externalObjectMentions, externalObjects, issueComments, issueDocuments, issues, plugins } from "@paperclipai/db"; +import { companies, documents, externalObjectMentions, externalObjects, issueComments, issueDocuments, issues, plugins, pluginCompanySettings } from "@paperclipai/db"; import { formatExternalObjectMentionSourceLabel, type ExternalObjectCanonicalUrl, @@ -18,6 +18,10 @@ import { notFound } from "../errors.js"; import { logger } from "../middleware/logger.js"; import { logActivity, type LogActivityInput } from "./activity-log.js"; import { createGitHubExternalObjectProvider, type GitHubExternalObjectProviderOptions } from "./github-external-object-provider.js"; +import { notifyExternalObjectRefresh } from "./external-object-refresh-notifications.js"; +import { DELIVERY_QUEUES } from "./delivery-work-notifications.js"; +import type { createDeliveryWorkCoordinator } from "./delivery-work-coordinator.js"; +import { registerScopedDeadlineWork } from "./scoped-deadline-work.js"; import { publishLiveEvent } from "./live-events.js"; import type { PluginWorkerManager } from "./plugin-worker-manager.js"; @@ -95,6 +99,18 @@ const DEFAULT_RETRY_AFTER_SECONDS = 300; const DEFAULT_REFRESH_LEASE_SECONDS = 300; const REFRESH_LEASE_RENEW_INTERVAL_MS = 60_000; +type RefreshScope = { companyId: string; objectId?: string }; +type RefreshDeadline = { id: string; companyId: string; dueAt: number }; + + +function refreshableObjectWhere() { + return or(eq(externalObjects.isTerminal, false), and( + eq(externalObjects.providerKey, "github"), + eq(externalObjects.statusKey, "not_found"), + not(sql`coalesce(${externalObjects.data}->>'probeVerified', 'false') = 'true'`), + ))!; +} + function sourceWhere(input: ExternalObjectSourceContext) { const conditions = [ eq(externalObjectMentions.companyId, input.companyId), @@ -245,7 +261,7 @@ function objectSnapshot(object: ExternalObjectRecord): PluginExternalObjectRecor }; } -async function readyObjectReferencePlugins(db: Db) { +async function readyObjectReferencePlugins(db: Db, companyId?: string) { return db .select({ id: plugins.id, @@ -253,7 +269,12 @@ async function readyObjectReferencePlugins(db: Db) { manifestJson: plugins.manifestJson, }) .from(plugins) - .where(eq(plugins.status, "ready")) + .where(and(eq(plugins.status, "ready"), companyId ? not(exists(db.select({ id: pluginCompanySettings.id }) + .from(pluginCompanySettings).where(and( + eq(pluginCompanySettings.pluginId, plugins.id), + eq(pluginCompanySettings.companyId, companyId), + eq(pluginCompanySettings.enabled, false), + )))) : undefined)) .orderBy(asc(plugins.installOrder)) .then((rows) => rows.filter((row) => (row.manifestJson.objectReferences?.length ?? 0) > 0), @@ -267,7 +288,7 @@ function createPluginProviderDetector( return { key: "plugin-object-reference-providers", async detect(input) { - const providers = await readyObjectReferencePlugins(db); + const providers = await readyObjectReferencePlugins(db, input.companyId); const detections: ExternalObjectDetection[] = []; for (const provider of providers) { @@ -326,7 +347,7 @@ async function resolveViaPluginProvider( object: ExternalObjectRecord, ): Promise { if (!pluginWorkerManager) return null; - const providers = await readyObjectReferencePlugins(db); + const providers = await readyObjectReferencePlugins(db, object.companyId); for (const provider of providers) { const manifest = provider.manifestJson; if (!manifest.capabilities.includes("external.objects.read")) continue; @@ -374,10 +395,38 @@ export function externalObjectService( ...(opts.detectors ?? []), ...(githubProvider ? [githubProvider.detector] : []), ]); - const resolverRegistry = createExternalObjectResolverRegistry([ + const resolvers = [ ...(opts.resolvers ?? []), ...(githubProvider?.resolvers ?? []), - ]); + ]; + const resolverRegistry = createExternalObjectResolverRegistry(resolvers); + + function automaticRefreshWhere(database: Db = db) { + const supported = resolvers.map(resolver => and( + eq(externalObjects.providerKey, resolver.providerKey), + resolver.objectType ? eq(externalObjects.objectType, resolver.objectType) : undefined, + )!); + if (opts.pluginWorkerManager) supported.push(exists(database.select({ id: plugins.id }).from(plugins).where(and( + eq(plugins.status, "ready"), + sql`${plugins.manifestJson}->'capabilities' ? 'external.objects.read'`, + sql`exists (select 1 from jsonb_array_elements(coalesce(${plugins.manifestJson}->'objectReferences', '[]'::jsonb)) as reference + where reference->>'providerKey' = ${externalObjects.providerKey} + and reference->'objectTypes' ? ${externalObjects.objectType})`, + not(exists(database.select({ id: pluginCompanySettings.id }).from(pluginCompanySettings).where(and( + eq(pluginCompanySettings.pluginId, plugins.id), + eq(pluginCompanySettings.companyId, externalObjects.companyId), + eq(pluginCompanySettings.enabled, false), + )))), + )))); + return and( + refreshableObjectWhere(), isNotNull(externalObjects.nextRefreshAt), + supported.length ? or(...supported) : sql`false`, + exists(database.select({ id: externalObjectMentions.id }).from(externalObjectMentions).where(and( + eq(externalObjectMentions.companyId, externalObjects.companyId), + eq(externalObjectMentions.objectId, externalObjects.id), + ))), + )!; + } async function isEnabled() { if (typeof opts.enabled === "function") return await opts.enabled(); @@ -422,6 +471,7 @@ export function externalObjectService( nextRefreshAt: now, updatedAt: now, }; + await notifyExternalObjectRefresh(dbOrTx, companyId); const inserted = await dbOrTx .insert(externalObjects) .values(values) @@ -451,6 +501,7 @@ export function externalObjectService( dbOrTx: any = db, ) { const urls = extractExternalObjectCanonicalUrls(input.text ?? ""); + await notifyExternalObjectRefresh(dbOrTx, input.companyId); await dbOrTx.delete(externalObjectMentions).where(sourceWhere(input)); if (urls.length === 0) return; @@ -534,6 +585,7 @@ export function externalObjectService( .where(eq(issueComments.id, commentId)) .then((rows: Array<{ id: string; companyId: string; issueId: string; body: string }>) => rows[0] ?? null); if (!comment) { + await notifyExternalObjectRefresh(tx); await tx .delete(externalObjectMentions) .where(and(eq(externalObjectMentions.sourceKind, "comment"), eq(externalObjectMentions.sourceRecordId, commentId))); @@ -568,6 +620,7 @@ export function externalObjectService( .where(eq(documents.id, documentId)) .then((rows: Array<{ documentId: string; companyId: string; issueId: string; key: string; body: string }>) => rows[0] ?? null); if (!document) { + await notifyExternalObjectRefresh(tx); await tx .delete(externalObjectMentions) .where(and(eq(externalObjectMentions.sourceKind, "document"), eq(externalObjectMentions.sourceRecordId, documentId))); @@ -780,6 +833,7 @@ export function externalObjectService( companyId: string; actor?: Pick; force?: boolean; + scheduled?: boolean; now?: Date; }; @@ -801,6 +855,18 @@ export function externalObjectService( return { object: toObjectPayload(latest, now), refreshed: false, reason: "refresh_superseded" as const }; } + async function persistRefresh(object: ExternalObjectRecord, token: string, + patch: Partial, scheduled = false) { + return db.transaction(async tx => { + await notifyExternalObjectRefresh(tx, object.companyId, object.id); + const rows = await tx.update(externalObjects).set(patch) + .where(and(refreshOwnerWhere(object, token), scheduled ? automaticRefreshWhere(tx as unknown as Db) : undefined)).returning(); + if (scheduled && rows.length === 0) await tx.update(externalObjects) + .set({ refreshStartedAt: null, refreshToken: null }).where(refreshOwnerWhere(object, token)); + return rows; + }); + } + async function resolveObjectRefresh( object: ExternalObjectRecord, input: RefreshObjectInput, @@ -810,17 +876,13 @@ export function externalObjectService( const pluginResult = await resolveViaPluginProvider(db, opts.pluginWorkerManager, object); const resolver = pluginResult ? null : resolverRegistry.find(object); if (!pluginResult && !resolver) { - const [updated] = await db - .update(externalObjects) - .set({ + const [updated] = await persistRefresh(object, refreshToken, { liveness: object.isTerminal ? object.liveness : visibleLiveness(object, now) === "fresh" ? "stale" : object.liveness, nextRefreshAt: addSeconds(now, DEFAULT_RETRY_AFTER_SECONDS), refreshStartedAt: null, refreshToken: null, updatedAt: now, - }) - .where(refreshOwnerWhere(object, refreshToken)) - .returning(); + }, input.scheduled); if (!updated) return refreshSupersededResult(object, now); return { object: toObjectPayload(updated ?? object, now), refreshed: false, reason: "no_resolver" as const }; } @@ -831,9 +893,7 @@ export function externalObjectService( // terminal. Clear a previous terminal verdict so the object keeps being // scheduled for automatic refresh and self-heals (e.g. a GitHub "not // found" that later resolves once repo access improves). - const [updated] = await db - .update(externalObjects) - .set({ + const [updated] = await persistRefresh(object, refreshToken, { liveness: result.liveness, isTerminal: false, lastErrorAt: now, @@ -843,9 +903,7 @@ export function externalObjectService( refreshStartedAt: null, refreshToken: null, updatedAt: now, - }) - .where(refreshOwnerWhere(object, refreshToken)) - .returning(); + }, input.scheduled); if (!updated) return refreshSupersededResult(object, now); publishLiveEvent({ companyId: object.companyId, @@ -879,14 +937,10 @@ export function externalObjectService( refreshToken: null, updatedAt: now, }; - const [updated] = await db - .update(externalObjects) - .set({ + const [updated] = await persistRefresh(object, refreshToken, { ...patch, lastChangedAt: objectChanged(object, { ...object, ...patch }) ? now : object.lastChangedAt, - }) - .where(refreshOwnerWhere(object, refreshToken)) - .returning(); + }, input.scheduled); if (!updated) return refreshSupersededResult(object, now); const next = updated ?? object; if (objectChanged(object, next) && input.actor) { @@ -947,6 +1001,8 @@ export function externalObjectService( eq(externalObjects.id, object.id), eq(externalObjects.companyId, object.companyId), dueOrForced, + input.scheduled ? automaticRefreshWhere() : undefined, + input.scheduled ? sql`exists (select 1 from ${companies} where ${companies.id} = ${externalObjects.companyId} and ${companies.status} = 'active')` : undefined, )) .returning(); return claimed ?? null; @@ -1032,26 +1088,7 @@ export function externalObjectService( async function refreshDueObjectsUnchecked(companyId: string, limit = 50, now = new Date()) { const staleRefreshStartedBefore = new Date(now.getTime() - DEFAULT_REFRESH_LEASE_SECONDS * 1000); - // Terminal objects are usually skipped forever, but a GitHub "not found" - // verdict is only trustworthy while the repository is actually visible to - // the configured credentials. GitHub hides inaccessible (often private) - // repos behind a 404, so a not-found object can become resolvable later - // when credentials or repo visibility improve. Re-check objects that were - // frozen as "not found" before the repository probe existed so they - // self-heal instead of freezing. Once the provider has probed the repo and - // confirmed it is visible (data.probeVerified = "true"), the not-found - // verdict is definitive, so stop re-polling that object to avoid burning - // the company's GitHub API budget on deleted references forever. - const probeVerifiedNotFound = sql`coalesce(${externalObjects.data}->>'probeVerified', 'false') = 'true'`; - const refreshable = or( - eq(externalObjects.isTerminal, false), - and( - eq(externalObjects.isTerminal, true), - eq(externalObjects.providerKey, "github"), - eq(externalObjects.statusKey, "not_found"), - not(probeVerifiedNotFound), - ), - )!; + const refreshable = refreshableObjectWhere(); const due = await db .select({ id: externalObjects.id }) .from(externalObjects) @@ -1099,6 +1136,59 @@ export function externalObjectService( return { companies: activeCompanies.length, checked, refreshed }; } + function registerRefreshWork( + coordinator: ReturnType, + canRun: () => boolean, + ) { + const cursors = new Map(); + const scopeKey = (scope?: RefreshScope) => JSON.stringify(scope ?? null); + async function load(scope?: RefreshScope) { + if (!(await isEnabled())) { cursors.clear(); return []; } + const key = scopeKey(scope); + const cursor = cursors.get(key); + const rows = await db.select({ id: externalObjects.id, companyId: externalObjects.companyId, + nextRefreshAt: externalObjects.nextRefreshAt, refreshStartedAt: externalObjects.refreshStartedAt }).from(externalObjects) + .innerJoin(companies, eq(companies.id, externalObjects.companyId)) + .where(and( + eq(companies.status, "active"), automaticRefreshWhere(), + scope ? eq(externalObjects.companyId, scope.companyId) : undefined, + scope?.objectId ? eq(externalObjects.id, scope.objectId) : undefined, + cursor ? gt(externalObjects.id, cursor) : undefined, + )).orderBy(asc(externalObjects.id)).limit(250); + const entries = rows.map(deadline); + if (rows.length === 250) cursors.set(key, rows[rows.length - 1]!.id); + else cursors.delete(key); + return { rows: entries, complete: rows.length < 250 }; + } + function deadline(object: Pick): RefreshDeadline { + return { id: object.id, companyId: object.companyId, + dueAt: Math.max(object.nextRefreshAt!.getTime(), object.refreshStartedAt + ? object.refreshStartedAt.getTime() + DEFAULT_REFRESH_LEASE_SECONDS * 1000 : 0) }; + } + return registerScopedDeadlineWork(coordinator, { + queue: DELIVERY_QUEUES.externalObjectRefresh, + batchSize: 50, + minimumPassIntervalMs: 0, + decodeScope: hint => hint === "*" ? undefined : JSON.parse(hint) as RefreshScope, + matches: (entry, scope) => !scope || (entry.companyId === scope.companyId + && (!scope.objectId || entry.id === scope.objectId)), + load, + check: async (entry, signal) => { + if (signal?.aborted || !canRun()) return entry; + const current = await load({ companyId: entry.companyId, objectId: entry.id }); + const scheduled = Array.isArray(current) ? current[0] : current.rows[0]; + if (!scheduled || scheduled.dueAt > Date.now()) return scheduled ?? null; + await refreshObject(entry.id, { + companyId: entry.companyId, + scheduled: true, + actor: { actorType: "system", actorId: "external-object-resolver", agentId: null, runId: null }, + }); + const updated = await load({ companyId: entry.companyId, objectId: entry.id }); + return (Array.isArray(updated) ? updated[0] : updated.rows[0]) ?? null; + }, + }, canRun); + } + return { syncIssue, syncComment, @@ -1114,5 +1204,6 @@ export function externalObjectService( refreshIssueObjects, refreshDueObjects, refreshDueObjectsForActiveCompanies, + registerRefreshWork, }; } diff --git a/server/src/services/github-connection-events.ts b/server/src/services/github-connection-events.ts index 47bdac122a2..76755ce30c3 100644 --- a/server/src/services/github-connection-events.ts +++ b/server/src/services/github-connection-events.ts @@ -1,3 +1,4 @@ +import { notifyExternalObjectRefresh } from "./external-object-refresh-notifications.js"; import { notifyToolConnectionHealth } from "./tool-connection-health-notifications.js"; import { dispatchGitHubBotCloudEvent } from "./chat-github-cloud-ingress.js"; import { chatEndpoints } from "@paperclipai/db"; @@ -244,25 +245,30 @@ export function githubConnectionEventService( const snapshot = githubSnapshotUpdate(event.payload); if (!snapshot) return; const appliedAt = now(); - await db.update(externalObjects).set({ - statusKey: snapshot.statusKey, - statusLabel: snapshot.statusLabel, - statusCategory: snapshot.statusCategory, - statusTone: snapshot.statusTone, - statusIconKey: snapshot.statusIconKey, - isTerminal: snapshot.merged || snapshot.state === "closed", - data: sql`${externalObjects.data} || ${JSON.stringify(snapshot.data)}::jsonb`, - remoteVersion: snapshot.remoteVersion, - lastResolvedAt: appliedAt, - lastChangedAt: appliedAt, - nextRefreshAt: appliedAt, - updatedAt: appliedAt, - }).where(and( - eq(externalObjects.companyId, companyId), - eq(externalObjects.providerKey, "github"), - eq(externalObjects.objectType, "pull_request"), - sql`lower(${externalObjects.externalId}) = lower(${snapshot.externalId})`, - )); + await db.transaction(async tx => { + await notifyExternalObjectRefresh(tx, companyId); + await tx.update(externalObjects).set({ + statusKey: snapshot.statusKey, + statusLabel: snapshot.statusLabel, + statusCategory: snapshot.statusCategory, + statusTone: snapshot.statusTone, + statusIconKey: snapshot.statusIconKey, + isTerminal: snapshot.merged || snapshot.state === "closed", + data: sql`${externalObjects.data} || ${JSON.stringify(snapshot.data)}::jsonb`, + remoteVersion: snapshot.remoteVersion, + lastResolvedAt: appliedAt, + lastChangedAt: appliedAt, + nextRefreshAt: appliedAt, + refreshStartedAt: null, + refreshToken: null, + updatedAt: appliedAt, + }).where(and( + eq(externalObjects.companyId, companyId), + eq(externalObjects.providerKey, "github"), + eq(externalObjects.objectType, "pull_request"), + sql`lower(${externalObjects.externalId}) = lower(${snapshot.externalId})`, + )); + }); if (snapshot.merged && event.action === "closed") { await issueThreadInteractionService(db, { wakeup: options.wakeup }) .sweepMergedPullRequestConfirmations([{ diff --git a/server/src/services/instance-settings.ts b/server/src/services/instance-settings.ts index 0208653a86f..a2f5eb5015c 100644 --- a/server/src/services/instance-settings.ts +++ b/server/src/services/instance-settings.ts @@ -13,6 +13,7 @@ import { notifyInstructionCommit } from "./instruction-commit-notifications.js"; import { notifyScheduledRetryCancellation } from "./scheduled-retry-cancellation-notifications.js"; import { notifyVoiceChange } from "./voice/voice-recovery-notifications.js"; import { notifyToolConnectionHealth } from "./tool-connection-health-notifications.js"; +import { notifyExternalObjectRefresh } from "./external-object-refresh-notifications.js"; import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js"; import type { Db } from "@paperclipai/db"; import { companies, instanceSettings } from "@paperclipai/db"; @@ -628,6 +629,7 @@ export function instanceSettingsService(db: Db, options: InstanceSettingsService await notifyDeliveryWork(tx, DELIVERY_QUEUES.mcpEvents, "*"); await notifyDeliveryWork(tx, DELIVERY_QUEUES.dotEvents, "*"); await notifyDeliveryWork(tx, DELIVERY_QUEUES.grokDispatch); + if (patch.enableExternalObjects !== undefined) await notifyExternalObjectRefresh(tx); if (patch.enableWorktreeRunExecution !== undefined) await notifyScheduledRetryCancellation(tx); if (patch.enableWorktreeRunExecution !== undefined) { await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers); diff --git a/server/src/services/issues.ts b/server/src/services/issues.ts index 42c65bb5d02..53ac68718a6 100644 --- a/server/src/services/issues.ts +++ b/server/src/services/issues.ts @@ -1,3 +1,4 @@ +import { notifyExternalObjectRefresh } from "./external-object-refresh-notifications.js"; import { notifyExternalConversationSettlement } from "./external-conversation-settlement-notifications.js"; import { notifyNativeRecoveryNotices } from "./native-recovery-notice-notifications.js"; import { notifyNativeRunProjection } from "./native-run-projection-notifications.js"; @@ -11953,6 +11954,7 @@ export function issueService(db: Db) { const [owner] = await tx.select({ companyId: issues.companyId }).from(issues).where(eq(issues.id, id)); if (owner) { await notifyExternalConversationSettlement(tx, { companyId: owner.companyId }); + await notifyExternalObjectRefresh(tx, owner.companyId); await notifyApprovalWork(tx, owner.companyId); } await notifyDecisionTargetIssueRemoval(tx, id); diff --git a/server/src/services/plugin-registry.ts b/server/src/services/plugin-registry.ts index 239c3c50490..62d9b7cde70 100644 --- a/server/src/services/plugin-registry.ts +++ b/server/src/services/plugin-registry.ts @@ -1,3 +1,4 @@ +import { notifyExternalObjectRefresh } from "./external-object-refresh-notifications.js"; import { isDeepStrictEqual } from "node:util"; import { writePluginJobChange } from "./plugin-job-work-notifications.js"; import { asc, eq, isNull, ne, sql, and } from "drizzle-orm"; @@ -239,12 +240,15 @@ export function pluginRegistryService(db: Db) { setClause.categories = data.manifest.categories; } - return db - .update(plugins) - .set(setClause) - .where(eq(plugins.id, id)) - .returning() - .then((rows) => rows[0] ?? null); + return db.transaction(async tx => { + await notifyExternalObjectRefresh(tx); + return tx + .update(plugins) + .set(setClause) + .where(eq(plugins.id, id)) + .returning() + .then((rows) => rows[0] ?? null); + }); }, // ----- Status --------------------------------------------------------- @@ -254,16 +258,19 @@ export function pluginRegistryService(db: Db) { const plugin = await getById(id); if (!plugin) throw notFound("Plugin not found"); - return db - .update(plugins) - .set({ - status: input.status, - lastError: input.lastError ?? null, - updatedAt: new Date(), - }) - .where(eq(plugins.id, id)) - .returning() - .then((rows) => rows[0] ?? null); + return db.transaction(async tx => { + await notifyExternalObjectRefresh(tx); + return tx + .update(plugins) + .set({ + status: input.status, + lastError: input.lastError ?? null, + updatedAt: new Date(), + }) + .where(eq(plugins.id, id)) + .returning() + .then((rows) => rows[0] ?? null); + }); }, // ----- Uninstall / Remove -------------------------------------------- @@ -280,6 +287,7 @@ export function pluginRegistryService(db: Db) { if (!plugin) throw notFound("Plugin not found"); return writePluginJobChange(db, async db => { + await notifyExternalObjectRefresh(db); if (removeData) { // Hard delete – plugin_config cascades via FK onDelete return db @@ -343,6 +351,7 @@ export function pluginRegistryService(db: Db) { return db.transaction(async tx => { const db = tx as unknown as Db; await notifyDeliveryWork(db, DELIVERY_QUEUES.pluginLifecycle); + await notifyExternalObjectRefresh(db, companyId); const plugin = await getById(pluginId, db); if (!plugin) throw notFound("Plugin not found"); @@ -385,6 +394,7 @@ export function pluginRegistryService(db: Db) { return db.transaction(async tx => { const db = tx as unknown as Db; await notifyDeliveryWork(db, DELIVERY_QUEUES.pluginLifecycle); + await notifyExternalObjectRefresh(db, companyId); const plugin = await getById(pluginId, db); if (!plugin) throw notFound("Plugin not found"); @@ -467,6 +477,7 @@ export function pluginRegistryService(db: Db) { return db.transaction(async tx => { const db = tx as unknown as Db; await notifyDeliveryWork(db, DELIVERY_QUEUES.pluginLifecycle); + await notifyExternalObjectRefresh(db, companyId); const plugin = await getById(pluginId, db); if (!plugin) throw notFound("Plugin not found"); From 25a8a4392c807d4b5eb19cbf713b5eda99c017d3 Mon Sep 17 00:00:00 2001 From: Dotta Date: Fri, 30 Oct 2026 16:35:43 -0500 Subject: [PATCH 06/70] fix(external-objects): cover budget resume and scoped deletions Co-Authored-By: Paperclip --- .../2026-10-30-external-objects-sleep.md | 4 +- .../external-object-refresh-work.test.ts | 63 ++++++++++++++++++- .../external-objects-service.test.ts | 4 +- server/src/services/budgets.ts | 3 + server/src/services/external-objects.ts | 18 +++--- 5 files changed, 79 insertions(+), 13 deletions(-) diff --git a/doc/plans/2026-10-30-external-objects-sleep.md b/doc/plans/2026-10-30-external-objects-sleep.md index 639bbaea343..ab566999917 100644 --- a/doc/plans/2026-10-30-external-objects-sleep.md +++ b/doc/plans/2026-10-30-external-objects-sleep.md @@ -47,10 +47,10 @@ and fences automatic result persistence. It does not modify that draft PR. | Mutation | Owner and activation | | --- | --- | | Issue title/description, comment and document references | Existing sync methods register company intent before mention replacement and object upsert. Routes, runtime tools, adapters and plugin paths reuse these methods. | -| Comment/document disappearance | Sync registers reconstruction before deleting obsolete mentions. | +| Comment/document disappearance | Sync derives the retained mention owners and registers only those companies before deleting obsolete mentions. | | Last mention removal and reattachment | Mention replacement invalidates company deadlines. Retained objects stay available for reattachment. | | Issue deletion | Issue service registers its company before cascading mention deletion. | -| Company status/archive/deletion | Company services register company reconstruction before their status or deletion writes. | +| Company status/archive/deletion, including budget pause/resume | Company and budget services register company reconstruction before status or deletion writes. | | Feature enable/disable | Instance settings registers global reconstruction before the experimental-settings write. | | Plugin manifest/status/uninstall | Plugin registry registers reconstruction before lifecycle writes. Installed plugins acquire support through the ordinary ready transition. | | Plugin company configuration and enabled setting | Registry writes register only the affected company. There is no plugin-readiness poll. | diff --git a/server/src/__tests__/external-object-refresh-work.test.ts b/server/src/__tests__/external-object-refresh-work.test.ts index 8d1f5cdc00c..f429c31396b 100644 --- a/server/src/__tests__/external-object-refresh-work.test.ts +++ b/server/src/__tests__/external-object-refresh-work.test.ts @@ -2,10 +2,12 @@ import { randomUUID } from "node:crypto"; import { setTimeout as delay } from "node:timers/promises"; import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest"; import { eq, sql } from "drizzle-orm"; -import { activityLog, companies, createDb, externalObjects, issues, plugins } from "@paperclipai/db"; +import { activityLog, approvals, applyPendingMigrations, budgetIncidents, budgetPolicies, companies, costEvents, createDb, externalObjects, issues, issueComments, plugins } from "@paperclipai/db"; import { externalObjectService, type ExternalObjectResolveResult } from "../services/external-objects.js"; import type { PaperclipPluginManifestV1 } from "@paperclipai/shared"; import type { PluginWorkerManager } from "../services/plugin-worker-manager.js"; +import { documentService } from "../services/documents.js"; +import { budgetService } from "../services/budgets.js"; import { pluginRegistryService } from "../services/plugin-registry.js"; import { companyService } from "../services/companies.js"; import { instanceSettingsService } from "../services/instance-settings.js"; @@ -34,14 +36,19 @@ const active: ExternalObjectResolveResult = { ok: true, snapshot: { const errors = vi.fn(); const companyIds: string[] = []; beforeAll(async () => { + if (url) await applyPendingMigrations(url); if (!url) temp = await startEmbeddedPostgresTestDatabase("external-refresh-work-"); db = createDb(url ?? temp!.connectionString, { maxConnections: 6 }); await db.execute(sql`truncate table ${companies} cascade`); - }); + }, 90_000); afterEach(async () => { await coordinator?.stop(); coordinator = undefined; vi.useRealTimers(); vi.restoreAllMocks(); for (const id of companyIds.splice(0)) { + await db.delete(budgetIncidents).where(eq(budgetIncidents.companyId, id)); + await db.delete(budgetPolicies).where(eq(budgetPolicies.companyId, id)); + await db.delete(approvals).where(eq(approvals.companyId, id)); + await db.delete(costEvents).where(eq(costEvents.companyId, id)); await db.delete(activityLog).where(eq(activityLog.companyId, id)); await db.delete(issues).where(eq(issues.companyId, id)); await db.delete(externalObjects).where(eq(externalObjects.companyId, id)); @@ -256,6 +263,58 @@ const active: ExternalObjectResolveResult = { ok: true, snapshot: { expect(calls).toHaveBeenCalledOnce(); }); + it("reactivates refresh after ordinary budget pause and budget raise", async () => { + clock(); const svc = service(); provider.mockResolvedValue(active); await start(svc); + const f = await fixture(); await svc.syncIssue(f.issueId); + await settle(async () => (await row(f.companyId))?.lastResolvedAt !== null); + await db.insert(costEvents).values({ companyId: f.companyId, usageKind: "decision", provider: "fixture", + biller: "fixture", billingType: "metered_api", model: "simulated", costCents: 200, occurredAt: new Date() }); + const budgets = budgetService(db); + await budgets.upsertPolicy(f.companyId, { scopeType: "company", scopeId: f.companyId, + amount: 100, hardStopEnabled: true }, null); + expect((await db.select().from(companies).where(eq(companies.id, f.companyId)))[0]!.status).toBe("paused"); + await quiet(); + const reads = vi.spyOn(db, "select"); provider.mockClear(); + await vi.advanceTimersByTimeAsync(900_000); expect(reads).not.toHaveBeenCalled(); expect(provider).not.toHaveBeenCalled(); + provider.mockResolvedValue(terminal); + await budgets.upsertPolicy(f.companyId, { scopeType: "company", scopeId: f.companyId, amount: 300 }, null); + await settle(async () => (await row(f.companyId)).isTerminal); await quiet(); + expect(provider).toHaveBeenCalledOnce(); + }); + + it.each(["comment", "document"] as const)("keeps hard-deleted %s activation scoped to its retained mention owner", async kind => { + clock(); const svc = service(); provider.mockResolvedValue(active); await start(svc); + const a = await fixture(); const b = await fixture(); await svc.syncIssue(b.issueId); + let id: string; + if (kind === "comment") { + const [comment] = await db.insert(issueComments).values({ companyId: a.companyId, issueId: a.issueId, + body: `https://example.com/${a.companyId}` }).returning(); + id = comment!.id; await svc.syncComment(id); + } else { + const result = await documentService(db).upsertIssueDocument({ issueId: a.issueId, + key: "notes", format: "markdown", body: `https://example.com/${a.companyId}` }); + id = result.document.id; await svc.syncDocument(id); + } + await settle(async () => (await row(a.companyId))?.lastResolvedAt !== null && (await row(b.companyId))?.lastResolvedAt !== null); + await settle(() => coordinator!.nextWakeAt() !== null); + const reads: Array<{ query: string; params: unknown[] }> = []; + const debug = db.$client.options.debug; + db.$client.options.debug = (_connection, query, params) => { + if (/^select/i.test(query) && query.includes('from "external_objects"') && query.includes('inner join "companies"')) + reads.push({ query, params: [...params] }); + }; + try { + if (kind === "comment") { await db.delete(issueComments).where(eq(issueComments.id, id)); await svc.syncComment(id); } + else { await documentService(db).deleteIssueDocument(a.issueId, "notes"); await svc.syncDocument(id); } + await settle(() => reads.length > 0); + for (const read of reads) { + expect(read.query).toContain('"external_objects"."company_id" ='); + expect(read.params).toContain(a.companyId); expect(read.params).not.toContain(b.companyId); + } + } finally { db.$client.options.debug = debug; } + expect(provider).toHaveBeenCalledTimes(2); + }); + it("keeps warm standby startup SQL-free and resumes a first committed producer", async () => { clock(); admitted = false; reconcile = false; const svc = service(); const selects = vi.spyOn(db, "select"); diff --git a/server/src/__tests__/external-objects-service.test.ts b/server/src/__tests__/external-objects-service.test.ts index e50e298b466..bc268d0ba6a 100644 --- a/server/src/__tests__/external-objects-service.test.ts +++ b/server/src/__tests__/external-objects-service.test.ts @@ -3,6 +3,7 @@ import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest import { eq } from "drizzle-orm"; import { companies, + applyPendingMigrations, createDb, activityLog, externalObjectMentions, @@ -711,9 +712,10 @@ describeEmbeddedPostgres("externalObjectService", () => { let tempDb: Awaited> | null = null; beforeAll(async () => { + if (externalDatabaseUrl) await applyPendingMigrations(externalDatabaseUrl); if (!externalDatabaseUrl) tempDb = await startEmbeddedPostgresTestDatabase("paperclip-external-objects-"); db = createDb(externalDatabaseUrl ?? tempDb!.connectionString, { maxConnections: 6 }); - }, 20_000); + }, 90_000); afterEach(async () => { await db.delete(activityLog); diff --git a/server/src/services/budgets.ts b/server/src/services/budgets.ts index 6777124ee90..88c3255da4e 100644 --- a/server/src/services/budgets.ts +++ b/server/src/services/budgets.ts @@ -1,3 +1,4 @@ +import { notifyExternalObjectRefresh } from "./external-object-refresh-notifications.js"; import { notifyApprovalWork } from "./approval-work-notifications.js"; import { notifyCompanyRetentionWork } from "./retention-work-notifications.js"; import { addCents, compareCents, normalizeCents } from "@paperclipai/shared"; @@ -301,6 +302,7 @@ export function budgetServiceInTransaction(db: Db, publications: ActivityPublica const scope = await resolveScopeRecord(db, "company", policy.scopeId); if (scope.paused) return; await notifyCompanyRetentionWork(db, policy.companyId); + await notifyExternalObjectRefresh(db, policy.companyId); await notifyDeliveryWork(db, DELIVERY_QUEUES.agentTimers); await notifyDeliveryWork(db, DELIVERY_QUEUES.routineSchedules); await notifyDeliveryWork(db, DELIVERY_QUEUES.issueMonitors); @@ -361,6 +363,7 @@ export function budgetServiceInTransaction(db: Db, publications: ActivityPublica const scope = await resolveScopeRecord(db, "company", policy.scopeId); if (scope.pauseReason !== "budget") return; await notifyCompanyRetentionWork(db, policy.companyId); + await notifyExternalObjectRefresh(db, policy.companyId); await notifyDeliveryWork(db, DELIVERY_QUEUES.agentTimers); await notifyDeliveryWork(db, DELIVERY_QUEUES.routineSchedules); await notifyDeliveryWork(db, DELIVERY_QUEUES.issueMonitors); diff --git a/server/src/services/external-objects.ts b/server/src/services/external-objects.ts index c49df7ff076..d524b473e1a 100644 --- a/server/src/services/external-objects.ts +++ b/server/src/services/external-objects.ts @@ -571,6 +571,14 @@ export function externalObjectService( return dbOrTx === db ? db.transaction(runSync) : runSync(dbOrTx); } + async function deleteMissingSourceMentions(kind: "comment" | "document", recordId: string, tx: any) { + const source = and(eq(externalObjectMentions.sourceKind, kind), eq(externalObjectMentions.sourceRecordId, recordId)); + const owners: Array<{ companyId: string }> = await tx.selectDistinct({ companyId: externalObjectMentions.companyId }) + .from(externalObjectMentions).where(source); + for (const owner of owners) await notifyExternalObjectRefresh(tx, owner.companyId); + await tx.delete(externalObjectMentions).where(source); + } + async function syncComment(commentId: string, dbOrTx: any = db) { const runSync = async (tx: any) => { if (!(await isEnabled())) return; @@ -585,10 +593,7 @@ export function externalObjectService( .where(eq(issueComments.id, commentId)) .then((rows: Array<{ id: string; companyId: string; issueId: string; body: string }>) => rows[0] ?? null); if (!comment) { - await notifyExternalObjectRefresh(tx); - await tx - .delete(externalObjectMentions) - .where(and(eq(externalObjectMentions.sourceKind, "comment"), eq(externalObjectMentions.sourceRecordId, commentId))); + await deleteMissingSourceMentions("comment", commentId, tx); return; } await replaceSourceMentions({ @@ -620,10 +625,7 @@ export function externalObjectService( .where(eq(documents.id, documentId)) .then((rows: Array<{ documentId: string; companyId: string; issueId: string; key: string; body: string }>) => rows[0] ?? null); if (!document) { - await notifyExternalObjectRefresh(tx); - await tx - .delete(externalObjectMentions) - .where(and(eq(externalObjectMentions.sourceKind, "document"), eq(externalObjectMentions.sourceRecordId, documentId))); + await deleteMissingSourceMentions("document", documentId, tx); return; } await replaceSourceMentions({ From 922ae50f3fda4c911f21949408a21b54655d949f Mon Sep 17 00:00:00 2001 From: Dotta Date: Fri, 30 Oct 2026 16:36:26 -0500 Subject: [PATCH 07/70] test(heartbeat): verify dispatch against real PostgreSQL clocks Co-Authored-By: Paperclip --- .../heartbeat-run-dispatch-work.test.ts | 87 +++++++++++++++---- .../server-startup-feedback-export.test.ts | 14 +++ server/src/services/budgets.ts | 5 ++ .../services/heartbeat/dispatch-work.test.ts | 4 +- server/src/services/heartbeat/retries.test.ts | 10 ++- 5 files changed, 98 insertions(+), 22 deletions(-) diff --git a/server/src/__tests__/heartbeat-run-dispatch-work.test.ts b/server/src/__tests__/heartbeat-run-dispatch-work.test.ts index 527309051fa..3d6f90aa71f 100644 --- a/server/src/__tests__/heartbeat-run-dispatch-work.test.ts +++ b/server/src/__tests__/heartbeat-run-dispatch-work.test.ts @@ -1,11 +1,14 @@ import { randomUUID } from "node:crypto"; import { and, eq, sql } from "drizzle-orm"; import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; -import { agents, companies, createDb, heartbeatRuns } from "@paperclipai/db"; +import { agents, companies, costEvents, createDb, databaseWorkPending, heartbeatRuns } from "@paperclipai/db"; import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js"; +import { agentService } from "../services/agents.js"; +import { budgetService } from "../services/budgets.js"; import { heartbeatService } from "../services/heartbeat.js"; import { createDeliveryWorkCoordinator } from "../services/delivery-work-coordinator.js"; +import { DELIVERY_QUEUES } from "../services/delivery-work-notifications.js"; import { notifyRunDispatch } from "../services/run-dispatch-notifications.js"; const external = resolveExternalTestDatabaseUrl("PAPERCLIP_RUN_DISPATCH_TEST_DATABASE_URL"); @@ -23,12 +26,12 @@ describe.skipIf(!support.supported)("run dispatch production wiring (controlled beforeAll(async () => { if (!external) database = await startEmbeddedPostgresTestDatabase("run-dispatch-work-"); db = createDb(external ?? database!.connectionString, { maxConnections: 6, idleTimeoutSeconds: 0, maxLifetimeSeconds: 0 }); + await Promise.all(Array.from({ length: 6 }, () => db.execute(sql`select pg_sleep(0.01)`))); }, 90_000); beforeEach(async () => { await db.execute(sql`TRUNCATE companies CASCADE`); heartbeat = heartbeatService(db); draining = false; errors.length = 0; reads = vi.spyOn(db, "select"); - vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"], shouldAdvanceTime: true }); }); afterEach(async () => { await worker?.stop(); await coordinator?.stop(); @@ -61,11 +64,13 @@ describe.skipIf(!support.supported)("run dispatch production wiring (controlled } async function quiet() { await heartbeat.drainActiveRunExecutions(); - await vi.advanceTimersByTimeAsync(1); await vi.waitFor(() => expect(coordinator!.nextWakeAt()).toBeNull(), { timeout: 5_000 }); const count = reads.mock.calls.length; - await vi.advanceTimersByTimeAsync(20 * 60_000); - expect(reads).toHaveBeenCalledTimes(count); + vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"] }); + try { + await vi.advanceTimersByTimeAsync(20 * 60_000); + expect(reads).toHaveBeenCalledTimes(count); + } finally { vi.useRealTimers(); } } async function status(id: string) { return db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, id)).then(rows => rows[0]); @@ -81,15 +86,14 @@ describe.skipIf(!support.supported)("run dispatch production wiring (controlled }); it("ordinary retry creation wakes the worker, respects the exact deadline, dispatches and settles", async () => { const f = await fixture(); await start(); await quiet(); - const result = await heartbeat.scheduleBoundedRetry(f.runId, { delayMs: 60_000, random: () => 0 }); + const result = await heartbeat.scheduleBoundedRetry(f.runId, { delayMs: 2_000, random: () => 0 }); expect(result.outcome).toBe("scheduled"); if (result.outcome !== "scheduled") throw new Error("Expected retry creation"); - await vi.advanceTimersByTimeAsync(1); await vi.waitFor(() => expect(coordinator!.nextWakeAt()).toBe(result.dueAt.getTime())); expect((await status(result.run.id)).status).toBe("scheduled_retry"); - await vi.advanceTimersByTimeAsync(result.dueAt.getTime() - Date.now() - 1); + await new Promise(resolve => setTimeout(resolve, 100)); + expect(Date.now()).toBeLessThan(result.dueAt.getTime()); expect((await status(result.run.id)).status).toBe("scheduled_retry"); - await vi.advanceTimersByTimeAsync(1); await completed(result.run.id); expect(await db.select().from(heartbeatRuns).where(and(eq(heartbeatRuns.companyId, f.companyId), eq(heartbeatRuns.retryOfRunId, f.runId)))).toHaveLength(1); await quiet(); @@ -105,20 +109,71 @@ describe.skipIf(!support.supported)("run dispatch production wiring (controlled const f = await fixture(); await start(); const scheduled = await heartbeat.scheduleBoundedRetry(f.runId, { delayMs: 600_000 }); if (scheduled.outcome !== "scheduled") throw new Error("Expected retry creation"); - await vi.advanceTimersByTimeAsync(1); await vi.waitFor(() => expect(coordinator!.nextWakeAt()).toBe(scheduled.dueAt.getTime())); await heartbeat.cancelRun(scheduled.run.id, "operator stop", { suppressImmediateRecovery: true }); await quiet(); expect((await status(scheduled.run.id)).status).toBe("cancelled"); }); it("retains the first retry written during drain and catches it up after resume", async () => { const f = await fixture(); await start(); await quiet(); draining = true; - const scheduled = await heartbeat.scheduleBoundedRetry(f.runId, { delayMs: 10_000 }); + const scheduled = await heartbeat.scheduleBoundedRetry(f.runId, { delayMs: 200 }); if (scheduled.outcome !== "scheduled") throw new Error("Expected retry creation"); - await vi.advanceTimersByTimeAsync(60_000); + await new Promise(resolve => setTimeout(resolve, 400)); expect((await status(scheduled.run.id)).status).toBe("scheduled_retry"); - draining = false; worker!.wake(); await vi.advanceTimersByTimeAsync(1); + draining = false; worker!.wake(); await completed(scheduled.run.id); await quiet(); }); + it("uses a committed budget raise to resume a queued run in the company it paused", async () => { + const f = await fixture(); + const budgets = budgetService(db); + await budgets.upsertPolicy(f.companyId, { scopeType: "company", scopeId: f.companyId, amount: 100, + metric: "billed_cents", windowKind: "lifetime", hardStopEnabled: true }, "dispatch-user"); + const [cost] = await db.insert(costEvents).values({ companyId: f.companyId, agentId: f.agentId, + provider: "fixture", biller: "fixture", model: "controlled", inputTokens: 0, outputTokens: 0, + costCents: 125, occurredAt: new Date() }).returning(); + await budgets.evaluateCostEvent(cost); + expect((await db.select().from(companies).where(eq(companies.id, f.companyId)))[0].status).toBe("paused"); + const [queued] = await db.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: f.agentId, + status: "queued", responsibleUserId: "dispatch-user", invocationSource: "automation", contextSnapshot: {} }).returning(); + await start(); await quiet(); expect((await status(queued.id)).status).toBe("queued"); + await budgets.upsertPolicy(f.companyId, { scopeType: "company", scopeId: f.companyId, amount: 10_000, + metric: "billed_cents", windowKind: "lifetime", hardStopEnabled: true }, "dispatch-user"); + await completed(queued.id); await quiet(); + }); + it("clearing an agent error wakes saved queue work only after its own transaction commits", async () => { + const f = await fixture(); await start(); await quiet(); + await db.update(agents).set({ status: "error" }).where(eq(agents.id, f.agentId)); + const [queued] = await db.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: f.agentId, + status: "queued", responsibleUserId: "dispatch-user", invocationSource: "automation", contextSnapshot: {} }).returning(); + let writing: ReturnType["clearError"]> | undefined; + try { + await db.transaction(async blocker => { + await blocker.select().from(agents).where(eq(agents.id, f.agentId)).for("update"); + writing = agentService(db).clearError(f.agentId); + void writing.catch(() => {}); + await vi.waitFor(() => expect(databaseWorkPending(db, DELIVERY_QUEUES.queuedRunDispatch)).toBe(true)); + expect((await status(queued.id)).status).toBe("queued"); + expect((await db.select().from(agents).where(eq(agents.id, f.agentId)))[0].status).toBe("error"); + }); + } finally { await writing; } + await completed(queued.id); await quiet(); + }); + it("keeps a foreign company's future retry intact while concurrent workers claim one due retry", async () => { + const selected = await fixture(), foreign = await fixture(); await start(); + const later = await heartbeat.scheduleBoundedRetry(foreign.runId, { delayMs: 600_000 }); + const due = await heartbeat.scheduleBoundedRetry(selected.runId, { delayMs: 500 }); + if (later.outcome !== "scheduled" || due.outcome !== "scheduled") throw new Error("Expected retries"); + const other = heartbeatService(db); + const competing = createDeliveryWorkCoordinator({ owner: db, canRun: () => true, onError: error => errors.push(error) }); + const competingWorker = other.registerRunDispatchWork(competing, () => true); + try { + await competingWorker.ready; await completed(due.run.id); await other.drainActiveRunExecutions(); + expect((await status(later.run.id)).status).toBe("scheduled_retry"); + expect((await status(later.run.id)).scheduledRetryAt).toEqual(later.dueAt); + expect(await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.retryOfRunId, selected.runId))).toHaveLength(1); + await heartbeat.cancelRun(later.run.id, "fixture teardown", { suppressImmediateRecovery: true }); + await quiet(); + } finally { await competingWorker.stop(); await competing.stop(); await other.drainActiveRunExecutions(); } + }); it("holds nested producer work until outer commit and rolls back without a successor", async () => { const f = await fixture(); await start(); await quiet(); let release!: () => void; const held = new Promise(resolve => { release = resolve; }); @@ -133,14 +188,14 @@ describe.skipIf(!support.supported)("run dispatch production wiring (controlled await held; }); await vi.waitFor(() => expect(queuedId).not.toBe("")); - await vi.advanceTimersByTimeAsync(1_000); + await new Promise(resolve => setTimeout(resolve, 100)); expect(await status(queuedId)).toBeUndefined(); - release(); await writing; await vi.advanceTimersByTimeAsync(1); await completed(queuedId); await quiet(); + release(); await writing; await completed(queuedId); await quiet(); await expect(db.transaction(async tx => { await tx.transaction(async nested => { await notifyRunDispatch(nested); await nested.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: f.agentId, status: "queued" }); }); throw new Error("rollback outer"); })).rejects.toThrow("rollback outer"); - await vi.advanceTimersByTimeAsync(1); await quiet(); + await quiet(); expect(await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.companyId, f.companyId))).toHaveLength(2); }); }); diff --git a/server/src/__tests__/server-startup-feedback-export.test.ts b/server/src/__tests__/server-startup-feedback-export.test.ts index b645d7437c8..c6d2ca536a0 100644 --- a/server/src/__tests__/server-startup-feedback-export.test.ts +++ b/server/src/__tests__/server-startup-feedback-export.test.ts @@ -134,6 +134,7 @@ const { })), reconcileHotRestartAdoption: vi.fn(async () => ({ mode: "none" })), reapOrphanedRuns: vi.fn(async () => ({ reaped: 0, runIds: [] })), + registerRunDispatchWork: vi.fn((_coordinator: unknown, _canRun: () => boolean) => ({ ready: Promise.resolve() })), registerScheduledRetryCancellationWork: vi.fn((_coordinator: unknown, _canRun: () => boolean) => ({ ready: Promise.resolve() })), registerStoppedInstructionCollectionWork: vi.fn((_coordinator: unknown, _canRun: () => boolean) => ({ ready: Promise.resolve() })), registerRetainedNativeSessionCleanupWork: vi.fn((_coordinator: unknown, _canRun: () => boolean) => ({ ready: Promise.resolve(), stop: vi.fn() })), @@ -858,6 +859,18 @@ describe("startServer feedback export wiring", () => { if (enabled) expect(heartbeatServiceMock.registerCapturedInstructionCommitWork.mock.calls[0][1]()).toBe(true); }); + it.each([false, true])("registers retry and queued dispatch only when maintenance is enabled (%s)", async enabled => { + loadConfigMock.mockReturnValue(buildTestConfig({ heartbeatSchedulerEnabled: enabled })); + await startServer(); await settleStartupHousekeeping(); + expect(heartbeatServiceMock.registerRunDispatchWork).toHaveBeenCalledTimes(enabled ? 1 : 0); + if (enabled) { + const [coordinator, admitted] = heartbeatServiceMock.registerRunDispatchWork.mock.calls[0]; + const app = await createAppMock.mock.results.at(-1)!.value; + expect(coordinator).toBe(app.locals.deliveryWork); + expect(admitted()).toBe(true); + } + }); + it.each([false, true])("registers retry cancellation recovery only when maintenance is enabled (%s)", async enabled => { loadConfigMock.mockReturnValue(buildTestConfig({ heartbeatSchedulerEnabled: enabled })); await startServer(); await settleStartupHousekeeping(); @@ -1262,6 +1275,7 @@ describe("startServer feedback export wiring", () => { expect(heartbeatServiceMock.sweepStaleIssueLocks).toHaveBeenCalledTimes(2); expect(heartbeatServiceMock.reapOrphanedRuns).toHaveBeenLastCalledWith({ staleThresholdMs: 300_000, skipScheduledRetryCancellation: true, skipCapturedInstructionCommit: true, skipInstructionDirectoryCleanup: true, skipStoppedInstructionCollection: true, skipCancelledWorkspacePreservation: true, skipObsoletePolicyReviews: true, skipAutomaticCompletionReviews: true, skipUnsafeWorkspaceExports: true, skipNativePendingEffects: true, skipNativeResponsePresentation: true, skipNativeReviewResponse: true, skipNativeStatusReassessment: true, skipRetainedNativeSessionCleanup: true, skipNativeWorkspaceFinalization: true, skipNativeResultCommit: true, skipNativeRunProjection: true, skipNativeRecoveryNotices: true, skipExternalConversationSettlement: true, skipExternalAgentSettlement: true, skipNativeEnvironmentSettlement: true }); expect(heartbeatServiceMock.reconcileCostAccounting).toHaveBeenCalledTimes(1); + expect(heartbeatServiceMock.promoteDueScheduledRetries).toHaveBeenCalledTimes(1); expect(retiredDetector).not.toHaveBeenCalled(); } finally { delete (runtime as Partial).reconcileProductivityReviews; diff --git a/server/src/services/budgets.ts b/server/src/services/budgets.ts index 6777124ee90..86d00c4d108 100644 --- a/server/src/services/budgets.ts +++ b/server/src/services/budgets.ts @@ -1,3 +1,4 @@ +import { notifyRunDispatch } from "./run-dispatch-notifications.js"; import { notifyApprovalWork } from "./approval-work-notifications.js"; import { notifyCompanyRetentionWork } from "./retention-work-notifications.js"; import { addCents, compareCents, normalizeCents } from "@paperclipai/shared"; @@ -287,6 +288,7 @@ export function budgetServiceInTransaction(db: Db, publications: ActivityPublica } if (policy.scopeType === "project") { + await notifyRunDispatch(db); await db .update(projects) .set({ @@ -300,6 +302,7 @@ export function budgetServiceInTransaction(db: Db, publications: ActivityPublica const scope = await resolveScopeRecord(db, "company", policy.scopeId); if (scope.paused) return; + await notifyRunDispatch(db); await notifyCompanyRetentionWork(db, policy.companyId); await notifyDeliveryWork(db, DELIVERY_QUEUES.agentTimers); await notifyDeliveryWork(db, DELIVERY_QUEUES.routineSchedules); @@ -346,6 +349,7 @@ export function budgetServiceInTransaction(db: Db, publications: ActivityPublica } if (policy.scopeType === "project") { + await notifyRunDispatch(db); await db .update(projects) .set({ @@ -360,6 +364,7 @@ export function budgetServiceInTransaction(db: Db, publications: ActivityPublica await notifyDeliveryWork(db, DELIVERY_QUEUES.skillSourceSync); const scope = await resolveScopeRecord(db, "company", policy.scopeId); if (scope.pauseReason !== "budget") return; + await notifyRunDispatch(db); await notifyCompanyRetentionWork(db, policy.companyId); await notifyDeliveryWork(db, DELIVERY_QUEUES.agentTimers); await notifyDeliveryWork(db, DELIVERY_QUEUES.routineSchedules); diff --git a/server/src/services/heartbeat/dispatch-work.test.ts b/server/src/services/heartbeat/dispatch-work.test.ts index 2ba36f38ef2..09a5f8b7ec0 100644 --- a/server/src/services/heartbeat/dispatch-work.test.ts +++ b/server/src/services/heartbeat/dispatch-work.test.ts @@ -1,6 +1,6 @@ import { PgDialect } from "drizzle-orm/pg-core"; import { afterEach, describe, expect, it, vi } from "vitest"; -import { installDatabaseWorkSignals } from "../../../../packages/db/src/work-signals.js"; +import { installDatabaseWorkSignals } from "@paperclipai/db"; import { createDeliveryWorkCoordinator } from "../delivery-work-coordinator.js"; import { notifyRunDispatch } from "../run-dispatch-notifications.js"; import { registerRunDispatchWork } from "./dispatch-work.js"; @@ -18,7 +18,7 @@ function fixture(input: { due?: number; suppressed?: boolean; draining?: boolean if (dialect.sqlToQuery(query).sql.includes("pg_current_xact_id")) return [{ xid: "42" }]; throw new Error("unexpected database query"); }); - const db = installDatabaseWorkSignals({ execute, async transaction(callback: (tx: any) => Promise) { + const db: { execute: typeof execute; transaction: (callback: (tx: any) => Promise) => Promise } = installDatabaseWorkSignals({ execute, async transaction(callback: (tx: any) => Promise): Promise { return callback({ execute, transaction: db.transaction }); } }); const state = { due: input.due ?? null as number | null, suppressed: input.suppressed ?? false, diff --git a/server/src/services/heartbeat/retries.test.ts b/server/src/services/heartbeat/retries.test.ts index a6382081167..3d7e9a1ec34 100644 --- a/server/src/services/heartbeat/retries.test.ts +++ b/server/src/services/heartbeat/retries.test.ts @@ -184,7 +184,9 @@ describe("heartbeat retry module callbacks", () => { }); }); -const support = await getEmbeddedPostgresTestSupport(); +import { resolveExternalTestDatabaseUrl } from "../../__tests__/helpers/external-test-database.js"; +const external = resolveExternalTestDatabaseUrl("PAPERCLIP_HEARTBEAT_RETRIES_TEST_DATABASE_URL"); +const support = external ? { supported: true, reason: undefined } : await getEmbeddedPostgresTestSupport(); const describePostgres = support.supported ? describe : describe.skip; if (!support.supported) console.warn(`Skipping retry PostgreSQL tests: ${support.reason ?? "unsupported host"}`); @@ -194,11 +196,11 @@ describePostgres("heartbeat retry module database wiring", () => { const now = new Date("2026-10-08T12:00:00Z"); beforeAll(async () => { - database = await startEmbeddedPostgresTestDatabase("heartbeat-retries-"); - db = createDb(database.connectionString); + if (!external) database = await startEmbeddedPostgresTestDatabase("heartbeat-retries-"); + db = createDb(external ?? database.connectionString, { maxConnections: 6 }); }, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS); - afterAll(async () => { await database?.cleanup(); }); + afterAll(async () => { await db?.$client.end({ timeout: 1 }); await database?.cleanup(); }); afterEach(async () => { await db.delete(issues); From 098111c2ca48da84a9eb89ab9c5112a011d4944b Mon Sep 17 00:00:00 2001 From: Dotta Date: Fri, 30 Oct 2026 16:42:12 -0500 Subject: [PATCH 08/70] test(heartbeat): retain process output and lifecycle acceptance evidence Co-Authored-By: Paperclip --- .../heartbeat-run-dispatch-lifecycle.test.ts} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename server/src/{services/heartbeat/dispatch-work.test.ts => __tests__/heartbeat-run-dispatch-lifecycle.test.ts} (100%) diff --git a/server/src/services/heartbeat/dispatch-work.test.ts b/server/src/__tests__/heartbeat-run-dispatch-lifecycle.test.ts similarity index 100% rename from server/src/services/heartbeat/dispatch-work.test.ts rename to server/src/__tests__/heartbeat-run-dispatch-lifecycle.test.ts From 50217ad02afcf37575c1ca24a266e3006f6bfb03 Mon Sep 17 00:00:00 2001 From: Dotta Date: Fri, 30 Oct 2026 16:42:27 -0500 Subject: [PATCH 09/70] test(heartbeat): use test-only signal setup and inspect saved stdout Co-Authored-By: Paperclip --- .../__tests__/heartbeat-run-dispatch-lifecycle.test.ts | 8 ++++---- server/src/__tests__/heartbeat-run-dispatch-work.test.ts | 1 + 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/server/src/__tests__/heartbeat-run-dispatch-lifecycle.test.ts b/server/src/__tests__/heartbeat-run-dispatch-lifecycle.test.ts index 09a5f8b7ec0..3d6566e543d 100644 --- a/server/src/__tests__/heartbeat-run-dispatch-lifecycle.test.ts +++ b/server/src/__tests__/heartbeat-run-dispatch-lifecycle.test.ts @@ -1,9 +1,9 @@ import { PgDialect } from "drizzle-orm/pg-core"; import { afterEach, describe, expect, it, vi } from "vitest"; -import { installDatabaseWorkSignals } from "@paperclipai/db"; -import { createDeliveryWorkCoordinator } from "../delivery-work-coordinator.js"; -import { notifyRunDispatch } from "../run-dispatch-notifications.js"; -import { registerRunDispatchWork } from "./dispatch-work.js"; +import { installDatabaseWorkSignals } from "../../../packages/db/src/work-signals.js"; +import { createDeliveryWorkCoordinator } from "../services/delivery-work-coordinator.js"; +import { notifyRunDispatch } from "../services/run-dispatch-notifications.js"; +import { registerRunDispatchWork } from "../services/heartbeat/dispatch-work.js"; const cleanup: Array<() => Promise> = []; afterEach(async () => { diff --git a/server/src/__tests__/heartbeat-run-dispatch-work.test.ts b/server/src/__tests__/heartbeat-run-dispatch-work.test.ts index 3d6f90aa71f..4a616887479 100644 --- a/server/src/__tests__/heartbeat-run-dispatch-work.test.ts +++ b/server/src/__tests__/heartbeat-run-dispatch-work.test.ts @@ -78,6 +78,7 @@ describe.skipIf(!support.supported)("run dispatch production wiring (controlled async function completed(id: string) { await vi.waitFor(async () => expect((await status(id))?.status).toBe("succeeded"), { timeout: 10_000 }); await heartbeat.drainActiveRunExecutions(); + expect((await status(id)).stdoutExcerpt).toContain("dispatch-proof"); } it("keeps fresh empty, configured idle and retained terminal history quiet", async () => { await start(); await quiet(); From 0f3017c96de9cc2c9adf907ae2d7c9145ba4aac0 Mon Sep 17 00:00:00 2001 From: Dotta Date: Fri, 30 Oct 2026 16:43:45 -0500 Subject: [PATCH 10/70] perf(server): archive attention from source deadlines Co-Authored-By: Paperclip --- .../2026-10-30-attention-retention-sleep.md | 48 +++ .../attention-retention-work.test.ts | 339 ++++++++++++++++++ .../src/__tests__/attention-service.test.ts | 2 +- server/src/index.ts | 25 +- .../__tests__/lifecycle.test.ts | 2 +- .../agent-lifecycle/adapters/records.ts | 13 +- server/src/services/accounting-integrity.ts | 3 + server/src/services/agent-configuration.ts | 2 + server/src/services/agent-deletion.ts | 2 + server/src/services/agent-hiring.ts | 2 + server/src/services/agent-lifecycle.ts | 2 + server/src/services/agent-project-access.ts | 2 + server/src/services/agents.ts | 6 +- server/src/services/attention-retention.ts | 204 +++++++++++ .../services/attention-source-eligibility.ts | 108 ++++++ server/src/services/attention.ts | 110 +----- server/src/services/budgets.ts | 6 +- server/src/services/company-portability.ts | 6 +- server/src/services/decision-retention.ts | 43 ++- .../services/decision-work-notifications.ts | 5 +- .../services/delivery-work-notifications.ts | 1 + .../execution-control-notifications.ts | 3 + .../services/grok-bot/workspace-inspection.ts | 2 + server/src/services/heartbeat-run-events.ts | 5 +- server/src/services/heartbeat/agent-status.ts | 3 + server/src/services/heartbeat/retries.ts | 2 + server/src/services/issue-recovery-actions.ts | 2 + .../issue-review-state-notifications.ts | 2 + server/src/services/issue-title.ts | 2 + server/src/services/issue-visibility.ts | 8 +- server/src/services/issues.ts | 9 + .../native-finalization-notifications.ts | 2 + .../native-provider-capacity-retry.ts | 2 + .../native-review-participant.ts | 2 + .../native-workspace-reconstruction.ts | 2 + .../paperclip-runner-tool-authority.ts | 2 + .../superseded-external-review-requests.ts | 2 + ...ative-status-reassessment-notifications.ts | 4 +- server/src/services/plugin-managed-agents.ts | 2 + .../services/retention-work-notifications.ts | 9 +- .../services/slack-conversation-lifecycle.ts | 2 + .../src/services/slack-conversation-state.ts | 2 + 42 files changed, 845 insertions(+), 155 deletions(-) create mode 100644 doc/plans/2026-10-30-attention-retention-sleep.md create mode 100644 server/src/__tests__/attention-retention-work.test.ts create mode 100644 server/src/services/attention-retention.ts create mode 100644 server/src/services/attention-source-eligibility.ts diff --git a/doc/plans/2026-10-30-attention-retention-sleep.md b/doc/plans/2026-10-30-attention-retention-sleep.md new file mode 100644 index 00000000000..c0e4e168bd0 --- /dev/null +++ b/doc/plans/2026-10-30-attention-retention-sleep.md @@ -0,0 +1,48 @@ +# Archive attention sources without repeated feed scans + +Attention retention now reconstructs source deadlines at startup and after committed source changes. It no longer rebuilds every active company's full feed on each heartbeat tick. The archive age remains 90 days. Keep and manual archive state remain authoritative, and automatic archive still commits its notification outbox atomically. + +This document records source ownership, producer coverage, and the evidence needed to review the replacement. It makes no claim about host suspension or external wake alarms. + +## Existing source rows own eligibility and deadlines + +`attention-retention.ts` selects at most 100 unfinished source candidates per reconstruction page. It excludes inactive companies and kept or archived retention states. The scoped coordinator retains eligible deadlines, retries concrete failures, and cancels entries that leave attention. It does not schedule another scan after settlement unless a producer changes a relevant source. + +| Source | Eligible durable state | Activity owner | +| --- | --- | --- | +| Decision | Open decision with a visible execution issue | Decision updated time | +| Interaction | Pending board-visible interaction; newest ordinary confirmation or independent quality gate | Interaction updated time | +| Recovery action | Active or escalated action owned by a user or board, with a visible source issue | Recovery updated time | +| Blocker attention | Board unblock descriptor or non-live terminal blocker selected by the issue owner | Blocked transition or terminal issue updated time | +| Review | Human participant, user assignee, pending approval, or stalled review selected by the issue owner | Issue updated time | +| Failed run | Failed or timed-out run with an exhaustion event, eligible agent and issue, and no newer matching run | Finished time, falling back to updated or created time | +| Budget alert | Open hard incident, or soft incident meeting the existing 85 percent rounded threshold | Incident updated time | +| Agent error | Agent in error status | Agent updated time | + +The feed and worker share confirmation collapse, newer-run probes, issue visibility, review eligibility, and terminal-blocker predicates. Issue graph and agent invokability logic remain with their existing owners. Blocker entries retain both the root used to reconstruct the graph and the terminal subject that owns retention state. PostgreSQL deadlines round submillisecond timestamps upward. The worker locks the source and terminal subject before archival. + +`productivity_review` generates no feed items and owes no retention work. Queue retention-day overrides control shelf presentation, which is computed on reads. They do not change the fixed automatic archive age. Approval retention, join-request retention, and archive notification delivery keep their existing workers. + +## Existing mutation boundaries reactivate work + +Committed hints identify a company, source kind, and source ID where those values are known. Retention's own writes use exact source hints; blocker retention changes invalidate the company/source because multiple blocked roots can project one terminal subject. Reconstruction pages and due batches remain bounded. + +Producer coverage includes decision creation and changes; interaction creation, resolution, expiry and supersession; issue creation, updates, blocker relations, checkout, adoption and release; agent hire, configuration, lifecycle, error clearing and deletion; budget incidents and company pause/resume; exhausted-run events and retry-run creation; native status, review, finalization and reconstruction; external conversation transitions; and imported blocker relations. Existing service transactions retain outer commit ownership. Savepoint release does not publish work before outer commit. + +The writer contract follows the existing delivery workers: serving processes use the transaction-aware service boundaries, and each serving process registers its workers. Independent raw SQL writers must supply equivalent ingress or restart reconstruction. Restores reconstruct work at startup. The change adds no permanent listener, independent timer, table, or Cloud wake mechanism. Existing sleep-safety gates remain in force. + +## Real PostgreSQL evidence proves quiet and reactivation + +The new acceptance suite exercises all eight sources, including terminal blocker identity and date, newest confirmations, independent quality gates, unhealthy reporting chains, conversation visibility, maintained reviews and pending approvals, newer-run suppression, ordinary task and interaction producers, ordinary exhaustion-event ingress, and ordinary error clearing. + +It also checks keep/revive, moved and cancelled deadlines, company activation and isolation, retained terminal history, restart, nested outer commit and rollback, first writes during drain, retry across drain, atomic archive/outbox rollback, and notifications during reconstruction. A 225-item backlog uses three startup pages and source-specific self-invalidations. SQL is counted at the PostgreSQL driver boundary. + +The old aggregate path issued 12 statements per tick in each of three configured-idle samples. The replacement issued zero statements during 300 seconds after settlement. One transaction-status read after an injected rollback is classified as uncertainty reconciliation; the following 300-second drain window must remain quiet. + +The focused suite supports `PAPERCLIP_ATTENTION_RETENTION_TEST_DATABASE_URL`, validates its dedicated `_test` database name, applies pending migrations, and prewarms a six-connection pool before advancing fake time. Providers are not involved in retention selection; the existing archive outbox delivery tests verify the separate controlled notification boundary. Fake time proves scheduling and SQL quiet, not database or host suspension. + +## Review and operational limits remain explicit + +Local embedded PostgreSQL could not start on the test host because system semaphores were exhausted. The accepted runs use isolated PostgreSQL 17 databases. A mixed consumer run also exposed fixture contamination from a lifecycle suite that retains native finalization rows; isolated reruns are required and preserve the failed run in the review evidence. Broad Linux CI remains the required full-suite gate. + +Review the source parity fixtures and producer hints together. The intended outcome is unchanged archive state and notifications with no recurring retention SQL when the sources remain unchanged. No merge or deployment is part of this change. diff --git a/server/src/__tests__/attention-retention-work.test.ts b/server/src/__tests__/attention-retention-work.test.ts new file mode 100644 index 00000000000..22a966d5fef --- /dev/null +++ b/server/src/__tests__/attention-retention-work.test.ts @@ -0,0 +1,339 @@ +import { randomUUID } from "node:crypto"; +import { and, eq, sql } from "drizzle-orm"; +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; +import { authUsers, companyMemberships, agents, approvals, budgetIncidents, budgetPolicies, companies, createDb, applyPendingMigrations, decisionArchiveNotificationOutbox, + decisionRetention, decisions, heartbeatRunEvents, heartbeatRuns, issueApprovals, issueRecoveryActions, + issueRelations, issueThreadInteractions, issues, type Db } from "@paperclipai/db"; +import { attentionRetentionService, registerAttentionRetentionWork } from "../services/attention-retention.js"; +import { attentionService } from "../services/attention.js"; +import { companyService } from "../services/companies.js"; +import { createDeliveryWorkCoordinator } from "../services/delivery-work-coordinator.js"; +import { decisionRetentionService, DEFAULT_DECISION_ARCHIVE_DAYS } from "../services/decision-retention.js"; +import { appendHeartbeatRunEvent } from "../services/heartbeat-run-events.js"; +import { issueThreadInteractionService } from "../services/issue-thread-interactions.js"; +import { createAgentLifecycle } from "../services/agent-lifecycle.js"; +import { issueService } from "../services/issues.js"; +import { notifyAttentionRetentionWork } from "../services/retention-work-notifications.js"; +import { idleWorkSnapshot } from "../services/task-admission.js"; +import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; + +import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js"; + +const externalUrl = resolveExternalTestDatabaseUrl("PAPERCLIP_ATTENTION_RETENTION_TEST_DATABASE_URL"); +const support = externalUrl ? { supported: true } : await getEmbeddedPostgresTestSupport(); +const delay = DEFAULT_DECISION_ARCHIVE_DAYS * 86_400_000; +(support.supported ? describe : describe.skip)("remaining attention retention", () => { + let temporary: Awaited> | undefined; + let db: ReturnType; + let companyId: string, agentId: string, draining = false; + let coordinator: ReturnType | undefined; + const errors: unknown[] = []; + const statements: string[] = []; + beforeAll(async () => { + if (externalUrl) await applyPendingMigrations(externalUrl); + else temporary = await startEmbeddedPostgresTestDatabase("attention-retention-"); + db = createDb(externalUrl ?? temporary!.connectionString, { maxConnections: 6, idleTimeoutSeconds: 0, maxLifetimeSeconds: 0 }); + db.$client.options.onnotice = () => {}; + db.$client.options.debug = (_connection, query) => statements.push(query); + }, 30_000); + beforeEach(async () => { + companyId = randomUUID(); agentId = randomUUID(); draining = false; errors.length = 0; + await db.insert(companies).values({ id: companyId, name: "Retention", issuePrefix: `A${companyId.slice(0, 6)}` }); + await db.insert(authUsers).values({ id: "board", name: "Board", email: "board@retention.test", createdAt: new Date(), updatedAt: new Date() }).onConflictDoNothing(); + await db.insert(companyMemberships).values({ companyId, principalType: "user", principalId: "board", membershipRole: "owner", status: "active" }); + await db.insert(agents).values({ id: agentId, companyId, name: "Origin", role: "engineer", status: "active" }); + await Promise.all(Array.from({ length: 6 }, () => db.execute(sql`select pg_sleep(0.01)`))); + vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout", "setInterval", "clearInterval"], shouldAdvanceTime: true }); + vi.setSystemTime(new Date("2026-11-01T12:00:00Z")); + }); + afterEach(async () => { + await coordinator?.stop(); coordinator = undefined; + vi.useRealTimers(); vi.restoreAllMocks(); + await db.transaction(async tx => { + await tx.execute(sql`set local client_min_messages = warning`); + await tx.execute(sql`drop trigger if exists attention_retention_fail on decision_retention`); + await tx.execute(sql`drop function if exists attention_retention_fail()`); + await tx.execute(sql`truncate companies cascade`); + }); + expect(errors).toEqual([]); + }); + afterAll(async () => { await db?.$client.end({ timeout: 1 }); await temporary?.cleanup(); }); + const old = () => new Date(Date.now() - delay - 600_000); + async function insertIssue(status = "todo", values: Partial = {}, database: Db = db) { + const [row] = await database.insert(issues).values({ companyId, title: randomUUID(), status, + createdByAgentId: agentId, createdAt: old(), updatedAt: old(), ...values }).returning(); + return row; + } + async function change(write: (tx: Db) => PromiseLike, scope = { companyId }) { + return db.transaction(async tx => { await notifyAttentionRetentionWork(tx, scope); return write(tx as unknown as Db); }); + } + const state = (kind: string, id: string) => db.select().from(decisionRetention).where(and( + eq(decisionRetention.companyId, companyId), eq(decisionRetention.sourceKind, kind), eq(decisionRetention.sourceId, id))) + .then(rows => rows[0] ?? null); + async function settle() { + await vi.waitFor(async () => { await vi.advanceTimersByTimeAsync(1000); expect(idleWorkSnapshot().active).toBe(0); }, { timeout: 15_000 }); + } + async function quiet() { + await settle(); expect(coordinator!.nextWakeAt()).toBeNull(); + const select = vi.spyOn(db, "select"), execute = vi.spyOn(db, "execute"), transaction = vi.spyOn(db, "transaction"); + statements.length = 0; + await vi.advanceTimersByTimeAsync(300_000); + expect(statements).toEqual([]); + expect(select).not.toHaveBeenCalled(); expect(execute).not.toHaveBeenCalled(); expect(transaction).not.toHaveBeenCalled(); + select.mockRestore(); execute.mockRestore(); transaction.mockRestore(); + } + async function start(service = attentionRetentionService(db)) { + coordinator = createDeliveryWorkCoordinator({ owner: db, canRun: () => !draining, canReconcile: () => true, onError: error => errors.push(error) }); + await registerAttentionRetentionWork(coordinator, service, () => !draining).ready; + return service; + } + async function archived(kind: string, id: string) { + await vi.waitFor(async () => { await vi.advanceTimersByTimeAsync(1000); expect((await state(kind, id))?.archivedAt).toBeInstanceOf(Date); }, { timeout: 15_000 }); + } + async function loadAll(service = attentionRetentionService(db)) { + const entries = []; + for (;;) { const page = await service.load({ companyId }); entries.push(...page.rows ?? []); if (page.complete) return entries; } + } + const mutation = (sourceKind: "review", sourceId: string) => ({ companyId, sourceKind, sourceId, + authActor: { type: "board" as const, source: "local_implicit" as const, userId: "board", companyIds: [companyId], isInstanceAdmin: true }, + actor: { actorType: "user" as const, actorId: "board", userId: "board", agentId: null, runId: null, agentApiKeyId: null, responsibleUserId: "board" } }); + + it("records the aggregate baseline SQL across old ticks and eliminates it after settlement", async () => { + await insertIssue("done"); + const ticks: number[] = []; + for (let tick = 0; tick < 3; tick++) { + statements.length = 0; + const feed = await attentionService(db).list(companyId, { includeDismissed: true, excludeSources: ["approval", "join_request"], all: true, allowUnscopedAll: true }); + await decisionRetentionService(db).autoArchive({ companyId, items: feed.items }); + ticks.push(statements.length); + await vi.advanceTimersByTimeAsync(30_000); + } + expect(ticks.every(count => count > 0)).toBe(true); + process.stdout.write(JSON.stringify({ baselineStatementsPer30sTick: ticks, treatmentIdle300s: 0 }) + "\n"); + await start(); await quiet(); + }); + + it("has no recurring SQL when empty or when terminal source history remains", async () => { + await insertIssue("done"); await insertIssue("cancelled"); + await start(); await quiet(); + }); + + it("wakes from the ordinary task producer, waits until due, archives with an outbox, and settles", async () => { + await start(); await quiet(); + const row = await issueService(db).create(companyId, { title: "Review a delivered result", status: "in_review", assigneeUserId: "board", createdByAgentId: agentId }); + await settle(); + const [saved] = await db.select({ at: sql`ceil(extract(epoch from ${issues.updatedAt}) * 1000)`.mapWith(Number) }).from(issues).where(eq(issues.id, row.id)); + const due = saved.at + delay; + expect(coordinator!.nextWakeAt()).toBe(due); expect(await state("review", row.id)).toBeNull(); + const select = vi.spyOn(db, "select"), execute = vi.spyOn(db, "execute"); + await vi.advanceTimersByTimeAsync(due - Date.now() - 1000); + expect(select).not.toHaveBeenCalled(); expect(execute).not.toHaveBeenCalled(); select.mockRestore(); execute.mockRestore(); + await vi.advanceTimersByTimeAsync(2000); await archived("review", row.id); await quiet(); + const outbox = await db.select().from(decisionArchiveNotificationOutbox); + expect(outbox).toHaveLength(1); expect(outbox[0]).toMatchObject({ companyId, sourceKind: "review", sourceId: row.id, originAgentId: agentId, originIssueId: row.id }); + expect((await db.select().from(issues).where(eq(issues.id, row.id)))[0].status).toBe("in_review"); + await coordinator!.stop(); coordinator = undefined; await start(); await quiet(); + }); + + it("matches the existing feed across all eight remaining source kinds and exclusion variants", async () => { + const origin = await insertIssue(); + const review = await insertIssue("in_review", { assigneeUserId: "board" }); + const blocker = await insertIssue("blocked", { unblockDescriptor: { owner: "board", action: "Choose a dependency" }, blockedTransitionAt: old() }); + await insertIssue("in_review", { hiddenAt: old(), assigneeUserId: "board" }); + const hidden = await insertIssue("todo", { hiddenAt: old() }); + const [run] = await db.insert(heartbeatRuns).values({ companyId, agentId, status: "failed", contextSnapshot: { issueId: origin.id }, createdAt: old(), updatedAt: old(), finishedAt: old() }).returning(); + await db.insert(heartbeatRunEvents).values({ companyId, agentId, runId: run.id, seq: 1, eventType: "lifecycle", message: "Bounded retry exhausted: fixture" }); + await db.insert(decisions).values({ companyId, originAgentId: agentId, originIssueId: origin.id, originRunId: run.id, + title: "Decision", body: "Choose", options: [], signedSpec: "fixture", targetSnapshots: {}, expiresAt: new Date(Date.now() + 86_400_000), updatedAt: old() }); + await db.insert(issueThreadInteractions).values([ + { companyId, issueId: origin.id, kind: "ask_user_questions", payload: { questions: [] }, updatedAt: old() }, + { companyId, issueId: origin.id, kind: "ask_user_questions", payload: { questions: [] }, addresseeAgentId: agentId, updatedAt: old() }, + { companyId, issueId: origin.id, kind: "ask_user_questions", payload: { questions: [] }, addresseeUserId: "board", updatedAt: old() }, + { companyId, issueId: hidden.id, kind: "ask_user_questions", payload: { questions: [] }, updatedAt: old() }, + ]); + await db.insert(issueRecoveryActions).values({ companyId, sourceIssueId: origin.id, kind: "execution", ownerType: "board", cause: "fixture", fingerprint: "fixture", nextAction: "Choose", updatedAt: old() }); + const [policy] = await db.insert(budgetPolicies).values({ companyId, scopeType: "company", scopeId: companyId, windowKind: "calendar_month_utc", amount: 100 }).returning(); + await db.insert(budgetIncidents).values({ companyId, policyId: policy.id, scopeType: "company", scopeId: companyId, + metric: "billed_cents", windowKind: "calendar_month_utc", windowStart: old(), windowEnd: new Date(), thresholdType: "hard", amountLimit: 100, amountObserved: 100, updatedAt: old() }); + await db.insert(agents).values({ companyId, name: "Error", role: "engineer", status: "error", updatedAt: old() }); + const feed = await attentionService(db).list(companyId, { all: true, allowUnscopedAll: true, includeDismissed: true, excludeSources: ["approval", "join_request"] }); + const expected = [...new Set(feed.items.map(item => `${item.sourceKind}:${item.subject.id}`))].sort(); + const actual = [...new Set((await loadAll()).map(item => `${item.sourceKind}:${item.sourceId}`))].sort(); + expect(new Set(feed.items.map(item => item.sourceKind)).size).toBe(8); expect(actual).toEqual(expected); + expect(actual).toContain(`review:${review.id}`); expect(actual).toContain(`blocker_attention:${blocker.id}`); + await start(); for (const item of feed.items) await archived(item.sourceKind, item.subject.id); await quiet(); + }); + + it("keeps unseen sources, cancels their deadline, and reactivates when unkept or revived", async () => { + const row = await insertIssue("in_review", { assigneeUserId: "board" }); + await decisionRetentionService(db).setKeep({ ...mutation("review", row.id), keep: true }); + await start(); await quiet(); expect((await state("review", row.id))?.archivedAt).toBeNull(); + await decisionRetentionService(db).setKeep({ ...mutation("review", row.id), keep: false }); + await archived("review", row.id); await quiet(); + await decisionRetentionService(db).revive(mutation("review", row.id)); await archived("review", row.id); await quiet(); + }); + + it("moves deadlines on source activity and cancels when the source leaves attention", async () => { + const row = await insertIssue("in_review", { assigneeUserId: "board", updatedAt: new Date() }); + await start(); await settle(); const first = coordinator!.nextWakeAt()!; + await change(tx => tx.update(issues).set({ updatedAt: new Date(Date.now() - delay + 600_000) }).where(eq(issues.id, row.id))); + await settle(); expect(coordinator!.nextWakeAt()).toBeLessThan(first); + await issueService(db).update(row.id, { status: "todo" }); await quiet(); expect(await state("review", row.id)).toBeNull(); + }); + + it("preserves outer transaction ownership and first writes during drain", async () => { + await start(); await quiet(); draining = true; + let id: string | undefined; + await db.transaction(async outer => { + await outer.transaction(async nested => { + await notifyAttentionRetentionWork(nested, { companyId }); + id = (await insertIssue("in_review", { assigneeUserId: "board" }, nested as unknown as Db)).id; + }); + await vi.advanceTimersByTimeAsync(60_000); expect(await state("review", id!)).toBeNull(); + }); + await vi.advanceTimersByTimeAsync(120_000); expect(await state("review", id!)).toBeNull(); + draining = false; await vi.advanceTimersByTimeAsync(30_000); await archived("review", id!); await quiet(); + await expect(db.transaction(async outer => { + await outer.transaction(async nested => { await notifyAttentionRetentionWork(nested, { companyId }); await insertIssue("in_review", { assigneeUserId: "board" }, nested as unknown as Db); }); + throw new Error("rollback"); + })).rejects.toThrow("rollback"); + await quiet(); expect(await db.select().from(decisionRetention)).toHaveLength(1); + }); + + it("isolates companies, catches up at activation, and bounds retained-history pages", async () => { + const other = randomUUID(); await db.insert(companies).values({ id: other, name: "Paused", issuePrefix: "RP", status: "paused" }); + const row = await insertIssue("in_review", { companyId: other, assigneeUserId: "board" }); + await db.insert(issues).values(Array.from({ length: 1000 }, () => ({ companyId, title: randomUUID(), status: "done", updatedAt: old() }))); + const service = attentionRetentionService(db); const page = await service.load({ companyId }); + expect(page).toEqual({ rows: [], complete: true }); + await start(service); await quiet(); expect(await db.select().from(decisionRetention)).toHaveLength(0); + await companyService(db).update(other, { status: "active" }); + await vi.waitFor(async () => { await vi.advanceTimersByTimeAsync(1000); expect(await db.select().from(decisionRetention)).toHaveLength(1); }); + expect((await db.select().from(decisionRetention))[0]).toMatchObject({ companyId: other, sourceId: row.id }); await quiet(); + }); + it("preserves terminal blocker source identity and its own activity deadline", async () => { + const root = await insertIssue("blocked", { unblockDescriptor: { owner: "board", action: "Pick an owner" }, blockedTransitionAt: old() }); + const terminal = await insertIssue("backlog", { assigneeAgentId: agentId, updatedAt: new Date(old().getTime() + 60_000) }); + await db.insert(issueRelations).values({ companyId, issueId: terminal.id, relatedIssueId: root.id, type: "blocks" }); + const feed = await attentionService(db).list(companyId, { all: true, allowUnscopedAll: true }); + const entries = await loadAll(); + const expected = feed.items.filter(item => item.sourceKind === "blocker_attention") + .map(item => `${item.subject.id}:${new Date(item.activityAt).toISOString()}`).sort(); + expect(entries.filter(item => item.sourceKind === "blocker_attention") + .map(item => `${item.sourceId}:${new Date(item.activityAt).toISOString()}`).sort()).toEqual(expected); + expect(expected).toContain(`${terminal.id}:${terminal.updatedAt.toISOString()}`); + await start(); await archived("blocker_attention", terminal.id); await archived("blocker_attention", root.id); await quiet(); + expect((await state("blocker_attention", terminal.id))?.sourceActivityAt).toEqual(terminal.updatedAt); + expect((await state("blocker_attention", root.id))?.sourceActivityAt).toEqual(root.blockedTransitionAt); + }); + + it("matches newest confirmations, independent gates, org invokability, and conversation visibility", async () => { + const origin = await insertIssue(); + const conversation = await insertIssue("in_review", { conversationAgentId: agentId, conversationUserId: "board", conversationState: "waiting", assigneeAgentId: agentId }); + const hidden = await insertIssue("todo", { hiddenAt: old() }); + const [paused, manager, unhealthy] = await db.insert(agents).values([ + { companyId, name: "Paused", role: "engineer", status: "paused" }, + { companyId, name: "Terminated", role: "manager", status: "terminated" }, + { companyId, name: "Unhealthy", role: "engineer", status: "active" }, + ]).returning(); + await db.update(agents).set({ reportsTo: manager.id }).where(eq(agents.id, unhealthy.id)); + const common = { companyId, issueId: origin.id, kind: "request_confirmation", payload: { prompt: "Approve?" }, createdAt: old(), updatedAt: old() }; + const [older, newest, independent] = await db.insert(issueThreadInteractions).values([ + common, { ...common, createdAt: new Date(old().getTime() + 1000) }, + { ...common, payload: { prompt: "Independent", qualityReview: { artifactKey: "copy" } } }, + { ...common, createdAt: new Date(), addresseeUserId: "board" }, + { ...common, addresseeAgentId: agentId }, + { ...common, issueId: hidden.id }, + { ...common, issueId: conversation.id, kind: "ask_user_questions", payload: { questions: [] } }, + { ...common, issueId: conversation.id, addresseeAgentId: paused.id }, + { ...common, issueId: conversation.id, addresseeAgentId: unhealthy.id, kind: "ask_user_questions", payload: { questions: [] } }, + ] as typeof issueThreadInteractions.$inferInsert[]).returning(); + const feed = await attentionService(db).list(companyId, { all: true, allowUnscopedAll: true }); + const expected = [...new Set(feed.items.map(item => `${item.sourceKind}:${item.subject.id}`))].sort(); + expect([...new Set((await loadAll()).map(item => `${item.sourceKind}:${item.sourceId}`))].sort()).toEqual(expected); + expect(expected).not.toContain(`issue_thread_interaction:${older.id}`); + expect(expected).toContain(`issue_thread_interaction:${newest.id}`); expect(expected).toContain(`issue_thread_interaction:${independent.id}`); + await start(); await archived("issue_thread_interaction", newest.id); await archived("issue_thread_interaction", independent.id); await quiet(); + await change(tx => tx.update(issueThreadInteractions).set({ status: "cancelled" }).where(eq(issueThreadInteractions.id, newest.id))); + await archived("issue_thread_interaction", older.id); await quiet(); + }); + + it("excludes maintained reviews until a pending approval creates human attention", async () => { + const row = await insertIssue("in_review", { assigneeAgentId: agentId }); + const [interaction] = await db.insert(issueThreadInteractions).values({ companyId, issueId: row.id, + kind: "request_confirmation", addresseeAgentId: agentId, payload: { prompt: "Agent verdict" } }).returning(); + expect((await loadAll()).some(entry => entry.sourceKind === "review")).toBe(false); + const [approval] = await db.insert(approvals).values({ companyId, type: "hire_agent", payload: {}, status: "pending" }).returning(); + await db.insert(issueApprovals).values({ companyId, issueId: row.id, approvalId: approval.id }); + const feed = await attentionService(db).list(companyId, { all: true, allowUnscopedAll: true }); + expect(feed.items.some(item => item.sourceKind === "review" && item.subject.id === row.id)).toBe(true); + await start(); await archived("review", row.id); await quiet(); + expect(await state("issue_thread_interaction", interaction.id)).toBeNull(); + }); + + it("reactivates an exhausted failed run through the ordinary durable event producer", async () => { + const origin = await insertIssue(); + const [run, newer] = await db.insert(heartbeatRuns).values([ + { companyId, agentId, status: "failed", contextSnapshot: { issueId: origin.id }, createdAt: old(), finishedAt: old() }, + { companyId, agentId, status: "succeeded", contextSnapshot: { issueId: origin.id }, createdAt: new Date(old().getTime() + 1000), finishedAt: old() }, + ]).returning(); + await start(); await quiet(); + await appendHeartbeatRunEvent(db, { companyId, agentId, runId: run.id, eventType: "lifecycle", message: "Bounded retry exhausted after 3 attempts" }); + await quiet(); expect(await state("failed_run", run.id)).toBeNull(); + await change(tx => tx.delete(heartbeatRuns).where(eq(heartbeatRuns.id, newer.id))); + await archived("failed_run", run.id); await quiet(); + }); + + it("cancels an agent error deadline through ordinary clearError", async () => { + await db.update(agents).set({ status: "error", updatedAt: new Date() }).where(eq(agents.id, agentId)); + await start(); await settle(); expect(coordinator!.nextWakeAt()).not.toBeNull(); + await createAgentLifecycle(db).clearError(agentId); await quiet(); + expect(await state("agent_error_alert", agentId)).toBeNull(); + }); + + it("drains a paged backlog with source-scoped self notifications and leaves completed history quiet", async () => { + await db.insert(issues).values(Array.from({ length: 225 }, () => ({ companyId, title: randomUUID(), status: "in_review", assigneeUserId: "board", updatedAt: old() }))); + const service = attentionRetentionService(db); + const load = vi.spyOn(service, "load"); + await start(service); + await vi.waitFor(async () => { await vi.advanceTimersByTimeAsync(1000); expect(await db.select().from(decisionRetention)).toHaveLength(225); }, { timeout: 30_000 }); + await quiet(); + const scopes = load.mock.calls.map(([scope]) => scope); + expect(scopes.filter(scope => !scope)).toHaveLength(3); + expect(scopes.filter(scope => scope && !scope.sourceId)).toHaveLength(0); + expect(scopes.filter(scope => scope?.sourceId).length).toBeLessThanOrEqual(225); + expect(new Set((await db.select().from(decisionRetention)).map(row => row.sourceId)).size).toBe(225); + }); + + it("retries a failed atomic archive across drain and preserves notifications during a scan", async () => { + const row = await insertIssue("in_review", { assigneeUserId: "board" }); + await db.execute(sql`create function attention_retention_fail() returns trigger language plpgsql as $$ begin raise exception 'retention injected'; end $$`); + await db.execute(sql`create trigger attention_retention_fail before insert on decision_retention for each row execute function attention_retention_fail()`); + await start(); expect(errors).toHaveLength(1); errors.length = 0; + expect(await state("review", row.id)).toBeNull(); expect(await db.select().from(decisionArchiveNotificationOutbox)).toHaveLength(0); + draining = true; await db.execute(sql`drop trigger attention_retention_fail on decision_retention`); + statements.length = 0; await vi.advanceTimersByTimeAsync(120_000); + expect(statements.length).toBeLessThanOrEqual(1); + expect(statements.every(query => query.includes("pg_xact_status"))).toBe(true); + statements.length = 0; await vi.advanceTimersByTimeAsync(300_000); expect(statements).toEqual([]); + draining = false; await vi.advanceTimersByTimeAsync(30_000); await archived("review", row.id); await quiet(); + await coordinator!.stop(); coordinator = undefined; + let release!: () => void; let scanned!: () => void; + const gate = new Promise(resolve => { release = resolve; }); + const entered = new Promise(resolve => { scanned = resolve; }); + const service = attentionRetentionService(db), original = service.load; + vi.spyOn(service, "load").mockImplementationOnce(async (...args) => { const result = await original(...args); scanned(); await gate; return result; }); + const starting = start(service); await entered; + const next = await change(async tx => insertIssue("in_review", { assigneeUserId: "board" }, tx)); + release(); await starting; await archived("review", next.id); await quiet(); + }); + + it("registers and cancels a deadline through ordinary interaction creation and withdrawal", async () => { + const origin = await insertIssue(); await start(); await quiet(); + const service = issueThreadInteractionService(db); + const interaction = await service.create(origin, { kind: "request_confirmation", payload: { version: 1, prompt: "Approve the result?" } }, { userId: "board" }); + await settle(); expect(coordinator!.nextWakeAt()).not.toBeNull(); expect(await state("issue_thread_interaction", interaction.id)).toBeNull(); + await service.withdrawInteraction(origin, interaction.id, { reason: "No longer needed" }, { userId: "board" }); await quiet(); + }); + +}); diff --git a/server/src/__tests__/attention-service.test.ts b/server/src/__tests__/attention-service.test.ts index a024ed421d7..5af78c2cb8b 100644 --- a/server/src/__tests__/attention-service.test.ts +++ b/server/src/__tests__/attention-service.test.ts @@ -62,7 +62,7 @@ describeEmbeddedPostgres("attention service", () => { beforeAll(async () => { if (!externalUrl) tempDb = await startEmbeddedPostgresTestDatabase("paperclip-attention-service-"); - db = createDb(externalUrl ?? tempDb!.connectionString); + db = createDb(externalUrl ?? tempDb!.connectionString, { maxConnections: 6 }); }, 30_000); afterEach(async () => { diff --git a/server/src/index.ts b/server/src/index.ts index c127753889c..48bcade6531 100644 --- a/server/src/index.ts +++ b/server/src/index.ts @@ -1,3 +1,4 @@ +import { attentionRetentionService, registerAttentionRetentionWork } from "./services/attention-retention.js"; import { registerProviderTraceRetentionWork } from "./services/provider-trace-retention-work.js"; import { registerRunEvidenceMigrationWork } from "./services/run-evidence-migration-work.js"; import { registerSkillVersionRetentionWork } from "./services/skill-version-retention-work.js"; @@ -127,7 +128,6 @@ import { cloudActorHeaderSourceFromHeaders, resolveCloudTenantActor } from "./mi import { feedbackService, applyManagedEnvironments, - attentionService, backfillPrincipalAccessCompatibility, backfillLegacyToolOAuthTokens, bootstrapExecutionPolicyFromEnv, @@ -1926,23 +1926,6 @@ async function startServerWithDatabaseTeardown( if (config.heartbeatSchedulerEnabled) await heartbeat.registerStoppedInstructionCollectionWork(deliveryWork, () => !isIdleTaskDrainActive() && !heartbeatSchedulerStopped && !isWarmStandby()).ready; - const runRetentionSweep = async () => { - const activeCompanies = await db.select({ id: companies.id }).from(companies).where(eq(companies.status, "active")); - let archived = 0; - for (const company of activeCompanies) { - // Cursor pagination rebuilds the whole feed for every page; one - // unscoped all-items build keeps this sweep at a single feed build - // per company per tick. - const page = await attentionService(db as any).list(company.id, { - includeDismissed: true, - excludeSources: ["approval", "join_request"], - all: true, - allowUnscopedAll: true, - }); - archived += await retentionExecutor.autoArchive({ companyId: company.id, items: page.items }); - } - return { archived }; - }; // Startup housekeeping: provider and connection sweeps that every timer tick // repeats and that neither serving requests nor run recovery depends on. // Readiness does not wait for it. It is tracked as scheduler work, so idle @@ -1985,7 +1968,8 @@ async function startServerWithDatabaseTeardown( () => !isIdleTaskDrainActive() && !isWarmStandby() && !heartbeatSchedulerStopped).ready); await step("join_request_retention", () => registerJoinRequestRetentionWork(deliveryWork, joinRequestRetentionService(db as any), () => !isIdleTaskDrainActive() && !isWarmStandby() && !heartbeatSchedulerStopped).ready); - await step("retention_sweep", runRetentionSweep); + await step("attention_retention", () => registerAttentionRetentionWork(deliveryWork, attentionRetentionService(db as any), + () => !isIdleTaskDrainActive() && !isWarmStandby() && !heartbeatSchedulerStopped).ready); }; const createHeartbeatSchedulerTick = (scheduled: ReturnType) => () => { @@ -1995,9 +1979,6 @@ async function startServerWithDatabaseTeardown( trackHeartbeatSchedulerWork((async () => { if (isIdleTaskDrainActive() || heartbeatSchedulerStopped) return; scheduleRoutineStatusReconcile(); - trackHeartbeatSchedulerWork(runRetentionSweep().catch((err: unknown) => { - logger.error({ err }, "decision retention sweep failed"); - })); const sweptRuntimeStatuses = heartbeat.sweepExpiredRuntimeStatuses(); if (sweptRuntimeStatuses > 0) { logger.info( diff --git a/server/src/modules/agent-lifecycle/__tests__/lifecycle.test.ts b/server/src/modules/agent-lifecycle/__tests__/lifecycle.test.ts index 246cb9f707a..404e76d6b30 100644 --- a/server/src/modules/agent-lifecycle/__tests__/lifecycle.test.ts +++ b/server/src/modules/agent-lifecycle/__tests__/lifecycle.test.ts @@ -40,7 +40,7 @@ const support = external ? { supported: true } : await getEmbeddedPostgresTestSu let companyId: string; const workers: ReturnType[] = []; beforeAll(async () => { - if (external) { await applyPendingMigrations(external); db = createDb(external); } + if (external) { await applyPendingMigrations(external); db = createDb(external, { maxConnections: 6 }); } else { database = await startEmbeddedPostgresTestDatabase("paperclip-agent-lifecycle-"); db = createDb(database.connectionString); } }, 90_000); afterAll(async () => { diff --git a/server/src/modules/agent-lifecycle/adapters/records.ts b/server/src/modules/agent-lifecycle/adapters/records.ts index 1c59758dd91..db842941836 100644 --- a/server/src/modules/agent-lifecycle/adapters/records.ts +++ b/server/src/modules/agent-lifecycle/adapters/records.ts @@ -80,18 +80,19 @@ export function agentRecords(db: Db, effects: LifecycleEffects) { throw conflict("Only agents in error status can have their error cleared"); } - const updated = await db - .update(agents) - .set({ + const updated = await effects.transaction(db, existing.companyId, async tx => { + await effects.beforeStatusChange(tx, existing.companyId, id, "error", "idle"); + return tx.update(agents).set({ status: "idle", pauseReason: null, pausedAt: null, errorReason: null, updatedAt: new Date(), }) - .where(and(eq(agents.id, id), eq(agents.status, "error"), eq(agents.lifecycleState, "ready"))) - .returning() - .then((rows) => rows[0] ?? null); + .where(and(eq(agents.id, id), eq(agents.status, "error"), eq(agents.lifecycleState, "ready"))) + .returning() + .then((rows) => rows[0] ?? null); + }); if (!updated) { throw conflict("Only agents in error status can have their error cleared"); diff --git a/server/src/services/accounting-integrity.ts b/server/src/services/accounting-integrity.ts index e3f5ef1f649..02b6043ab2d 100644 --- a/server/src/services/accounting-integrity.ts +++ b/server/src/services/accounting-integrity.ts @@ -1,3 +1,4 @@ +import { notifyAttentionRetentionWork } from "./retention-work-notifications.js"; import { and, eq, sql } from "drizzle-orm"; import { agents, agentRuntimeState, companies, heartbeatRuns, type Db } from "@paperclipai/db"; import { normalizeCents, usdToCents, type AccountingFinding, type AccountingHealth, type AccountingInspection } from "@paperclipai/shared"; @@ -75,6 +76,8 @@ export function accountingIntegrityService(db: Db, hooks: BudgetServiceHooks = { if (!reason.trim()) throw conflict("A repair reason is required"); const before = await inspect(tx, companyId); if (before.fingerprint !== fingerprint) throw conflict("Accounting changed since inspection; inspect again before repairing"); + if (before.findings.some(f => f.repairable && f.kind === "agent_projection")) + await notifyAttentionRetentionWork(tx, { companyId, sourceKind: "agent_error_alert" }); for (const finding of before.findings.filter(f => f.repairable)) { if (finding.kind === "company_projection") await tx.update(companies).set({ spentMonthlyCents: sql`${finding.expected.cents}::numeric`, updatedAt: new Date() }).where(eq(companies.id, companyId)); if (finding.kind === "agent_projection") await tx.update(agents).set({ spentMonthlyCents: sql`${finding.expected.cents}::numeric`, updatedAt: new Date() }).where(and(eq(agents.id, finding.entityId), eq(agents.companyId, companyId))); diff --git a/server/src/services/agent-configuration.ts b/server/src/services/agent-configuration.ts index 4139dc6dfe0..2c2c18ac1c7 100644 --- a/server/src/services/agent-configuration.ts +++ b/server/src/services/agent-configuration.ts @@ -1,3 +1,4 @@ +import { notifyAttentionRetentionWork } from "./retention-work-notifications.js"; import { notifySkillVersionPinChange } from "./skill-version-retention-notifications.js"; import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js"; import { notifyExternalConversationSettlement } from "./external-conversation-settlement-notifications.js"; @@ -16,6 +17,7 @@ import { clearPrimaryAgent } from "./primary-agent.js"; import { budgetServiceInTransaction } from "./budgets.js"; export async function prepareAgentConfiguration(db: Db, existing: AgentConfigurationRecord, data: AgentConfigurationPatch, options?: UpdateAgentOptions): Promise { + await notifyAttentionRetentionWork(db, { companyId: existing.companyId }); const id = existing.id; const { ensureManager, assertNoCycle, assertCompanyShortnameAvailable, assertBuiltInAgentMetadataMutationAllowed } = agentRecordQueries(db); if (existing.status === "pending_approval" && !options?.allowPendingApprovalConfigUpdate) { diff --git a/server/src/services/agent-deletion.ts b/server/src/services/agent-deletion.ts index 3650c40f1a6..50564fd9c4a 100644 --- a/server/src/services/agent-deletion.ts +++ b/server/src/services/agent-deletion.ts @@ -1,3 +1,4 @@ +import { notifyAttentionRetentionWork } from "./retention-work-notifications.js"; import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js"; import { notifyExternalConversationSettlement } from "./external-conversation-settlement-notifications.js"; import { notifyUnsafeWorkspaceExportRecovery } from "./unsafe-workspace-export-notifications.js"; @@ -34,6 +35,7 @@ export async function deleteAgentDependencies(tx: Db, companyId: string, id: str if (classifierHold) throw conflict("Wait for active prompt injection checks or resolve their unknown charges in Costs before deleting this agent", { code: "agent_prompt_injection_accounting_pending", }); + await notifyAttentionRetentionWork(tx, { companyId }); await notifyExternalConversationSettlement(tx, { companyId }); await issueThreadInteractionService(tx).cancelPendingForDeletedAddressee(companyId, id); await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors); diff --git a/server/src/services/agent-hiring.ts b/server/src/services/agent-hiring.ts index a150937a212..0c238907bbc 100644 --- a/server/src/services/agent-hiring.ts +++ b/server/src/services/agent-hiring.ts @@ -1,3 +1,4 @@ +import { notifyAttentionRetentionWork } from "./retention-work-notifications.js"; import { notifySkillVersionRetention, notifySkillVersionPinChange } from "./skill-version-retention-notifications.js"; import { agentAppearanceSchema, randomAgentAppearance } from "@paperclipai/shared"; import { normalizePaperclipRunnerAdapterConfig } from "@paperclipai/adapter-utils/server-utils"; @@ -16,6 +17,7 @@ import { initializePrimaryAgent } from "./primary-agent.js"; import { resolveAgentRunnerTaskDriverForCompany, resolveNewAgentRunnerForCompany } from "./agent-runner-selection.js"; export async function prepareAgentHire(db: Db, companyId: string, input: CreateAgentData, options?: CreateAgentOptions): Promise { + await notifyAttentionRetentionWork(db, { companyId }); const { runner, ...data } = input; const selection = options?.runnerResolved ? { adapterType: data.adapterType ?? "process", adapterConfig: isPlainRecord(data.adapterConfig) ? data.adapterConfig : {} } diff --git a/server/src/services/agent-lifecycle.ts b/server/src/services/agent-lifecycle.ts index 0a6dcaf0fc9..cb8818c4a15 100644 --- a/server/src/services/agent-lifecycle.ts +++ b/server/src/services/agent-lifecycle.ts @@ -1,3 +1,4 @@ +import { notifyAttentionRetentionWork } from "./retention-work-notifications.js"; import { notifyNativeReviewResponse } from "./native-review-response-notifications.js"; import { isAgentStatusInvokable } from "@paperclipai/shared"; import { notifyNativeResponsePresentation } from "./native-response-presentation-notifications.js"; @@ -38,6 +39,7 @@ export function createAgentLifecycleEffects(_hooks: BudgetServiceHooks = {}): li }, notifyWork: tx => notifyDeliveryWork(tx, DELIVERY_QUEUES.agentLifecycle), async beforeStatusChange(tx, companyId, agentId, before, after) { + await notifyAttentionRetentionWork(tx, { companyId }); if (before !== after) await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers); if (!isAgentStatusInvokable(before) && isAgentStatusInvokable(after)) { await notifyNativeResponsePresentation(tx, { companyId, agentId }); diff --git a/server/src/services/agent-project-access.ts b/server/src/services/agent-project-access.ts index 4c6ee81659a..feb183924c1 100644 --- a/server/src/services/agent-project-access.ts +++ b/server/src/services/agent-project-access.ts @@ -1,3 +1,4 @@ +import { notifyAttentionRetentionWork } from "./retention-work-notifications.js"; import { and, eq, inArray } from "drizzle-orm"; import { agents, projects, type Db } from "@paperclipai/db"; import { trustAuthorizationPolicySchema, type AgentProjectAccess } from "@paperclipai/shared"; @@ -55,6 +56,7 @@ export async function addAgentProjectAccess(db: Db, actor: AuthorizationActor, c const added = requested.filter(id => !previous.includes(id)); const next = [...previous, ...added]; if (added.length) { + await notifyAttentionRetentionWork(tx, { companyId, sourceKind: "agent_error_alert" }); await tx.update(agents).set({ permissions: { ...agent.permissions, authorizationPolicy: { ...policy, trustBoundary: { ...boundary, mode: "low_trust_review", companyId, projectIds: next }, } }, updatedAt: new Date() }).where(and(eq(agents.id, agentId), eq(agents.companyId, companyId))); diff --git a/server/src/services/agents.ts b/server/src/services/agents.ts index c9dd824f495..8a5f7551ee2 100644 --- a/server/src/services/agents.ts +++ b/server/src/services/agents.ts @@ -1,3 +1,4 @@ +import { notifyAttentionRetentionWork } from "./retention-work-notifications.js"; import { type BudgetServiceHooks } from "./budgets.js"; import { and, desc, eq, inArray } from "drizzle-orm"; @@ -52,7 +53,9 @@ export function agentService(db: Db, budgetHooks: BudgetServiceHooks = {}) { }); } - const updated = await db + const updated = await db.transaction(async tx => { + await notifyAttentionRetentionWork(tx, { companyId: existing.companyId }); + return tx .update(agents) .set({ permissions: normalizeAgentPermissions({ ...existing.permissions, ...permissions }), @@ -61,6 +64,7 @@ export function agentService(db: Db, budgetHooks: BudgetServiceHooks = {}) { .where(eq(agents.id, id)) .returning() .then((rows) => rows[0] ?? null); + }); return updated ? getById(updated.id) : null; }, diff --git a/server/src/services/attention-retention.ts b/server/src/services/attention-retention.ts new file mode 100644 index 00000000000..4622c4883bf --- /dev/null +++ b/server/src/services/attention-retention.ts @@ -0,0 +1,204 @@ +import { and, eq, isNull, sql, type SQL } from "drizzle-orm"; +import { agents, decisionRetention, decisions, heartbeatRuns, + issueApprovals, issueRecoveryActions, issueThreadInteractions, issues, approvals, type Db } from "@paperclipai/db"; +import type { AttentionSourceKind } from "@paperclipai/shared"; +import { evaluateAgentInvokability } from "./agent-invokability.js"; +import { collapsePendingConfirmationsToNewest, listFailedRunsWithNewerRun, readRunIssueId, requiresReviewAttention, terminalBlockerRequiresAttention } from "./attention-source-eligibility.js"; +import { decisionRetentionService, DEFAULT_DECISION_ARCHIVE_DAYS } from "./decision-retention.js"; +import type { createDeliveryWorkCoordinator } from "./delivery-work-coordinator.js"; +import { DELIVERY_QUEUES } from "./delivery-work-notifications.js"; +import { attentionIssueCondition } from "./issue-visibility.js"; +import { issueService } from "./issues.js"; +import { parseIssueExecutionState } from "./issue-execution-policy.js"; +import { isProspectiveBlockedTransition } from "./routable-blocked.js"; +import { registerScopedDeadlineWork } from "./scoped-deadline-work.js"; + +const archiveDelay = DEFAULT_DECISION_ARCHIVE_DAYS * 86_400_000; +const kinds = ["decision", "issue_thread_interaction", "recovery_action", "blocker_attention", "review", + "failed_run", "budget_alert", "agent_error_alert"] as const; +type Kind = typeof kinds[number]; +type Scope = { companyId?: string; sourceKind?: AttentionSourceKind; sourceId?: string }; +type Candidate = { sourceKind: Kind; sourceId: string; companyId: string; activityAt: string; deadlineAt: number }; +type Entry = Candidate & { id: string; dueAt: number; rootId: string; variant?: "owner" | "terminal" }; +type Cursor = { sourceKind: string; sourceId: string }; + +const sourceTables: Record = { + decision: sql`decisions`, issue_thread_interaction: sql`issue_thread_interactions`, + recovery_action: sql`issue_recovery_actions`, blocker_attention: sql`issues`, review: sql`issues`, + failed_run: sql`heartbeat_runs`, budget_alert: sql`budget_incidents`, agent_error_alert: sql`agents`, +}; + +const sources: Record = { + decision: sql`select 'decision' as source_kind, d.id::text as source_id, d.company_id, d.updated_at as activity_at + from decisions d where d.status = 'open'`, + issue_thread_interaction: sql`select 'issue_thread_interaction', i.id::text, i.company_id, i.updated_at + from issue_thread_interactions i where i.status = 'pending' and i.addressee_user_id is null`, + recovery_action: sql`select 'recovery_action', r.id::text, r.company_id, r.updated_at + from issue_recovery_actions r where r.status in ('active', 'escalated') and r.owner_type in ('user', 'board')`, + blocker_attention: sql`select 'blocker_attention', i.id::text, i.company_id, i.updated_at + from issues i where i.status = 'blocked' and i.hidden_at is null and i.harness_kind is null`, + review: sql`select 'review', i.id::text, i.company_id, i.updated_at + from issues i where i.status = 'in_review' and i.hidden_at is null and i.harness_kind is null`, + failed_run: sql`select 'failed_run', r.id::text, r.company_id, coalesce(r.finished_at, r.updated_at, r.created_at) + from heartbeat_runs r where r.status in ('failed', 'timed_out') and exists ( + select 1 from heartbeat_run_events e where e.run_id = r.id and e.company_id = r.company_id + and e.event_type = 'lifecycle' and e.message like 'Bounded retry exhausted%')`, + budget_alert: sql`select 'budget_alert', b.id::text, b.company_id, b.updated_at from budget_incidents b + where b.status = 'open' and (b.threshold_type = 'hard' or (b.amount_limit > 0 + and round(b.amount_observed::numeric / b.amount_limit * 10000) / 100 >= 85))`, + agent_error_alert: sql`select 'agent_error_alert', a.id::text, a.company_id, a.updated_at from agents a where a.status = 'error'`, +}; + +export function attentionRetentionService(db: Db) { + const cursors = new Map(); + async function candidates(database: Db, scope: Scope = {}, cursor?: Cursor, sourceId?: string): Promise { + const selected = kinds.filter(kind => !scope.sourceKind || scope.sourceKind === kind); + if (!selected.length) return []; + const rows = await database.execute(sql`with source_candidates(source_kind, source_id, company_id, activity_at) as (${sql.join(selected.map(kind => sources[kind]), sql` union all `)}) + select s.source_kind as "sourceKind", s.source_id as "sourceId", s.company_id as "companyId", + s.activity_at::text as "activityAt", ceil(extract(epoch from s.activity_at) * 1000 + ${archiveDelay}) as "deadlineAt" + from source_candidates s inner join companies c on c.id = s.company_id + left join decision_retention r on r.company_id = s.company_id and r.source_kind = s.source_kind and r.source_id = s.source_id + where c.status = 'active' + and (${scope.companyId ?? null}::uuid is null or s.company_id = ${scope.companyId ?? null}::uuid) + and (${sourceId ?? scope.sourceId ?? null}::text is null or s.source_id = ${sourceId ?? scope.sourceId ?? null}) + and (s.source_kind = 'blocker_attention' or ((r.id is null or r.keep = false) and r.archived_at is null)) + ${cursor ? sql`and (s.source_kind, s.source_id) > (${cursor.sourceKind}, ${cursor.sourceId})` : sql``} + order by s.source_kind, s.source_id limit 100`) as unknown as Candidate[]; + return rows.map(row => ({ ...row, deadlineAt: Number(row.deadlineAt) })); + } + + async function visibleIssue(database: Db, companyId: string, issueId: string, conversations = false) { + const [row] = await database.select().from(issues).where(and(eq(issues.companyId, companyId), eq(issues.id, issueId), + attentionIssueCondition(conversations))); + return row; + } + + async function eligible(database: Db, candidate: Candidate, variant?: "owner" | "terminal", lock = false): Promise { + let sourceId = candidate.sourceId; + let activityAt = candidate.activityAt; + let dueAt = candidate.deadlineAt; + if (candidate.sourceKind === "decision") { + const [row] = await database.select({ originIssueId: decisions.originIssueId }).from(decisions) + .where(and(eq(decisions.companyId, candidate.companyId), eq(decisions.id, sourceId), eq(decisions.status, "open"))); + if (!row || row.originIssueId && !await visibleIssue(database, candidate.companyId, row.originIssueId)) return null; + } else if (candidate.sourceKind === "issue_thread_interaction") { + const [row] = await database.select().from(issueThreadInteractions).where(and( + eq(issueThreadInteractions.companyId, candidate.companyId), eq(issueThreadInteractions.id, sourceId), + eq(issueThreadInteractions.status, "pending"), isNull(issueThreadInteractions.addresseeUserId))); + if (!row || !await visibleIssue(database, candidate.companyId, row.issueId, true)) return null; + const roster = row.addresseeAgentId ? await database.select().from(agents).where(eq(agents.companyId, candidate.companyId)) : []; + if (row.addresseeAgentId && evaluateAgentInvokability(roster.find(agent => agent.id === row.addresseeAgentId), roster).invokable) return null; + if (row.kind === "request_confirmation") { + const siblings = await database.select().from(issueThreadInteractions).where(and( + eq(issueThreadInteractions.companyId, candidate.companyId), eq(issueThreadInteractions.issueId, row.issueId), + eq(issueThreadInteractions.status, "pending"), eq(issueThreadInteractions.kind, "request_confirmation"), + isNull(issueThreadInteractions.addresseeUserId))); + const companyAgents = roster.length ? roster : await database.select().from(agents).where(eq(agents.companyId, candidate.companyId)); + const board = siblings.filter(sibling => !sibling.addresseeAgentId + || !evaluateAgentInvokability(companyAgents.find(agent => agent.id === sibling.addresseeAgentId), companyAgents).invokable); + if (!collapsePendingConfirmationsToNewest(board).some(sibling => sibling.id === sourceId)) return null; + } + } else if (candidate.sourceKind === "recovery_action") { + const [row] = await database.select().from(issueRecoveryActions).where(and(eq(issueRecoveryActions.companyId, candidate.companyId), eq(issueRecoveryActions.id, sourceId))); + if (!row || !await visibleIssue(database, candidate.companyId, row.sourceIssueId)) return null; + } else if (candidate.sourceKind === "review") { + const row = await visibleIssue(database, candidate.companyId, sourceId); + if (!row) return null; + const state = parseIssueExecutionState(row.executionState); + const [approval] = await database.select({ id: approvals.id }).from(issueApprovals).innerJoin(approvals, eq(approvals.id, issueApprovals.approvalId)) + .where(and(eq(issueApprovals.companyId, candidate.companyId), eq(approvals.companyId, candidate.companyId), + eq(issueApprovals.issueId, sourceId), eq(approvals.status, "pending"))).limit(1); + if (!requiresReviewAttention({ hasHumanParticipant: state?.status === "pending" && state.currentParticipant?.type === "user", + hasUserAssignee: Boolean(row.assigneeUserId), hasPendingApproval: Boolean(approval), stalled: false })) { + const attention = await issueService(database).listReviewAttention(candidate.companyId, [row]); + if (!requiresReviewAttention({ hasHumanParticipant: false, hasUserAssignee: false, hasPendingApproval: false, + stalled: attention.get(sourceId)?.state === "stalled" })) return null; + } + } else if (candidate.sourceKind === "blocker_attention") { + const row = await visibleIssue(database, candidate.companyId, sourceId); + if (!row) return null; + if (variant === "owner") { + if (row.unblockDescriptor?.owner !== "board" || !isProspectiveBlockedTransition(row)) return null; + const [activity] = await database.execute(sql`select blocked_transition_at::text as at, + ceil(extract(epoch from blocked_transition_at) * 1000 + ${archiveDelay}) as due from issues + where company_id = ${candidate.companyId}::uuid and id = ${row.id}::uuid`) as unknown as Array<{ at: string; due: string }>; + activityAt = activity.at; dueAt = Number(activity.due); + } else { + const attention = (await issueService(database).listBlockerAttention(candidate.companyId, [row])).get(row.id); + if (!terminalBlockerRequiresAttention(attention)) return null; + sourceId = attention.terminalBlockerIssueId ?? row.id; + if (lock && sourceId !== candidate.sourceId) await database.execute(sql`select id from issues + where company_id = ${candidate.companyId}::uuid and id = ${sourceId}::uuid for update`); + const terminal = await visibleIssue(database, candidate.companyId, sourceId); + if (!terminal) return null; + const [activity] = await database.execute(sql`select updated_at::text as at, + ceil(extract(epoch from updated_at) * 1000 + ${archiveDelay}) as due from issues + where company_id = ${candidate.companyId}::uuid and id = ${sourceId}::uuid`) as unknown as Array<{ at: string; due: string }>; + activityAt = activity.at; dueAt = Number(activity.due); + } + const [retention] = await database.select().from(decisionRetention).where(and(eq(decisionRetention.companyId, candidate.companyId), + eq(decisionRetention.sourceKind, candidate.sourceKind), eq(decisionRetention.sourceId, sourceId))); + if (retention?.keep || retention?.archivedAt) return null; + } else if (candidate.sourceKind === "failed_run") { + const [run] = await database.select().from(heartbeatRuns).where(and(eq(heartbeatRuns.companyId, candidate.companyId), eq(heartbeatRuns.id, sourceId))); + if (!run) return null; + const [agent] = await database.select({ id: agents.id }).from(agents).where(and(eq(agents.companyId, candidate.companyId), + eq(agents.id, run.agentId), sql`${agents.status} <> 'terminated'`)); + const issueId = readRunIssueId(run.contextSnapshot); + if (!agent || issueId && !await visibleIssue(database, candidate.companyId, issueId)) return null; + if ((await listFailedRunsWithNewerRun(database, candidate.companyId, [{ ...run, issueId }])).has(run.id)) return null; + } + return { ...candidate, sourceId, activityAt, rootId: candidate.sourceId, variant, + id: `${candidate.sourceKind}:${candidate.sourceId}:${variant ?? "source"}`, dueAt }; + } + + async function load(scope?: Scope, signal?: AbortSignal) { + const key = JSON.stringify(scope ?? {}); + const rows = await candidates(db, scope, cursors.get(key)); + const entries: Entry[] = []; + for (const row of rows) { + if (signal?.aborted) return { rows: null, complete: true }; + for (const variant of row.sourceKind === "blocker_attention" ? ["owner", "terminal"] as const : [undefined]) { + const entry = await eligible(db, row, variant); + if (entry) entries.push(entry); + } + } + if (rows.length === 100) { + const last = rows[rows.length - 1]!; + cursors.set(key, { sourceKind: last.sourceKind, sourceId: last.sourceId }); + } else cursors.delete(key); + return { rows: entries, complete: rows.length < 100 }; + } + + async function check(entry: Entry) { + return db.transaction(async transaction => { + const database = transaction as unknown as Db; + await database.execute(sql`select id from ${sourceTables[entry.sourceKind]} + where company_id = ${entry.companyId}::uuid and id = ${entry.rootId}::uuid for update`); + const [candidate] = await candidates(database, { companyId: entry.companyId, sourceKind: entry.sourceKind }, undefined, entry.rootId); + if (!candidate) return null; + const current = await eligible(database, candidate, entry.variant, true); + if (!current || current.dueAt > Date.now()) return current; + const retention = decisionRetentionService(database); + const item = { sourceKind: current.sourceKind, subject: { id: current.sourceId }, activityAt: current.activityAt }; + const state = (await retention.syncItems(current.companyId, [item])).get(`${current.sourceKind}:${current.sourceId}`)!; + await retention.autoArchive({ companyId: current.companyId, items: [{ ...item, + keep: state.keep, archivedAt: state.archivedAt?.toISOString() ?? null, retentionVersion: state.version }] }); + return null; + }); + } + return { load, check }; +} + +export function registerAttentionRetentionWork(coordinator: ReturnType, + service: ReturnType, canRun: () => boolean) { + return registerScopedDeadlineWork(coordinator, { + queue: DELIVERY_QUEUES.attentionRetention, + decodeScope: hint => hint === "*" ? undefined : JSON.parse(hint) as Scope, + matches: (entry, scope) => (!scope?.companyId || scope.companyId === entry.companyId) + && (!scope?.sourceKind || scope.sourceKind === entry.sourceKind) + && (!scope?.sourceId || scope.sourceId === entry.sourceId), + load: service.load, check: service.check, + }, canRun); +} diff --git a/server/src/services/attention-source-eligibility.ts b/server/src/services/attention-source-eligibility.ts new file mode 100644 index 00000000000..6e3f484c03a --- /dev/null +++ b/server/src/services/attention-source-eligibility.ts @@ -0,0 +1,108 @@ +import { and, eq, gt, inArray, sql, type SQL } from "drizzle-orm"; +import { heartbeatRuns, type Db } from "@paperclipai/db"; + +export function collapsePendingConfirmationsToNewest(rows: T[]) { + const independent = (row: T) => Boolean(row.payload && typeof row.payload === "object" && "qualityReview" in row.payload && row.payload.qualityReview); + const newestByGroup = new Map(); + for (const row of rows) { + if (row.kind !== "request_confirmation" || independent(row)) continue; + const groupKey = `${row.issueId}:${row.kind}`; + const newest = newestByGroup.get(groupKey); + if ( + !newest + || row.createdAt.getTime() > newest.createdAt.getTime() + || (row.createdAt.getTime() === newest.createdAt.getTime() && row.id > newest.id) + ) { + newestByGroup.set(groupKey, row); + } + } + + return rows.filter((row) => ( + row.kind !== "request_confirmation" + || independent(row) + || newestByGroup.get(`${row.issueId}:${row.kind}`)?.id === row.id + )); +} + +export function readRunIssueId(contextSnapshot: Record | null) { + const issueId = contextSnapshot?.issueId ?? contextSnapshot?.taskId; + return typeof issueId === "string" && issueId.length > 0 ? issueId : null; +} + +type FailedRunProbe = { id: string; agentId: string; issueId: string | null; createdAt: Date }; + +const FAILED_RUN_PROBE_BATCH_SIZE = 1000; + +export async function listFailedRunsWithNewerRun(db: Db, companyId: string, failedRuns: FailedRunProbe[]) { + const withNewerRun = new Set(); + const keyed = failedRuns.filter(run => run.issueId !== null); + for (let start = 0; start < keyed.length; start += FAILED_RUN_PROBE_BATCH_SIZE) { + const batch = keyed.slice(start, start + FAILED_RUN_PROBE_BATCH_SIZE); + for (const id of await listFailedRunBatchWithNewerRun(db, companyId, batch)) withNewerRun.add(id); + } + const unkeyed = failedRuns.filter(run => run.issueId === null); + if (unkeyed.length === 0) return withNewerRun; + const agentIds = [...new Set(unkeyed.map(run => run.agentId))]; + const oldestCreatedAt = unkeyed.reduce((oldest, run) => run.createdAt < oldest ? run.createdAt : oldest, + unkeyed[0]!.createdAt); + const latestByAgent = new Map(); + for (let start = 0; start < agentIds.length; start += FAILED_RUN_PROBE_BATCH_SIZE) { + const rows = await db.select({ + agentId: heartbeatRuns.agentId, + createdAt: sql`max(${heartbeatRuns.createdAt})`.mapWith(heartbeatRuns.createdAt), + }).from(heartbeatRuns).where(and( + eq(heartbeatRuns.companyId, companyId), + inArray(heartbeatRuns.agentId, agentIds.slice(start, start + FAILED_RUN_PROBE_BATCH_SIZE)), + gt(heartbeatRuns.createdAt, oldestCreatedAt), + sql`coalesce(${heartbeatRuns.contextSnapshot} ->> 'issueId', ${heartbeatRuns.contextSnapshot} ->> 'taskId', '') = ''`, + )).groupBy(heartbeatRuns.agentId); + for (const row of rows) latestByAgent.set(row.agentId, row.createdAt); + } + for (const run of unkeyed) { + if ((latestByAgent.get(run.agentId)?.getTime() ?? 0) > run.createdAt.getTime()) withNewerRun.add(run.id); + } + return withNewerRun; +} + +async function listFailedRunBatchWithNewerRun(db: Db, companyId: string, failedRuns: FailedRunProbe[]) { + if (failedRuns.length === 0) return []; + const failed = sql.join( + failedRuns.map((run) => sql`(${run.id}::uuid, ${run.agentId}::uuid, ${run.issueId}::text, ${new Date(run.createdAt.getTime() + 1).toISOString()}::timestamptz)`), + sql`, `, + ); + const runIssueId = sql`(${heartbeatRuns.contextSnapshot} ->> 'issueId')`; + const runTaskId = sql`(${heartbeatRuns.contextSnapshot} ->> 'taskId')`; + const newerRunExists = (match: SQL) => sql`exists ( + select 1 from ${heartbeatRuns} + where ${heartbeatRuns.companyId} = ${companyId} + and ${heartbeatRuns.agentId} = failed.agent_id + and ${heartbeatRuns.createdAt} >= failed.newer_from + and ${match} + )`; + const rows = await db.execute(sql` + select failed.id + from (values ${failed}) as failed (id, agent_id, issue_id, newer_from) + where ${newerRunExists(sql`${runIssueId} = failed.issue_id`)} + or ${newerRunExists(sql`${runTaskId} = failed.issue_id and ${runIssueId} is null`)} + `) as unknown as Array<{ id: string }>; + return rows.map((row) => row.id); +} + + +export function requiresReviewAttention(input: { + hasHumanParticipant: boolean; hasUserAssignee: boolean; hasPendingApproval: boolean; stalled: boolean; +}) { + return input.hasHumanParticipant || input.hasUserAssignee || input.hasPendingApproval || input.stalled; +} + +export function terminalBlockerRequiresAttention( + attention: T | null | undefined, +): attention is T & { state: "stalled" | "needs_attention" } { + return Boolean(attention && !attention.blockingTreeLive && ["stalled", "needs_attention"].includes(attention.state ?? "")); +} diff --git a/server/src/services/attention.ts b/server/src/services/attention.ts index 8d5dedd3b03..9f18f7d8104 100644 --- a/server/src/services/attention.ts +++ b/server/src/services/attention.ts @@ -1,5 +1,6 @@ +import { collapsePendingConfirmationsToNewest, readRunIssueId, listFailedRunsWithNewerRun, requiresReviewAttention, terminalBlockerRequiresAttention } from "./attention-source-eligibility.js"; import { approvalExecutionIssueCondition, firstApprovalIssueId } from "./approval-visibility.js"; -import { and, asc, desc, eq, gt, inArray, isNotNull, isNull, notInArray, or, sql, type SQL } from "drizzle-orm"; +import { and, asc, desc, eq, inArray, isNull, notInArray, or, sql, type SQL } from "drizzle-orm"; import type { Db } from "@paperclipai/db"; import { agents, @@ -58,7 +59,7 @@ import { BLOCKER_ATTENTION_MAX_NODES, issueService, } from "./issues.js"; -import { executionIssueCondition, visibleIssueCondition } from "./issue-visibility.js"; +import { attentionIssueCondition, executionIssueCondition } from "./issue-visibility.js"; import { parseIssueExecutionState } from "./issue-execution-policy.js"; import { isProspectiveBlockedTransition } from "./routable-blocked.js"; import { issueReadSqlCondition, type AuthorizationActor } from "./authorization.js"; @@ -766,37 +767,6 @@ export function interactionResolverAudience( }; } -function collapsePendingConfirmationsToNewest(rows: T[]) { - // Quality gates bind independent outputs; a newer visual approval must not - // hide a still-pending copy approval (or affect its badge count). - const independent = (row: T) => Boolean(row.payload && typeof row.payload === "object" && "qualityReview" in row.payload && row.payload.qualityReview); - const newestByGroup = new Map(); - for (const row of rows) { - if (row.kind !== "request_confirmation" || independent(row)) continue; - const groupKey = `${row.issueId}:${row.kind}`; - const newest = newestByGroup.get(groupKey); - if ( - !newest - || row.createdAt.getTime() > newest.createdAt.getTime() - || (row.createdAt.getTime() === newest.createdAt.getTime() && row.id > newest.id) - ) { - newestByGroup.set(groupKey, row); - } - } - - return rows.filter((row) => ( - row.kind !== "request_confirmation" - || independent(row) - || newestByGroup.get(`${row.issueId}:${row.kind}`)?.id === row.id - )); -} - function budgetObservedPercent(amountObserved: number, amountLimit: number) { return amountLimit > 0 ? Math.round((amountObserved / amountLimit) * 10_000) / 100 : 0; } @@ -866,9 +836,7 @@ async function issueSummaryMap( .where(and( eq(issues.companyId, companyId), inArray(issues.id, ids), - includeConversations - ? or(executionIssueCondition(), and(visibleIssueCondition(), isNotNull(issues.conversationAgentId))) - : executionIssueCondition(), + attentionIssueCondition(includeConversations), readCondition, )); return new Map(rows.map((row) => [row.id, { @@ -1108,70 +1076,6 @@ function resolveBlockingIssue( return null; } -function readRunIssueId(contextSnapshot: Record | null) { - const issueId = contextSnapshot?.issueId ?? contextSnapshot?.taskId; - return typeof issueId === "string" && issueId.length > 0 ? issueId : null; -} - -type FailedRunProbe = { id: string; agentId: string; issueId: string | null; createdAt: Date }; - -const FAILED_RUN_PROBE_BATCH_SIZE = 1000; - -async function listFailedRunsWithNewerRun(db: Db, companyId: string, failedRuns: FailedRunProbe[]) { - const withNewerRun = new Set(); - const keyed = failedRuns.filter(run => run.issueId !== null); - for (let start = 0; start < keyed.length; start += FAILED_RUN_PROBE_BATCH_SIZE) { - const batch = keyed.slice(start, start + FAILED_RUN_PROBE_BATCH_SIZE); - for (const id of await listFailedRunBatchWithNewerRun(db, companyId, batch)) withNewerRun.add(id); - } - const unkeyed = failedRuns.filter(run => run.issueId === null); - if (unkeyed.length === 0) return withNewerRun; - const agentIds = [...new Set(unkeyed.map(run => run.agentId))]; - const oldestCreatedAt = unkeyed.reduce((oldest, run) => run.createdAt < oldest ? run.createdAt : oldest, - unkeyed[0]!.createdAt); - const latestByAgent = new Map(); - for (let start = 0; start < agentIds.length; start += FAILED_RUN_PROBE_BATCH_SIZE) { - const rows = await db.select({ - agentId: heartbeatRuns.agentId, - createdAt: sql`max(${heartbeatRuns.createdAt})`.mapWith(heartbeatRuns.createdAt), - }).from(heartbeatRuns).where(and( - eq(heartbeatRuns.companyId, companyId), - inArray(heartbeatRuns.agentId, agentIds.slice(start, start + FAILED_RUN_PROBE_BATCH_SIZE)), - gt(heartbeatRuns.createdAt, oldestCreatedAt), - sql`coalesce(${heartbeatRuns.contextSnapshot} ->> 'issueId', ${heartbeatRuns.contextSnapshot} ->> 'taskId', '') = ''`, - )).groupBy(heartbeatRuns.agentId); - for (const row of rows) latestByAgent.set(row.agentId, row.createdAt); - } - for (const run of unkeyed) { - if ((latestByAgent.get(run.agentId)?.getTime() ?? 0) > run.createdAt.getTime()) withNewerRun.add(run.id); - } - return withNewerRun; -} - -async function listFailedRunBatchWithNewerRun(db: Db, companyId: string, failedRuns: FailedRunProbe[]) { - if (failedRuns.length === 0) return []; - const failed = sql.join( - failedRuns.map((run) => sql`(${run.id}::uuid, ${run.agentId}::uuid, ${run.issueId}::text, ${new Date(run.createdAt.getTime() + 1).toISOString()}::timestamptz)`), - sql`, `, - ); - const runIssueId = sql`(${heartbeatRuns.contextSnapshot} ->> 'issueId')`; - const runTaskId = sql`(${heartbeatRuns.contextSnapshot} ->> 'taskId')`; - const newerRunExists = (match: SQL) => sql`exists ( - select 1 from ${heartbeatRuns} - where ${heartbeatRuns.companyId} = ${companyId} - and ${heartbeatRuns.agentId} = failed.agent_id - and ${heartbeatRuns.createdAt} >= failed.newer_from - and ${match} - )`; - const rows = await db.execute(sql` - select failed.id - from (values ${failed}) as failed (id, agent_id, issue_id, newer_from) - where ${newerRunExists(sql`${runIssueId} = failed.issue_id`)} - or ${newerRunExists(sql`${runTaskId} = failed.issue_id and ${runIssueId} is null`)} - `) as unknown as Array<{ id: string }>; - return rows.map((row) => row.id); -} - export function attentionService(db: Db, serviceOptions: AttentionServiceOptions = {}) { const openDecisionLimit = Math.min( Math.max(Math.trunc(serviceOptions.openDecisionLimit ?? OPEN_DECISION_DEFAULT_LIMIT), 1), @@ -1636,8 +1540,7 @@ export function attentionService(db: Db, serviceOptions: AttentionServiceOptions })); } const blockerAttention = issue.blockerAttention; - if (blockerAttention?.state !== "stalled" && blockerAttention?.state !== "needs_attention") continue; - if (blockerAttention.blockingTreeLive) continue; + if (!terminalBlockerRequiresAttention(blockerAttention)) continue; const issueSummary = blockedIssueSummaries.get(issue.id) ?? null; const terminalIssueId = blockerAttention.terminalBlockerIssueId ?? issue.id; const terminalSummary = terminalBlockerSummaries.get(terminalIssueId) @@ -1739,7 +1642,8 @@ export function attentionService(db: Db, serviceOptions: AttentionServiceOptions const pendingApprovalId = pendingApprovalByIssueId.get(review.id) ?? null; const reviewAttention = reviewAttentionByIssueId.get(review.id); const stalled = reviewAttention?.state === "stalled"; - if (!hasHumanParticipant && !review.assigneeUserId && !pendingApprovalId && !stalled) continue; + if (!requiresReviewAttention({ hasHumanParticipant, hasUserAssignee: Boolean(review.assigneeUserId), + hasPendingApproval: Boolean(pendingApprovalId), stalled })) continue; const issue = reviewIssueMap.get(review.id); if (!issue) continue; const dedupKey = `review:${review.id}`; diff --git a/server/src/services/budgets.ts b/server/src/services/budgets.ts index 6777124ee90..93403926772 100644 --- a/server/src/services/budgets.ts +++ b/server/src/services/budgets.ts @@ -1,5 +1,5 @@ import { notifyApprovalWork } from "./approval-work-notifications.js"; -import { notifyCompanyRetentionWork } from "./retention-work-notifications.js"; +import { notifyAttentionRetentionWork, notifyCompanyRetentionWork } from "./retention-work-notifications.js"; import { addCents, compareCents, normalizeCents } from "@paperclipai/shared"; import { and, desc, eq, inArray, sql, or, isNull } from "drizzle-orm"; import type { Db } from "@paperclipai/db"; @@ -466,6 +466,7 @@ export function budgetServiceInTransaction(db: Db, publications: ActivityPublica windowEnd: end, }); + await notifyAttentionRetentionWork(db, { companyId: policy.companyId, sourceKind: "budget_alert" }); if (thresholdType === "hard") await notifyApprovalWork(db, policy.companyId); const approval = thresholdType === "hard" ? await db @@ -505,6 +506,7 @@ export function budgetServiceInTransaction(db: Db, publications: ActivityPublica } async function resolveOpenSoftIncidents(policyId: string) { + await notifyAttentionRetentionWork(db, { sourceKind: "budget_alert" }); await db .update(budgetIncidents) .set({ @@ -527,6 +529,7 @@ export function budgetServiceInTransaction(db: Db, publications: ActivityPublica decidedByUserId: string | null, thresholdType?: BudgetThresholdType, ) { + await notifyAttentionRetentionWork(db, { sourceKind: "budget_alert" }); const condition = and(eq(budgetIncidents.policyId, policyId), eq(budgetIncidents.status, "open"), thresholdType ? eq(budgetIncidents.thresholdType, thresholdType) : undefined); const openRows = await db @@ -937,6 +940,7 @@ export function budgetServiceInTransaction(db: Db, publications: ActivityPublica } const now = new Date(); + await notifyAttentionRetentionWork(db, { companyId, sourceKind: "budget_alert" }); await notifyDeliveryWork(db, DELIVERY_QUEUES.budgetEnforcement, companyId); await db .update(budgetPolicies) diff --git a/server/src/services/company-portability.ts b/server/src/services/company-portability.ts index dc7dca311d6..fff45394e08 100644 --- a/server/src/services/company-portability.ts +++ b/server/src/services/company-portability.ts @@ -1,3 +1,4 @@ +import { notifyAttentionRetentionWork } from "./retention-work-notifications.js"; import { createAgentLifecycle } from "./agent-lifecycle.js"; import { agentAppearanceSchema, agentHarnessType, paperclipRunnerProfileForHarness, agentRunner, type AgentRunnerChoice } from "@paperclipai/shared"; import { resolveNewAgentRunnerForCompany } from "./agent-runner-selection.js"; @@ -6465,7 +6466,9 @@ export function companyPortabilityService(db: Db, storage?: StorageService) { } if (relationRows.length > 0) { const relationCompanyId = targetCompany.id; - await db + await db.transaction(async tx => { + await notifyAttentionRetentionWork(tx, { companyId: relationCompanyId }); + return tx .insert(issueRelations) .values(relationRows.map((row) => ({ companyId: relationCompanyId, @@ -6476,6 +6479,7 @@ export function companyPortabilityService(db: Db, storage?: StorageService) { createdByUserId: actorUserId ?? null, }))) .onConflictDoNothing(); + }); } } diff --git a/server/src/services/decision-retention.ts b/server/src/services/decision-retention.ts index fb22df2a583..b5ca6c9f1fb 100644 --- a/server/src/services/decision-retention.ts +++ b/server/src/services/decision-retention.ts @@ -1,3 +1,4 @@ +import { isProspectiveBlockedTransition } from "./routable-blocked.js"; import { notifyRetentionWork } from "./retention-work-notifications.js"; import { decisionArchiveDeliveryService, type ArchiveNotificationBatch } from "./decision-archive-delivery.js"; import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js"; @@ -15,7 +16,6 @@ import { issueRecoveryActions, issueThreadInteractions, issues, - joinRequests, } from "@paperclipai/db"; import type { AttentionArchiveManifestEntry, @@ -145,7 +145,7 @@ export function decisionRetentionService( const existing = new Set(rows.map(row => sourceKey(row.sourceKind, row.sourceId))); const missing = unique.filter(item => !existing.has(itemSourceKey(item))); if (missing.length) { - for (const kind of new Set(missing.map(item => item.sourceKind))) await notifyRetentionWork(tx, kind); + for (const item of missing) await notifyRetentionWork(tx, item.sourceKind, { companyId, sourceId: item.subject.id }); await tx.insert(decisionRetention).values(missing.map(item => ({ companyId, sourceKind: item.sourceKind, sourceId: item.subject.id, sourceActivityAt: new Date(item.activityAt), }))).onConflictDoNothing({ target: [decisionRetention.companyId, decisionRetention.sourceKind, decisionRetention.sourceId] }); @@ -155,7 +155,7 @@ export function decisionRetentionService( for (const item of unique) { const key = itemSourceKey(item), row = byKey.get(key), activityAt = new Date(item.activityAt); if (!row || row.sourceActivityAt.getTime() === activityAt.getTime()) continue; - await notifyRetentionWork(tx, item.sourceKind); + await notifyRetentionWork(tx, item.sourceKind, { companyId, sourceId: item.subject.id }); const [updated] = await tx.update(decisionRetention).set({ sourceActivityAt: activityAt, version: sql`${decisionRetention.version} + 1`, updatedAt: new Date(), }).where(and(eq(decisionRetention.id, row.id), ne(decisionRetention.sourceActivityAt, activityAt))).returning(); @@ -169,9 +169,9 @@ export function decisionRetentionService( return sourceKey(item.sourceKind, item.subject.id); } - async function writeRetention(kind: AttentionSourceKind, write: (tx: Db) => Promise) { + async function writeRetention(companyId: string, kind: AttentionSourceKind, sourceId: string, write: (tx: Db) => Promise) { return db.transaction(async tx => { - await notifyRetentionWork(tx, kind); + await notifyRetentionWork(tx, kind, { companyId, sourceId }); return write(tx as unknown as Db); }); } @@ -185,13 +185,28 @@ export function decisionRetentionService( } async function materializeSourceState(database: Db, input: { companyId: string; sourceKind: AttentionSourceKind; sourceId: string }) { - const table = input.sourceKind === "approval" ? approvals : input.sourceKind === "join_request" ? joinRequests : null; - if (!table) return; - const [source] = await database.select({ updatedAt: table.updatedAt }).from(table) - .where(and(eq(table.companyId, input.companyId), eq(table.id, input.sourceId))).for("update"); + const tables: Record = { + approval: "approvals", join_request: "join_requests", decision: "decisions", + issue_thread_interaction: "issue_thread_interactions", recovery_action: "issue_recovery_actions", + failed_run: "heartbeat_runs", productivity_review: "issues", blocker_attention: "issues", review: "issues", + budget_alert: "budget_incidents", agent_error_alert: "agents", + }; + if (input.sourceKind === "blocker_attention") { + const [source] = await database.select().from(issues).where(and(eq(issues.companyId, input.companyId), eq(issues.id, input.sourceId))).for("update"); + if (!source) throw notFound("Attention source not found"); + const activityAt = source.unblockDescriptor?.owner === "board" && isProspectiveBlockedTransition(source) + ? source.blockedTransitionAt : source.updatedAt; + await decisionRetentionService(database).syncItems(input.companyId, [{ + sourceKind: input.sourceKind, subject: { id: input.sourceId }, activityAt: activityAt.toISOString(), + }]); + return; + } + const activity = input.sourceKind === "failed_run" ? sql`coalesce(finished_at, updated_at, created_at)` : sql`updated_at`; + const [source] = await database.execute(sql`select ${activity} as "activityAt" from ${sql.identifier(tables[input.sourceKind])} + where company_id = ${input.companyId}::uuid and id = ${input.sourceId}::uuid for update`) as unknown as Array<{ activityAt: Date | string }>; if (!source) throw notFound("Attention source not found"); await decisionRetentionService(database).syncItems(input.companyId, [{ - sourceKind: input.sourceKind, subject: { id: input.sourceId }, activityAt: source.updatedAt.toISOString(), + sourceKind: input.sourceKind, subject: { id: input.sourceId }, activityAt: new Date(source.activityAt).toISOString(), }]); } @@ -206,7 +221,7 @@ export function decisionRetentionService( if (!(await canReadDecisionSource(db, input.authActor, input.companyId, input.sourceKind, input.sourceId))) { throw notFound("Attention source not found"); } - const updated = await writeRetention(input.sourceKind, async tx => { + const updated = await writeRetention(input.companyId, input.sourceKind, input.sourceId, async tx => { await materializeSourceState(tx, input); return tx.update(decisionRetention).set({ keep: input.keep, @@ -247,7 +262,7 @@ export function decisionRetentionService( throw notFound("Attention source not found"); } const now = new Date(); - const updated = await writeRetention(input.sourceKind, async tx => { + const updated = await writeRetention(input.companyId, input.sourceKind, input.sourceId, async tx => { await materializeSourceState(tx, input); return tx.update(decisionRetention).set(input.archived ? { archivedAt: now, @@ -324,7 +339,7 @@ export function decisionRetentionService( } const now = new Date(); for (const entry of input.manifest) { - await notifyRetentionWork(txDb, entry.sourceKind as AttentionSourceKind); + await notifyRetentionWork(txDb, entry.sourceKind as AttentionSourceKind, { companyId: input.companyId, sourceId: entry.sourceId }); const archived = await txDb.update(decisionRetention).set({ archivedAt: now, archivedReason: "accepted_proposal", @@ -368,7 +383,7 @@ export function decisionRetentionService( await db.transaction(async (tx) => { const txDb = tx as unknown as Db; for (const item of candidates) { - await notifyRetentionWork(txDb, item.sourceKind); + await notifyRetentionWork(txDb, item.sourceKind, { companyId: input.companyId, sourceId: item.subject.id }); const updated = await txDb.update(decisionRetention).set({ archivedAt: now, archivedReason: "idle_ttl", diff --git a/server/src/services/decision-work-notifications.ts b/server/src/services/decision-work-notifications.ts index 081edc8fc39..0abc8e95829 100644 --- a/server/src/services/decision-work-notifications.ts +++ b/server/src/services/decision-work-notifications.ts @@ -1,11 +1,14 @@ +import { notifyAttentionRetentionWork } from "./retention-work-notifications.js"; import type { Db } from "@paperclipai/db"; import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js"; -export function notifyDecisionCreated(transaction: object) { +export async function notifyDecisionCreated(transaction: object) { + await notifyAttentionRetentionWork(transaction, { sourceKind: "decision" }); return notifyDeliveryWork(transaction, DELIVERY_QUEUES.decisionExpiry); } export async function notifyDecisionChange(transaction: object) { + await notifyAttentionRetentionWork(transaction, { sourceKind: "decision" }); await notifyDeliveryWork(transaction, DELIVERY_QUEUES.decisionContinuation); await notifyDeliveryWork(transaction, DELIVERY_QUEUES.decisionExpiry); } diff --git a/server/src/services/delivery-work-notifications.ts b/server/src/services/delivery-work-notifications.ts index 9195f1fcb12..8951cdffa50 100644 --- a/server/src/services/delivery-work-notifications.ts +++ b/server/src/services/delivery-work-notifications.ts @@ -35,6 +35,7 @@ export const DELIVERY_QUEUES = { executionDisposition: "execution-disposition", environmentCleanup: "environment-cleanup", approvalQueueSeeding: "approval-queue-seeding", + attentionRetention: "attention-retention", approvalRetention: "approval-retention", joinRequestRetention: "join-request-retention", decisionTargets: "decision-targets", diff --git a/server/src/services/execution-control-notifications.ts b/server/src/services/execution-control-notifications.ts index 22c8913bbcd..a96ca761ec2 100644 --- a/server/src/services/execution-control-notifications.ts +++ b/server/src/services/execution-control-notifications.ts @@ -1,7 +1,9 @@ +import { notifyAttentionRetentionWork } from "./retention-work-notifications.js"; import type { Db } from "@paperclipai/db"; import { DELIVERY_QUEUES, notifyDeliveryWork, type DeliveryQueue } from "./delivery-work-notifications.js"; export async function notifyExecutionRunWork(transaction: object) { + await notifyAttentionRetentionWork(transaction); for (const queue of [DELIVERY_QUEUES.executionStatus, DELIVERY_QUEUES.executionFinalization, DELIVERY_QUEUES.executionReplacement, DELIVERY_QUEUES.executionDisposition, DELIVERY_QUEUES.issueMonitors]) { await notifyDeliveryWork(transaction, queue); @@ -14,6 +16,7 @@ export function writeExecutionControl( write: (tx: Db) => PromiseLike, ): Promise { return db.transaction(async transaction => { + await notifyAttentionRetentionWork(transaction); await notifyDeliveryWork(transaction, DELIVERY_QUEUES.issueMonitors); for (const topic of typeof queue === "string" ? [queue] : queue) await notifyDeliveryWork(transaction, topic); return write(transaction as unknown as Db); diff --git a/server/src/services/grok-bot/workspace-inspection.ts b/server/src/services/grok-bot/workspace-inspection.ts index f1db9a4a851..73cad27c8aa 100644 --- a/server/src/services/grok-bot/workspace-inspection.ts +++ b/server/src/services/grok-bot/workspace-inspection.ts @@ -1,3 +1,4 @@ +import { notifyAttentionRetentionWork } from "../retention-work-notifications.js"; import { and, eq, inArray, isNotNull, or, sql } from "drizzle-orm"; import { dotRunnerAssignments, environmentLeases, heartbeatRuns, issueRecoveryActions, issues, nativeRunFinalizations, type Db } from "@paperclipai/db"; import { authorizeExternalBotOperator } from "../external-bot-authority.js"; @@ -74,6 +75,7 @@ export async function inspectGrokWorkspaceObligations(db: Db, companyId: string, return { ...rest, externalWorkspaceInspection: inspection }; }; await tx.update(heartbeatRuns).set({ resultJson: retireFileObligation(run.resultJson!) }).where(eq(heartbeatRuns.id, run.id)); + if (actions.length) await notifyAttentionRetentionWork(tx, { companyId, sourceKind: "recovery_action" }); for (const action of actions) { await tx.update(issueRecoveryActions).set({ evidence: retireFileObligation(action.evidence), updatedAt: new Date(), nextAction: "Inspect Grok and verify its connection before sending a new task message. The old work will not be replayed.", diff --git a/server/src/services/heartbeat-run-events.ts b/server/src/services/heartbeat-run-events.ts index ebce78a8266..22219811668 100644 --- a/server/src/services/heartbeat-run-events.ts +++ b/server/src/services/heartbeat-run-events.ts @@ -1,3 +1,4 @@ +import { notifyAttentionRetentionWork } from "./retention-work-notifications.js"; import { readNativeEventEvidenceOutcome } from "./native-runtime/evidence-classifier.js"; import { notifyNativeStatusReassessment } from "./native-status-reassessment-notifications.js"; import { isNativeResponseEvidence, notifyNativeResponsePresentation } from "./native-response-presentation-notifications.js"; @@ -74,10 +75,11 @@ export async function appendHeartbeatRunEvent( db: Db, input: AppendHeartbeatRunEventInput, ): Promise { + const projectsRetention = input.eventType === "lifecycle" && Boolean(input.message?.startsWith("Bounded retry exhausted")); const projectsReassessment = readNativeEventEvidenceOutcome(input.payload?.prpEvent) !== null; const projectsProgress = isSafeNativeChatProgressEventType(input.eventType); const projectsResponse = isNativeResponseEvidence(input.eventType, input.payload?.prpEvent); - if (!input.nativeSource && !input.retryExhaustion && !projectsProgress && !projectsResponse && !projectsReassessment) { + if (!input.nativeSource && !input.retryExhaustion && !projectsProgress && !projectsResponse && !projectsReassessment && !projectsRetention) { const allocated = db.$with("allocated_run_event", {}).as(sql` update ${heartbeatRuns} set next_event_seq = ${heartbeatRuns.nextEventSeq} + 1, @@ -187,6 +189,7 @@ export async function appendHeartbeatRunEvent( if (projectsResponse && run.runtimeMode === "native") { await notifyNativeResponsePresentation(tx, { companyId: run.companyId, runId: input.runId }); } + if (projectsRetention) await notifyAttentionRetentionWork(tx, { companyId: input.companyId, sourceKind: "failed_run" }); const seq = await allocateHeartbeatRunEventSeq( tx as unknown as Db, input.runId, diff --git a/server/src/services/heartbeat/agent-status.ts b/server/src/services/heartbeat/agent-status.ts index f034cd06594..23cef4b2f87 100644 --- a/server/src/services/heartbeat/agent-status.ts +++ b/server/src/services/heartbeat/agent-status.ts @@ -1,3 +1,4 @@ +import { notifyAttentionRetentionWork } from "../retention-work-notifications.js"; import { DELIVERY_QUEUES, notifyDeliveryWork } from "../delivery-work-notifications.js"; import { agents, heartbeatRuns, type Db } from "@paperclipai/db"; import { and, asc, desc, eq, lte, sql } from "drizzle-orm"; @@ -91,6 +92,7 @@ export function createHeartbeatAgentStatus(db: Db) { )).limit(1); const nextStatus = running ? "running" : settledStatus(outcome, options?.keepIdleOnFailure); const now = new Date(); + await notifyAttentionRetentionWork(tx, { companyId: existing.companyId }); await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers); const [updated] = await tx.update(agents).set({ status: nextStatus, @@ -140,6 +142,7 @@ export function createHeartbeatAgentStatus(db: Db) { eq(heartbeatRuns.agentId, candidate.id))) .orderBy(desc(heartbeatRuns.finishedAt), desc(heartbeatRuns.createdAt), desc(heartbeatRuns.id)).limit(1); const nextStatus = settledStatus(lastRun?.status); + await notifyAttentionRetentionWork(tx, { companyId: candidate.companyId }); const [agent] = await tx.update(agents).set({ status: nextStatus, errorReason: nextStatus === "error" ? errorReason(lastRun?.error) : null, updatedAt: now, }).where(and(eq(agents.id, candidate.id), eq(agents.companyId, candidate.companyId), diff --git a/server/src/services/heartbeat/retries.ts b/server/src/services/heartbeat/retries.ts index 946f5e309f8..63fa7dc4cc6 100644 --- a/server/src/services/heartbeat/retries.ts +++ b/server/src/services/heartbeat/retries.ts @@ -1,3 +1,4 @@ +import { notifyAttentionRetentionWork } from "../retention-work-notifications.js"; import { deriveTaskKeyWithHeartbeatFallback } from "./run-state.js"; import { infrastructureRetryDelayMs } from "../system-recovery-state.js"; import { @@ -997,6 +998,7 @@ export function createHeartbeatRetries(db: Db, dependencies: HeartbeatRetryDepen retryContextSnapshot.previousRunId = continuation.previousRunId; } + await notifyAttentionRetentionWork(tx, { companyId: run.companyId }); const wakeupRequest = await tx .insert(agentWakeupRequests) .values({ diff --git a/server/src/services/issue-recovery-actions.ts b/server/src/services/issue-recovery-actions.ts index 5fb28bcb6d8..355e0bc4399 100644 --- a/server/src/services/issue-recovery-actions.ts +++ b/server/src/services/issue-recovery-actions.ts @@ -1,3 +1,4 @@ +import { notifyAttentionRetentionWork } from "./retention-work-notifications.js"; import { notifyNativeRecoveryNotices } from "./native-recovery-notice-notifications.js"; import { notifyUnsafeWorkspaceExportRecovery } from "./unsafe-workspace-export-notifications.js"; import { writeExecutionControl } from "./execution-control-notifications.js"; @@ -322,6 +323,7 @@ export function issueRecoveryActionService(db: DbOrTransaction) { ): Promise { try { const created = await db.transaction(async (tx) => { + await notifyAttentionRetentionWork(tx, { companyId: input.companyId }); await notifyNativeRecoveryNotices(tx, { companyId: input.companyId, issueId: input.sourceIssueId }); await notifyDeliveryWork(tx, DELIVERY_QUEUES.executionDisposition); await notifyDeliveryWork(tx, DELIVERY_QUEUES.executionContinuation); diff --git a/server/src/services/issue-review-state-notifications.ts b/server/src/services/issue-review-state-notifications.ts index 679b7e37475..353114a9bf0 100644 --- a/server/src/services/issue-review-state-notifications.ts +++ b/server/src/services/issue-review-state-notifications.ts @@ -1,9 +1,11 @@ +import { notifyAttentionRetentionWork } from "./retention-work-notifications.js"; import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js"; import { notifyExternalConversationSettlement } from "./external-conversation-settlement-notifications.js"; import { notifyNativeReviewResponse } from "./native-review-response-notifications.js"; import { notifyNativeStatusReassessment } from "./native-status-reassessment-notifications.js"; export async function notifyIssueReviewState(tx: object, scope: { companyId: string; issueId?: string; runId?: string; agentId?: string }) { + await notifyAttentionRetentionWork(tx, { companyId: scope.companyId }); await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors); await notifyExternalConversationSettlement(tx, scope); await notifyNativeReviewResponse(tx, scope); diff --git a/server/src/services/issue-title.ts b/server/src/services/issue-title.ts index af6716e3aa4..5a7e3ffe0eb 100644 --- a/server/src/services/issue-title.ts +++ b/server/src/services/issue-title.ts @@ -1,3 +1,4 @@ +import { notifyAttentionRetentionWork } from "./retention-work-notifications.js"; import { and, eq } from "drizzle-orm"; import { heartbeatRuns, issues, type Db } from "@paperclipai/db"; import { setIssueTitleSchema, type SetIssueTitle } from "@paperclipai/shared"; @@ -76,6 +77,7 @@ export async function setIssueTitle( || (issue.title === title && !issue.titleNeedsGeneration)) { return { result: await saveReceipt({ id: issue.id, title: issue.title, titleNeedsGeneration: issue.titleNeedsGeneration, changed: false }), publication: null }; } + await notifyAttentionRetentionWork(tx, { companyId }); await tx.update(issues).set({ title, titleNeedsGeneration: false, updatedAt: new Date() }).where(predicate); await issueReferenceService(tx).syncIssue(issueId, tx); const { publication } = await persistActivity(tx, { diff --git a/server/src/services/issue-visibility.ts b/server/src/services/issue-visibility.ts index 44a04a089e6..b54832f29ee 100644 --- a/server/src/services/issue-visibility.ts +++ b/server/src/services/issue-visibility.ts @@ -1,5 +1,5 @@ import { nonIdleSlackIssueCondition } from "./slack-conversation-state.js"; -import { and, isNull, type SQL } from "drizzle-orm"; +import { and, isNull, isNotNull, or, type SQL } from "drizzle-orm"; import { issues } from "@paperclipai/db"; export function visibleIssueCondition(): SQL { @@ -14,3 +14,9 @@ export function visibleIssueSql(alias = "issues") { export function executionIssueCondition(): SQL { return and(visibleIssueCondition(), isNull(issues.conversationAgentId), nonIdleSlackIssueCondition())!; } + +export function attentionIssueCondition(includeConversations = false): SQL { + return includeConversations + ? or(executionIssueCondition(), and(visibleIssueCondition(), isNotNull(issues.conversationAgentId)))! + : executionIssueCondition(); +} diff --git a/server/src/services/issues.ts b/server/src/services/issues.ts index 42c65bb5d02..4ee7e681907 100644 --- a/server/src/services/issues.ts +++ b/server/src/services/issues.ts @@ -1,3 +1,4 @@ +import { notifyAttentionRetentionWork } from "./retention-work-notifications.js"; import { notifyExternalConversationSettlement } from "./external-conversation-settlement-notifications.js"; import { notifyNativeRecoveryNotices } from "./native-recovery-notice-notifications.js"; import { notifyNativeRunProjection } from "./native-run-projection-notifications.js"; @@ -7626,6 +7627,7 @@ export function issueService(db: Db) { await assertNoBlockingCycles(companyId, issueId, deduped, dbOrTx); } + await notifyAttentionRetentionWork(dbOrTx, { companyId }); await dbOrTx .delete(issueRelations) .where( @@ -10606,6 +10608,7 @@ export function issueService(db: Db) { } if (values.monitorNextCheckAt) await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors); await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers); + await notifyAttentionRetentionWork(tx, { companyId }); const [issue] = await tx.insert(issues).values(values).returning(); await attachOwnedDraftImages(tx, issue, issue.description, { userId: issueData.createdByUserId, agentId: issueData.createdByAgentId }); if (issue.visibility === "private" && !inheritedPrivateSubtree && issueData.createdByAgentId) { @@ -11491,6 +11494,7 @@ export function issueService(db: Db) { || issueData.parentId !== undefined || issueData.executionWorkspaceId !== undefined) { await notifyTerminalWorkspaceWork(tx, `issue:${id}`); } + await notifyAttentionRetentionWork(tx, { companyId: receiptExisting.companyId }); if (patch.status !== undefined || patch.assigneeAgentId !== undefined || patch.assigneeUserId !== undefined || patch.hiddenAt !== undefined || patch.conversationAgentId !== undefined) { await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers); @@ -11952,6 +11956,7 @@ export function issueService(db: Db) { await requestRestorePointDeletion(tx, { sourceIssueId: id }); const [owner] = await tx.select({ companyId: issues.companyId }).from(issues).where(eq(issues.id, id)); if (owner) { + await notifyAttentionRetentionWork(tx, { companyId: owner.companyId }); await notifyExternalConversationSettlement(tx, { companyId: owner.companyId }); await notifyApprovalWork(tx, owner.companyId); } @@ -12084,6 +12089,7 @@ export function issueService(db: Db) { ) : isNull(issues.executionRunId); const updated = await db.transaction(async tx => { + await notifyAttentionRetentionWork(tx, { companyId: issueCompany.companyId }); await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors); await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers); return tx @@ -12213,6 +12219,7 @@ export function issueService(db: Db) { } const staleExecutionRunId = current.executionRunId; const adopted = await db.transaction(async tx => { + await notifyAttentionRetentionWork(tx, { companyId: issueCompany.companyId }); await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors); await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers); return tx @@ -12468,6 +12475,7 @@ export function issueService(db: Db) { existing.status === "in_progress" ? "todo" : existing.status; await notifyTerminalWorkspaceWork(tx, `issue:${id}`); await notifyNativeStatusReassessment(tx, { companyId: existing.companyId, issueId: id }); + await notifyAttentionRetentionWork(tx, { companyId: existing.companyId }); await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors); await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers); const updated = await tx @@ -12520,6 +12528,7 @@ export function issueService(db: Db) { patch.assigneeAgentId = null; } + await notifyAttentionRetentionWork(tx, { companyId: existing.companyId }); await notifyTerminalWorkspaceWork(tx, `issue:${id}`); if (existing.checkoutRunId || existing.executionRunId || options.clearAssignee) { await notifyExternalConversationSettlement(tx, { companyId: existing.companyId, issueId: id }); diff --git a/server/src/services/native-finalization-notifications.ts b/server/src/services/native-finalization-notifications.ts index 72d4cbcac49..1feb1ba9703 100644 --- a/server/src/services/native-finalization-notifications.ts +++ b/server/src/services/native-finalization-notifications.ts @@ -1,3 +1,4 @@ +import { notifyAttentionRetentionWork } from "./retention-work-notifications.js"; import { notifyNativeEnvironmentSettlement } from "./native-environment-settlement-notifications.js"; import { notifyExternalAgentSettlement } from "../modules/external-agents/index.js"; import { notifyExternalConversationSettlement } from "./external-conversation-settlement-notifications.js"; @@ -14,6 +15,7 @@ import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notificatio export type NativeFinalizationWorkScope = { companyId: string; runId?: string; issueId?: string }; export async function notifyNativeFinalizationWork(transaction: object, scope: NativeFinalizationWorkScope) { + await notifyAttentionRetentionWork(transaction, { companyId: scope.companyId }); if (scope.issueId || scope.runId) { await notifyNativeEnvironmentSettlement(transaction, scope); await notifyExternalAgentSettlement(transaction, scope); diff --git a/server/src/services/native-runtime/native-provider-capacity-retry.ts b/server/src/services/native-runtime/native-provider-capacity-retry.ts index 5b54bd0cdfc..dc9cb0dac66 100644 --- a/server/src/services/native-runtime/native-provider-capacity-retry.ts +++ b/server/src/services/native-runtime/native-provider-capacity-retry.ts @@ -1,3 +1,4 @@ +import { notifyAttentionRetentionWork } from "../retention-work-notifications.js"; import { randomUUID } from "node:crypto"; import { and, eq } from "drizzle-orm"; import { agentWakeupRequests, heartbeatRuns, type Db } from "@paperclipai/db"; @@ -49,6 +50,7 @@ export async function materializeNativeProviderCapacityRetry(input: { scheduledRetryAt: dueAt.toISOString(), executionRetryAccounting: accountingForScheduledRetry(run, NATIVE_PROVIDER_OVERLOADED_CODE, attempt), }; + await notifyAttentionRetentionWork(input.tx, { companyId: input.companyId }); const [wake] = await input.tx.insert(agentWakeupRequests).values({ companyId: run.companyId, agentId: run.agentId, source: "automation", triggerDetail: "system", reason: NATIVE_PROVIDER_OVERLOADED_CODE, status: "queued", runId: retryId, diff --git a/server/src/services/native-runtime/native-review-participant.ts b/server/src/services/native-runtime/native-review-participant.ts index 573574dd19d..f57f3a7c889 100644 --- a/server/src/services/native-runtime/native-review-participant.ts +++ b/server/src/services/native-runtime/native-review-participant.ts @@ -1,3 +1,4 @@ +import { notifyAttentionRetentionWork } from "../retention-work-notifications.js"; import { agents, heartbeatRuns, @@ -240,6 +241,7 @@ export async function claimNativeReviewExecutionLock(db: Db, input: { if (!run || !["queued", "running"].includes(run.status) || !admitted || !expected || admitted.nativeReviewInteractionId !== expected.nativeReviewInteractionId || admitted.nativeReviewDecisionId !== expected.nativeReviewDecisionId) return false; + await notifyAttentionRetentionWork(tx, { companyId: input.companyId }); const claimed = await tx.update(issues).set({ executionRunId: input.runId, executionAgentNameKey: input.agentNameKey, diff --git a/server/src/services/native-runtime/native-workspace-reconstruction.ts b/server/src/services/native-runtime/native-workspace-reconstruction.ts index 5273b30216a..8ccad19395d 100644 --- a/server/src/services/native-runtime/native-workspace-reconstruction.ts +++ b/server/src/services/native-runtime/native-workspace-reconstruction.ts @@ -1,3 +1,4 @@ +import { notifyAttentionRetentionWork } from "../retention-work-notifications.js"; import { DELIVERY_QUEUES, notifyDeliveryWork } from "../delivery-work-notifications.js"; import { notifyUnsafeWorkspaceExportRecovery } from "../unsafe-workspace-export-notifications.js"; import { and, eq, gt, inArray, ne, notInArray, or, sql } from "drizzle-orm"; @@ -179,6 +180,7 @@ export async function reconstructNativeWorkspace(input: { ...(source.interveningExecutions.length ? { interveningExecutions: source.interveningExecutions } : {}) } }; await markExecutionReconciliation(db, source.action, decision, input.actorId); const now = new Date(); + await notifyAttentionRetentionWork(tx, { companyId: input.companyId }); await notifyUnsafeWorkspaceExportRecovery(tx, { companyId: input.companyId, issueId: input.issueId }); await tx.update(issueRecoveryActions).set({ status: "resolved", outcome: "handed_back", resolvedAt: now, updatedAt: now, resolutionNote: "workspace_reconstruction_authorized", nextAction: "Continue from saved sources. The original unexported edits are lost; verify the selected attachments before rebuilding. Do not repeat completed external actions or reset the original work limits.", diff --git a/server/src/services/native-runtime/paperclip-runner-tool-authority.ts b/server/src/services/native-runtime/paperclip-runner-tool-authority.ts index 02917045861..f2dbe3088e4 100644 --- a/server/src/services/native-runtime/paperclip-runner-tool-authority.ts +++ b/server/src/services/native-runtime/paperclip-runner-tool-authority.ts @@ -1,4 +1,5 @@ import { nativeRunKeyScope, nativeSkillTestToolAllowed, assertNativeSkillTestTool } from "./native-skill-test-scope.js"; +import { notifyAttentionRetentionWork } from "../retention-work-notifications.js"; import { requiredQualityReviewSchema } from "@paperclipai/shared"; import { priorRecoveryApiReceipt } from "../recovery-effect-receipts.js"; import { DELIVERY_QUEUES, notifyDeliveryWork } from "../delivery-work-notifications.js"; @@ -2067,6 +2068,7 @@ export class PaperclipRunnerToolAuthority { replayMonitor = summarizeIssueMonitor(target, normalizeIssueExecutionPolicy(target.executionPolicy)); return { taskId: target.id, status: target.status, monitor: replayMonitor }; } + await notifyAttentionRetentionWork(tx, { companyId: this.binding.companyId }); const patch = prepareIssueMonitorUpdate(target, input.monitor, actor); await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors); const [updated] = await tx.update(issues).set({ ...patch, updatedAt: new Date() }) diff --git a/server/src/services/native-runtime/superseded-external-review-requests.ts b/server/src/services/native-runtime/superseded-external-review-requests.ts index f04db57f7d7..9697b436ed3 100644 --- a/server/src/services/native-runtime/superseded-external-review-requests.ts +++ b/server/src/services/native-runtime/superseded-external-review-requests.ts @@ -1,3 +1,4 @@ +import { notifyAttentionRetentionWork } from "../retention-work-notifications.js"; import { and, eq, isNull, ne, sql } from "drizzle-orm"; import { completionContracts, heartbeatRuns, issues, issueThreadInteractions, statusDecisionEffects, statusDecisions, workAssessments, type Db } from "@paperclipai/db"; import { hasPublishedGitHubReview } from "../chat-channels/github/review-wait.js"; @@ -70,6 +71,7 @@ export async function supersedeExternalReviewRequestsInTransaction( publications: ActivityPublication[], ) { for (const { interaction } of candidates) { + await notifyAttentionRetentionWork(tx, { companyId: binding.companyId }); const reason = "A later published review no longer requests human input."; const [retired] = await tx.update(issueThreadInteractions).set({ status: "cancelled", result: { version: 1, outcome: "withdrawn", reason }, resolvedAt: new Date(), updatedAt: new Date(), diff --git a/server/src/services/native-status-reassessment-notifications.ts b/server/src/services/native-status-reassessment-notifications.ts index fb845523bb4..3a5ec8b6c37 100644 --- a/server/src/services/native-status-reassessment-notifications.ts +++ b/server/src/services/native-status-reassessment-notifications.ts @@ -1,8 +1,10 @@ +import { notifyAttentionRetentionWork } from "./retention-work-notifications.js"; import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js"; export type NativeStatusReassessmentScope = { companyId: string; issueId?: string; runId?: string; agentId?: string }; -export function notifyNativeStatusReassessment(tx: object, scope: NativeStatusReassessmentScope) { +export async function notifyNativeStatusReassessment(tx: object, scope: NativeStatusReassessmentScope) { + await notifyAttentionRetentionWork(tx, { companyId: scope.companyId }); return notifyDeliveryWork(tx, DELIVERY_QUEUES.nativeStatusReassessment, JSON.stringify({ companyId: scope.companyId, issueId: scope.issueId, runId: scope.runId, agentId: scope.agentId })); } diff --git a/server/src/services/plugin-managed-agents.ts b/server/src/services/plugin-managed-agents.ts index e2e2c965e8c..91a81fb5120 100644 --- a/server/src/services/plugin-managed-agents.ts +++ b/server/src/services/plugin-managed-agents.ts @@ -1,3 +1,4 @@ +import { notifyAttentionRetentionWork } from "./retention-work-notifications.js"; import { createAgentLifecycle } from "./agent-lifecycle.js"; import { and, eq, isNull, ne } from "drizzle-orm"; import type { Db } from "@paperclipai/db"; @@ -340,6 +341,7 @@ export function pluginManagedAgentService( // and its canonical history. Transfer only its proven stable-key marker to // the replacement installation, atomically with the new scoped bindings. await upsertBinding(companyId, declaration, agentId, {}, adapterType, tx); + await notifyAttentionRetentionWork(tx, { companyId, sourceKind: "agent_error_alert" }); await tx.update(agents).set({ metadata: managedMetadata(options.pluginId, options.pluginKey, declaration, agent.metadata), updatedAt: new Date(), diff --git a/server/src/services/retention-work-notifications.ts b/server/src/services/retention-work-notifications.ts index ffedb0b1a4d..0331f53f258 100644 --- a/server/src/services/retention-work-notifications.ts +++ b/server/src/services/retention-work-notifications.ts @@ -3,12 +3,19 @@ import type { Db } from "@paperclipai/db"; import type { AttentionSourceKind } from "@paperclipai/shared"; import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js"; -export async function notifyRetentionWork(transaction: object, kind: AttentionSourceKind) { +export async function notifyAttentionRetentionWork(transaction: object, scope: { companyId?: string; sourceKind?: AttentionSourceKind; sourceId?: string } = {}) { + await notifyDeliveryWork(transaction, DELIVERY_QUEUES.attentionRetention, JSON.stringify(scope)); +} + +export async function notifyRetentionWork(transaction: object, kind: AttentionSourceKind, scope: { companyId?: string; sourceId?: string } = {}) { if (kind === "approval") await notifyDeliveryWork(transaction, DELIVERY_QUEUES.approvalRetention); if (kind === "join_request") await notifyDeliveryWork(transaction, DELIVERY_QUEUES.joinRequestRetention); + if (kind !== "approval" && kind !== "join_request") await notifyAttentionRetentionWork(transaction, { ...scope, sourceKind: kind, + sourceId: kind === "blocker_attention" ? undefined : scope.sourceId }); } export async function notifyCompanyRetentionWork(transaction: object, companyId: string) { + await notifyAttentionRetentionWork(transaction, { companyId }); await notifyApprovalWork(transaction, companyId); await notifyRetentionWork(transaction, "join_request"); } diff --git a/server/src/services/slack-conversation-lifecycle.ts b/server/src/services/slack-conversation-lifecycle.ts index 0fd213dc084..fddf863b69e 100644 --- a/server/src/services/slack-conversation-lifecycle.ts +++ b/server/src/services/slack-conversation-lifecycle.ts @@ -1,3 +1,4 @@ +import { notifyAttentionRetentionWork } from "./retention-work-notifications.js"; import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js"; import { notifyUnsafeWorkspaceExportRecovery } from "./unsafe-workspace-export-notifications.js"; import { and, asc, desc, eq, gt, inArray, isNull, sql } from "drizzle-orm"; @@ -126,6 +127,7 @@ export async function settleExternalConversation(db: Db, companyId: string, issu const ready = await readSettlement(tx, companyId, issueId, true); if (!ready) return false; const { binding, issue, run } = ready; + await notifyAttentionRetentionWork(tx, { companyId }); const changed = await tx.update(chatConversations).set({ state: "waiting", updatedAt: new Date() }) .where(and(eq(chatConversations.id, binding.conversation.id), eq(chatConversations.companyId, companyId), eq(chatConversations.state, "active"))) .returning({ id: chatConversations.id }); diff --git a/server/src/services/slack-conversation-state.ts b/server/src/services/slack-conversation-state.ts index dedaa8d6676..5fbf7a2d43e 100644 --- a/server/src/services/slack-conversation-state.ts +++ b/server/src/services/slack-conversation-state.ts @@ -1,3 +1,4 @@ +import { notifyAttentionRetentionWork } from "./retention-work-notifications.js"; import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js"; import { notifyExternalConversationSettlement } from "./external-conversation-settlement-notifications.js"; import { chatConversations, issues, type Db } from "@paperclipai/db"; @@ -45,6 +46,7 @@ export function nonIdleSlackIssueCondition() { /** Called inside message admission while its transaction owns the issue lock. */ export async function resumeExternalConversation(tx: Db, companyId: string, issueId: string) { + await notifyAttentionRetentionWork(tx, { companyId }); await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers); await notifyExternalConversationSettlement(tx, { companyId, issueId }); // Only the verified idle convention may leave In Review automatically. From 7edb163a80d1aca60e6c780c9472befa523fcd5a Mon Sep 17 00:00:00 2001 From: Dotta Date: Fri, 30 Oct 2026 16:49:17 -0500 Subject: [PATCH 11/70] fix(server): cancel merge observation HTTP requests on shutdown Bound GitHub object and repository probes, keep readiness independent of external reads, and page pending cards through the owning parser. Co-Authored-By: Paperclip --- ...026-10-30-merged-pr-confirmations-sleep.md | 34 ++++++- .../external-objects-service.test.ts | 9 +- .../merged-pr-confirmation-work.test.ts | 88 +++++++++++++++++++ .../server-startup-feedback-export.test.ts | 17 +++- server/src/index.ts | 4 +- server/src/services/external-objects.ts | 1 + .../github-external-object-provider.ts | 18 ++-- .../src/services/github-pull-request-merge.ts | 17 ++-- .../src/services/issue-thread-interactions.ts | 38 +++++--- .../services/merged-pr-confirmation-work.ts | 14 ++- 10 files changed, 200 insertions(+), 40 deletions(-) diff --git a/doc/plans/2026-10-30-merged-pr-confirmations-sleep.md b/doc/plans/2026-10-30-merged-pr-confirmations-sleep.md index fb8b091f841..7b8699456fb 100644 --- a/doc/plans/2026-10-30-merged-pr-confirmations-sleep.md +++ b/doc/plans/2026-10-30-merged-pr-confirmations-sleep.md @@ -13,10 +13,12 @@ it receives no GitHub payloads. No database migration is required. ## Durable truth and producers Pending `request_confirmation` interaction rows remain the durable work source. -The candidate query excludes terminal and hidden issues, linked tool actions, -and text without merge intent or a PR reference. The existing fail-closed parser -then excludes additional actions, governed actions, and document/attachment targets. -No accepted or cancelled history keeps a deadline alive. +The candidate query pages pending cards in bounded groups of 100 and excludes +terminal and hidden issues and linked tool actions. The existing fail-closed +parser is the sole text eligibility source: it requires merge intent and a PR +reference and excludes additional actions, governed actions, and document/attachment targets. +No accepted or cancelled history keeps a deadline alive. A full page without any +merge candidates still advances its keyset cursor; later candidates are not lost. Ordinary creation registers the topic before inserting a confirmation in its transaction. Existing confirmation-resolution notifications invalidate its issue @@ -53,6 +55,14 @@ This is candidate-backed external observation. It does not eliminate all polling register a Cloud wake alarm, or prove that the host/database suspends. Existing sleep-safety blockers remain in force. +Startup registers this observation without waiting for live GitHub requests. +The internal resolver invocation accepts an optional host-owned abort signal; +that signal is not part of external-object records or plugin/RPC payloads. +The GitHub owner combines cancellation with a ten-second request budget and +passes it to the actual object fetch and repository-visibility probes, including +response body reads. Shutdown aborts the request and waits for its cancellation; +no post-abort acceptance is committed. A timeout leaves the durable card pending. + ## Acceptance evidence Focused tests use real PostgreSQL, the actual service producers, transaction @@ -68,6 +78,22 @@ provider failure across drain; standby/resume; restart; tenant-scoped and duplic merge hints; latest/superseded cards during provider observation; hidden/terminal issues; URL targets; no-continuation policy; and shutdown during an in-flight read. Existing startup and normalized GitHub event tests exercise the changed consumers. +Controlled local HTTP tests hold headers, partial response bodies, 404 probes, +and public-visibility probes. They observe server-side response close after +shutdown within one second, rather than only racing a promise. A shortened +request budget times out a real request; the saved card then accepts after +restart when that same HTTP fixture serves a merged response. Existing GitHub +provider and external-object service tests preserve their authorization behavior. + +## Verification follow-up + +The original head passed full-workspace typecheck and build. A later required +Linux CI shard failed the unchanged managed-skill maintenance CLI fixture in +`managed-skill-content.test.ts`: it returned `unavailable_or_corrupt_package` +where `local_storage` was expected. Both the isolated case and the complete +unchanged 17-test file pass against fresh local PostgreSQL. The Linux cause is +not established; this is retained as an unresolved CI failure, not called a flake. +The current-head required CI result remains the completion gate. ## Existing limitations outside this migration diff --git a/server/src/__tests__/external-objects-service.test.ts b/server/src/__tests__/external-objects-service.test.ts index e9022599779..af3b4667deb 100644 --- a/server/src/__tests__/external-objects-service.test.ts +++ b/server/src/__tests__/external-objects-service.test.ts @@ -1,3 +1,4 @@ +import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js"; import { randomUUID } from "node:crypto"; import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest"; import { eq } from "drizzle-orm"; @@ -26,7 +27,8 @@ import type { PaperclipPluginManifestV1 } from "@paperclipai/shared"; import type { PluginWorkerManager } from "../services/plugin-worker-manager.js"; import { createGitHubExternalObjectProvider } from "../services/github-external-object-provider.js"; -const embeddedPostgresSupport = await getEmbeddedPostgresTestSupport(); +const external = resolveExternalTestDatabaseUrl("PAPERCLIP_EXTERNAL_OBJECTS_TEST_DATABASE_URL"); +const embeddedPostgresSupport = external ? { supported: true } : await getEmbeddedPostgresTestSupport(); const describeEmbeddedPostgres = embeddedPostgresSupport.supported ? describe : describe.skip; if (!embeddedPostgresSupport.supported) { @@ -709,8 +711,8 @@ describeEmbeddedPostgres("externalObjectService", () => { let tempDb: Awaited> | null = null; beforeAll(async () => { - tempDb = await startEmbeddedPostgresTestDatabase("paperclip-external-objects-"); - db = createDb(tempDb.connectionString); + if (!external) tempDb = await startEmbeddedPostgresTestDatabase("paperclip-external-objects-"); + db = createDb(external ?? tempDb!.connectionString, { maxConnections: 6 }); }, 20_000); afterEach(async () => { @@ -725,6 +727,7 @@ describeEmbeddedPostgres("externalObjectService", () => { }); afterAll(async () => { + await db?.$client.end({ timeout: 1 }); await tempDb?.cleanup(); }); diff --git a/server/src/__tests__/merged-pr-confirmation-work.test.ts b/server/src/__tests__/merged-pr-confirmation-work.test.ts index 51195e6157b..95565fd96e7 100644 --- a/server/src/__tests__/merged-pr-confirmation-work.test.ts +++ b/server/src/__tests__/merged-pr-confirmation-work.test.ts @@ -1,3 +1,5 @@ +import { createServer } from "node:http"; +import { createPullRequestMergeStateResolver } from "../services/github-pull-request-merge.js"; import { randomUUID } from "node:crypto"; import { setTimeout as delay } from "node:timers/promises"; import { eq, sql } from "drizzle-orm"; @@ -262,6 +264,92 @@ const support = external ? { supported: true } : await getEmbeddedPostgresTestSu expect(wakeup).toHaveBeenCalledTimes(1); expect(errors).toEqual([]); }); + it.each(["headers", "body", "not-found probe", "public visibility probe"] as const)("cancels an actual unresponsive HTTP %s read promptly at stop", async phase => { + vi.useRealTimers(); + const card = await create(); + let observed!: () => void, closed!: () => void; + const requested = new Promise(resolve => { observed = resolve; }); + const disconnected = new Promise(resolve => { closed = resolve; }); + const signals: AbortSignal[] = []; + const server = createServer((request, response) => { + if (phase === "not-found probe" && request.url?.includes("/pulls/")) { response.writeHead(404).end(); return; } + response.on("close", closed); + if (phase === "body") { response.writeHead(200, { "content-type": "application/json" }); response.write('{"merged":'); } + observed(); + }); + await new Promise(resolve => { server.listen(0, "127.0.0.1", resolve); }); + const address = server.address(); + if (!address || typeof address === "string") throw new Error("Expected test HTTP address"); + const resolveState = createPullRequestMergeStateResolver(db, { + requestTimeoutMs: 10_000, + credentialProvider: () => phase === "public visibility probe" ? { token: "fixture", source: "server_env", secretName: null } : null, + fetch: (url, init) => { signals.push(init!.signal!); return fetch(`http://127.0.0.1:${address.port}${new URL(url).pathname}`, init); }, + }); + const real = issueThreadInteractionService(db, { resolvePullRequestState: resolveState, wakeup }); + coordinator = createDeliveryWorkCoordinator({ owner: db, canRun: () => true, onError: error => errors.push(error) }); + try { + const startupAt = performance.now(); + const worker = registerMergeConfirmationWork(coordinator, real, () => true); + expect(performance.now() - startupAt).toBeLessThan(100); + await Promise.race([requested, delay(2000).then(() => { throw new Error("HTTP observation did not start"); })]); + const stoppingAt = performance.now(); + await coordinator.stop(); coordinator = undefined; + expect(performance.now() - stoppingAt).toBeLessThan(1000); + await worker.ready; + await Promise.race([disconnected, delay(1000).then(() => { throw new Error("HTTP request remained open after stop"); })]); + expect(signals.at(-1)?.aborted).toBe(true); + expect((await stored(card.id)).status).toBe("pending"); expect(wakeup).not.toHaveBeenCalled(); expect(errors).toEqual([]); + } finally { + await coordinator?.stop(); coordinator = undefined; + server.closeAllConnections(); await new Promise(resolve => { server.close(() => resolve()); }); + } + }); + + it("times out a real unresponsive HTTP read and recovers its durable candidate on restart", async () => { + vi.useRealTimers(); const card = await create(); + let serveMerge = false, closed!: () => void; + const disconnected = new Promise(resolve => { closed = resolve; }); + const server = createServer((_request, response) => { + if (serveMerge) { response.writeHead(200, { "content-type": "application/json" }).end(JSON.stringify({ merged: true, state: "closed" })); return; } + response.on("close", closed); + }); + await new Promise(resolve => { server.listen(0, "127.0.0.1", resolve); }); + const address = server.address(); if (!address || typeof address === "string") throw new Error("Expected test HTTP address"); + const real = issueThreadInteractionService(db, { resolvePullRequestState: createPullRequestMergeStateResolver(db, { + requestTimeoutMs: 100, tokenProvider: () => null, + fetch: (url, init) => fetch(`http://127.0.0.1:${address.port}${new URL(url).pathname}`, init), + }), wakeup }); + coordinator = createDeliveryWorkCoordinator({ owner: db, canRun: () => true, onError: error => errors.push(error) }); + try { + const startedAt = performance.now(); + await registerMergeConfirmationWork(coordinator, real, () => true).ready; + expect(performance.now() - startedAt).toBeLessThan(2000); + await Promise.race([disconnected, delay(1000).then(() => { throw new Error("Timed out HTTP request remained open"); })]); + expect(errors).toHaveLength(1); expect((await stored(card.id)).status).toBe("pending"); expect(wakeup).not.toHaveBeenCalled(); + expect(coordinator.nextWakeAt()).not.toBeNull(); + await coordinator.stop(); coordinator = undefined; serveMerge = true; + coordinator = createDeliveryWorkCoordinator({ owner: db, canRun: () => true, onError: error => errors.push(error) }); + await registerMergeConfirmationWork(coordinator, real, () => true).ready; + expect((await stored(card.id)).status).toBe("accepted"); expect(wakeup).toHaveBeenCalledTimes(1); + } finally { + await coordinator?.stop(); coordinator = undefined; + server.closeAllConnections(); await new Promise(resolve => { server.close(() => resolve()); }); + } + }); + + it("pages beyond a full pending-card page without duplicating the merge parser in SQL", async () => { + const card = await create(); + const id = BigInt(`0x${card.id.replaceAll("-", "")}`); + await db.insert(issueThreadInteractions).values(Array.from({ length: 100 }, (_, index) => { + const hex = (id - BigInt(index + 1)).toString(16).padStart(32, "0"); + return { id: `${hex.slice(0,8)}-${hex.slice(8,12)}-${hex.slice(12,16)}-${hex.slice(16,20)}-${hex.slice(20)}`, + companyId, issueId, kind: "request_confirmation", title: "Approve the plan?", payload: { version: 1 as const, prompt: "Approve the plan?" } }; + })); + await start(); expect(provider).not.toHaveBeenCalled(); + await waitFor(async () => expect((await stored(card.id)).status).toBe("accepted")); await quiet(); + expect(provider).toHaveBeenCalledTimes(1); expect(wakeup).toHaveBeenCalledTimes(1); expect(errors).toEqual([]); + }); + it("limits normalized merge hints to the owning company and matching PR and tolerates duplicates", async () => { provider.mockResolvedValue("open"); const own = await create(); const foreignCompany = randomUUID(), foreignIssue = randomUUID(); diff --git a/server/src/__tests__/server-startup-feedback-export.test.ts b/server/src/__tests__/server-startup-feedback-export.test.ts index 9fc7234b2e6..4e7e3562156 100644 --- a/server/src/__tests__/server-startup-feedback-export.test.ts +++ b/server/src/__tests__/server-startup-feedback-export.test.ts @@ -180,6 +180,7 @@ const { const heartbeatServiceFactoryMock = vi.fn(() => heartbeatServiceMock); const issueThreadInteractionServiceMock = { sweepSupersededPendingRequestConfirmations: vi.fn(async () => ({ expired: 0 })), + mergedPullRequestConfirmationPage: vi.fn(async () => ({ rows: [], complete: true })), mergedPullRequestConfirmationCandidates: vi.fn(async () => []), sweepMergedPullRequestConfirmations: vi.fn(async () => ({ checked: 0, @@ -1335,7 +1336,7 @@ describe("startServer feedback export wiring", () => { expect(heartbeatServiceMock.tickAgentTimers).toHaveBeenCalledTimes(1); expect(externalObjectsServiceMock.refreshDueObjectsForActiveCompanies).toHaveBeenCalledTimes(1); - expect(issueThreadInteractionServiceMock.mergedPullRequestConfirmationCandidates).toHaveBeenCalledTimes(1); + expect(issueThreadInteractionServiceMock.mergedPullRequestConfirmationPage).toHaveBeenCalledTimes(1); expect(issueThreadInteractionServiceMock.sweepMergedPullRequestConfirmations).not.toHaveBeenCalled(); expect(deliveryWorkRegisterMock.mock.calls.some(([queue]) => queue === DELIVERY_QUEUES.mergedPullRequestConfirmations)).toBe(true); expect(executionWorkspaceServiceMock.sweepTerminalWorkspaces).toHaveBeenCalledTimes(1); @@ -1424,6 +1425,20 @@ describe("startServer feedback export wiring", () => { } }); + it("becomes ready without waiting for the merge-confirmation observation", async () => { + loadConfigMock.mockReturnValue(buildTestConfig({ heartbeatSchedulerEnabled: true })); + let release!: () => void; + const blocked = new Promise<{ rows: never[]; complete: boolean }>(resolve => { release = () => resolve({ rows: [], complete: true }); }); + issueThreadInteractionServiceMock.mergedPullRequestConfirmationPage.mockReturnValueOnce(blocked); + const timer = vi.spyOn(globalThis, "setInterval").mockImplementation(() => 1 as unknown as ReturnType); + let started = false; + const starting = startServer().then(server => { started = true; return server; }); + try { + await vi.waitFor(() => expect(issueThreadInteractionServiceMock.mergedPullRequestConfirmationPage).toHaveBeenCalled()); + await settleStartupHousekeeping(); expect(started).toBe(true); expect(getStartupRecoveryState().phase).toBe("ready"); + } finally { release(); await starting; timer.mockRestore(); } + }); + it.each(["retained cleanup", "workspace finalization", "environment settlement", "native run projection", "native result commit", "native recovery notices", "external conversation settlement"] as const)("becomes ready while %s is held but joins it at shutdown", async owner => { loadConfigMock.mockReturnValue(buildTestConfig({ heartbeatSchedulerEnabled: true })); let releaseCleanup!: () => void; diff --git a/server/src/index.ts b/server/src/index.ts index c1804998ef5..86074131dc3 100644 --- a/server/src/index.ts +++ b/server/src/index.ts @@ -1601,8 +1601,8 @@ async function startServerWithDatabaseTeardown( : config.workspaceCompactionDelayHours * 60 * 60 * 1000, terminalWorkspaceInspectionBackoffMs: TERMINAL_WORKSPACE_RETRY_MS, }); - await registerMergeConfirmationWork(deliveryWork, mergedPullRequestConfirmations, - () => !isIdleTaskDrainActive() && !heartbeatSchedulerStopped && !isWarmStandby()).ready; + registerMergeConfirmationWork(deliveryWork, mergedPullRequestConfirmations, + () => !isIdleTaskDrainActive() && !heartbeatSchedulerStopped && !isWarmStandby()); const gitRestorePointWorker = registerGitRestorePointWork(deliveryWork, gitRestorePointRetentionService(db as any, { storage: storageService, retentionDays: config.gitRestorePointRetentionDays, diff --git a/server/src/services/external-objects.ts b/server/src/services/external-objects.ts index 56a8fb91834..306f043d753 100644 --- a/server/src/services/external-objects.ts +++ b/server/src/services/external-objects.ts @@ -84,6 +84,7 @@ export interface ExternalObjectResolver { resolve(input: { companyId: string; object: ExternalObjectRecord; + signal?: AbortSignal; }): Promise; } diff --git a/server/src/services/github-external-object-provider.ts b/server/src/services/github-external-object-provider.ts index 6dedac97efb..83b47354035 100644 --- a/server/src/services/github-external-object-provider.ts +++ b/server/src/services/github-external-object-provider.ts @@ -15,6 +15,7 @@ type FetchLike = (url: string, init?: RequestInit) => Promise; export interface GitHubExternalObjectProviderOptions { fetch?: FetchLike; + requestTimeoutMs?: number; tokenProvider?: (companyId: string) => Promise | string | null; /** * Resolve the credential and its source. Prefer this over `tokenProvider` when the @@ -243,12 +244,12 @@ type RepoProbeOutcome = { async function probeRepoVisibility( identity: GitHubObjectIdentity, - opts: { fetch: FetchLike; headers?: Record | undefined }, + opts: { fetch: FetchLike; headers?: Record | undefined; signal?: AbortSignal }, ): Promise { try { const probe = await opts.fetch( `${gitHubApiBase(identity.host)}/repos/${encodeURIComponent(identity.owner)}/${encodeURIComponent(identity.repo)}`, - { headers: opts.headers }, + { headers: opts.headers, signal: opts.signal }, ); if (probe.ok) { const body = await safeJson(probe); @@ -427,12 +428,13 @@ export function createGitHubExternalObjectProvider( companyId: string, identity: GitHubObjectIdentity, headers: Record, + signal: AbortSignal, ): Promise { const credentialFingerprint = createHash("sha256").update(headers.authorization ?? "anonymous").digest("hex"); const key = `${companyId}\u0000${credentialFingerprint}\u0000${identity.owner.toLowerCase()}\u0000${identity.repo.toLowerCase()}`; const cached = probeCache.get(key); if (cached && cached.expiresAt > Date.now()) return cached.outcome; - const outcome = await probeRepoVisibility(identity, { fetch: fetchImpl, headers }); + const outcome = await probeRepoVisibility(identity, { fetch: fetchImpl, headers, signal }); if (outcome.visibility === "visible" || outcome.visibility === "hidden") { probeCache.set(key, { outcome, expiresAt: Date.now() + GITHUB_OBJECT_TTL_SECONDS * 1000 }); } @@ -464,7 +466,9 @@ export function createGitHubExternalObjectProvider( return { providerKey: "github", objectType, - async resolve({ companyId, object }) { + async resolve({ companyId, object, signal: ownerSignal }) { + const deadline = AbortSignal.timeout(opts.requestTimeoutMs ?? 10_000); + const signal = ownerSignal ? AbortSignal.any([ownerSignal, deadline]) : deadline; const identity = parseGitHubObject(object); if (!identity || identity.objectType !== objectType) { return { @@ -509,7 +513,7 @@ export function createGitHubExternalObjectProvider( // "public" verdict must not authorize a later read of its private contents. // The cache is still used below to classify a 404, where a stale verdict can // only make the object look deleted — it can never expose private data. - const probe = await probeRepoVisibility(identity, { fetch: fetchImpl, headers }); + const probe = await probeRepoVisibility(identity, { fetch: fetchImpl, headers, signal }); if (probe.visibility === "hidden") { return { ok: false, @@ -554,7 +558,7 @@ export function createGitHubExternalObjectProvider( let response: Response; try { - response = await fetchImpl(url, { headers }); + response = await fetchImpl(url, { headers, signal }); } catch { return { ok: false, @@ -573,7 +577,7 @@ export function createGitHubExternalObjectProvider( // Probe the repository metadata to classify: if the repo itself is visible // the object truly is gone; if it is not, surface credentials-needed // instead of permanently freezing a reference as "Not found". - const probe = await cachedProbeRepoVisibility(companyId, identity, headers); + const probe = await cachedProbeRepoVisibility(companyId, identity, headers, signal); if (probe.visibility === "visible") { return { ok: true, snapshot: notFoundSnapshot(identity, etag) }; } diff --git a/server/src/services/github-pull-request-merge.ts b/server/src/services/github-pull-request-merge.ts index bb92d37d7c2..1cdff01dc1f 100644 --- a/server/src/services/github-pull-request-merge.ts +++ b/server/src/services/github-pull-request-merge.ts @@ -1,5 +1,5 @@ import type { Db } from "@paperclipai/db"; -import { createGitHubExternalObjectProvider } from "./github-external-object-provider.js"; +import { createGitHubExternalObjectProvider, type GitHubExternalObjectProviderOptions } from "./github-external-object-provider.js"; export type GitHubPullRequestReference = { host: "github.com"; @@ -25,11 +25,13 @@ export type PullRequestMergeDetails = { export type PullRequestMergeStateResolver = ( companyId: string, reference: GitHubPullRequestReference, + signal?: AbortSignal, ) => Promise; export type PullRequestMergeDetailsResolver = ( companyId: string, reference: GitHubPullRequestReference, + signal?: AbortSignal, ) => Promise; export const PULL_REQUEST_CACHE_MAX_ENTRIES = 1_000; @@ -86,14 +88,15 @@ function readRecord(value: unknown): Record | null { : null; } -export function createPullRequestMergeDetailsResolver(db: Db): PullRequestMergeDetailsResolver { - const resolver = createGitHubExternalObjectProvider(db).resolvers +export function createPullRequestMergeDetailsResolver(db: Db, options?: GitHubExternalObjectProviderOptions): PullRequestMergeDetailsResolver { + const resolver = createGitHubExternalObjectProvider(db, options).resolvers .find((candidate) => candidate.objectType === "pull_request") ?? null; - return async (companyId, reference) => { + return async (companyId, reference, signal) => { if (!resolver) return { state: "unknown", headRef: null, headSha: null }; const result = await resolver.resolve({ companyId, + signal, object: { externalId: `${reference.owner}/${reference.repo}#pull/${reference.number}`, sanitizedCanonicalUrl: `https://github.com/${reference.owner}/${reference.repo}/pull/${reference.number}`, @@ -123,7 +126,7 @@ export function createPullRequestMergeDetailsResolver(db: Db): PullRequestMergeD }; } -export function createPullRequestMergeStateResolver(db: Db): PullRequestMergeStateResolver { - const resolveDetails = createPullRequestMergeDetailsResolver(db); - return async (companyId, reference) => (await resolveDetails(companyId, reference)).state; +export function createPullRequestMergeStateResolver(db: Db, options?: GitHubExternalObjectProviderOptions): PullRequestMergeStateResolver { + const resolveDetails = createPullRequestMergeDetailsResolver(db, options); + return async (companyId, reference, signal) => (await resolveDetails(companyId, reference, signal)).state; } diff --git a/server/src/services/issue-thread-interactions.ts b/server/src/services/issue-thread-interactions.ts index b306f637e27..b49ca48d5c1 100644 --- a/server/src/services/issue-thread-interactions.ts +++ b/server/src/services/issue-thread-interactions.ts @@ -204,6 +204,7 @@ export type IssueThreadInteractionServiceOptions = { resolvePullRequestState?: ( companyId: string, reference: GitHubPullRequestReference, + signal?: AbortSignal, ) => Promise; wakeup?: InteractionWakeup; pullRequestCacheTtlMs?: number; @@ -2045,10 +2046,11 @@ export function issueThreadInteractionService( async function resolvePullRequestState( companyId: string, reference: GitHubPullRequestReference, + signal?: AbortSignal, ): Promise { if (opts.resolvePullRequestState) - return opts.resolvePullRequestState(companyId, reference); - return defaultPullRequestStateResolver?.(companyId, reference) ?? "unknown"; + return opts.resolvePullRequestState(companyId, reference, signal); + return defaultPullRequestStateResolver?.(companyId, reference, signal) ?? "unknown"; } async function resolvePullRequestStates( @@ -2057,6 +2059,7 @@ export function issueThreadInteractionService( companyId: string; reference: GitHubPullRequestReference; }>, + signal?: AbortSignal, ) { const states = new Map(); const pending = entries.slice(); @@ -2064,11 +2067,13 @@ export function issueThreadInteractionService( await Promise.all( Array.from({ length: workerCount }, async () => { while (pending.length > 0) { + signal?.throwIfAborted(); const entry = pending.shift(); if (!entry) return; const state = await resolvePullRequestState( entry.companyId, entry.reference, + signal, ); states.set(entry.key, state); } @@ -2076,11 +2081,7 @@ export function issueThreadInteractionService( ); return states; } - async function mergedPullRequestConfirmationCandidates(scope?: MergeConfirmationScope) { - const text = sql`concat_ws(' ', ${issueThreadInteractions.title}, ${issueThreadInteractions.summary}, - ${issueThreadInteractions.payload}->>'prompt', ${issueThreadInteractions.payload}->>'acceptLabel', - ${issueThreadInteractions.payload}->>'detailsMarkdown', ${issueThreadInteractions.payload}->'target'->>'key', - ${issueThreadInteractions.payload}->'target'->>'label', ${issueThreadInteractions.payload}->'target'->>'href')`; + async function mergedPullRequestConfirmationPage(scope?: MergeConfirmationScope, after?: string) { const rows = await db.select({ interaction: issueThreadInteractions, issue: { id: issues.id, companyId: issues.companyId, projectId: issues.projectId, goalId: issues.goalId, status: issues.status, assigneeAgentId: issues.assigneeAgentId, @@ -2092,14 +2093,24 @@ export function issueThreadInteractionService( scope ? eq(issueThreadInteractions.companyId, scope.companyId) : undefined, scope?.issueId ? eq(issueThreadInteractions.issueId, scope.issueId) : undefined, scope?.interactionId ? eq(issueThreadInteractions.id, scope.interactionId) : undefined, - sql`${text} ~* '(merge|merging)'`, - sql`${text} ~* '(github\\.com/[^ /]+/[^ /]+/pull/[1-9][0-9]*|[a-z0-9_.-]+/[a-z0-9_.-]+#[1-9][0-9]*)'`, + after ? sql`${issueThreadInteractions.id} > ${after}::uuid` : undefined, sql`NOT EXISTS (SELECT 1 FROM ${toolActionRequests} WHERE ${toolActionRequests.interactionId} = ${issueThreadInteractions.id})`, - )); - return rows.flatMap(row => { + )).orderBy(asc(issueThreadInteractions.id)).limit(100); + return { rows: rows.flatMap(row => { const references = getMergeConfirmationPullRequestReferences(row.interaction); return references.length ? [{ ...row, references }] : []; - }); + }), after: rows.at(-1)?.interaction.id, complete: rows.length < 100 }; + } + + async function mergedPullRequestConfirmationCandidates(scope?: MergeConfirmationScope) { + const candidates: Awaited>["rows"] = []; + let after: string | undefined; + while (true) { + const page = await mergedPullRequestConfirmationPage(scope, after); + candidates.push(...page.rows); + if (page.complete) return candidates; + after = page.after; + } } async function getIdempotentInteraction(args: { @@ -3012,6 +3023,7 @@ export function issueThreadInteractionService( await emitInteractionResolvedTelemetry(db, interaction); return interaction; }, + mergedPullRequestConfirmationPage, mergedPullRequestConfirmationCandidates, sweepMergedPullRequestConfirmations: async ( mergedHints: Array<{ @@ -3064,7 +3076,7 @@ export function issueThreadInteractionService( const refreshedStates = await resolvePullRequestStates([ ...uniqueReferences.values(), - ]); + ], signal); if (scope && [...refreshedStates.values()].some(state => state === "unknown")) { throw new Error("Merge confirmation provider state is unavailable"); } diff --git a/server/src/services/merged-pr-confirmation-work.ts b/server/src/services/merged-pr-confirmation-work.ts index bfccaeea79f..6a754d5a88a 100644 --- a/server/src/services/merged-pr-confirmation-work.ts +++ b/server/src/services/merged-pr-confirmation-work.ts @@ -9,16 +9,24 @@ export const MERGE_CONFIRMATION_RECONCILE_MS = 5 * 60_000; export function registerMergeConfirmationWork( coordinator: ReturnType, service: Pick, - "mergedPullRequestConfirmationCandidates" | "sweepMergedPullRequestConfirmations">, + "mergedPullRequestConfirmationCandidates" | "mergedPullRequestConfirmationPage" | "sweepMergedPullRequestConfirmations">, canRun: () => boolean, ) { + const cursors = new Map(); return registerScopedDeadlineWork(coordinator, { queue: DELIVERY_QUEUES.mergedPullRequestConfirmations, decodeScope: hint => hint === "*" ? undefined : JSON.parse(hint) as MergeConfirmationScope, matches: (entry, scope?: MergeConfirmationScope) => !scope || (entry.companyId === scope.companyId && (!scope.issueId || entry.issueId === scope.issueId) && (!scope.interactionId || entry.id === scope.interactionId)), - load: async (scope?: MergeConfirmationScope) => (await service.mergedPullRequestConfirmationCandidates(scope)) - .map(({ interaction }) => ({ id: interaction.id, companyId: interaction.companyId, issueId: interaction.issueId, dueAt: Date.now() })), + load: async (scope?: MergeConfirmationScope) => { + const key = scope ? JSON.stringify(scope) : "*"; + const page = await service.mergedPullRequestConfirmationPage(scope, cursors.get(key)); + if (page.complete) cursors.delete(key); + else cursors.set(key, page.after!); + return { complete: page.complete, rows: page.rows.map(({ interaction }) => ({ + id: interaction.id, companyId: interaction.companyId, issueId: interaction.issueId, dueAt: Date.now(), + })) }; + }, check: async (entry, signal) => { const scope = { companyId: entry.companyId, issueId: entry.issueId, interactionId: entry.id }; await service.sweepMergedPullRequestConfirmations([], scope, signal); From d820f2bae4a8c00e1361b6504df2c7018639205d Mon Sep 17 00:00:00 2001 From: Dotta Date: Fri, 30 Oct 2026 16:53:01 -0500 Subject: [PATCH 12/70] refactor(recovery): expose scoped issue recovery obligations Co-Authored-By: Paperclip --- .../modules/wake-queue/adapters/postgres.ts | 4 +- server/src/services/agent-configuration.ts | 2 +- server/src/services/agent-lifecycle.ts | 2 +- server/src/services/budgets.ts | 8 +-- .../chat-run-milestone-notifications.ts | 2 +- server/src/services/companies.ts | 4 +- .../services/delivery-work-notifications.ts | 4 ++ server/src/services/heartbeat.ts | 27 +++++---- server/src/services/heartbeat/queue.ts | 2 +- server/src/services/heartbeat/retries.ts | 4 +- .../src/services/heartbeat/run-lifecycle.ts | 2 +- server/src/services/issue-comment-writes.ts | 2 + .../services/issue-recovery-notifications.ts | 11 ++++ .../issue-review-state-notifications.ts | 2 +- server/src/services/issues.ts | 18 +++--- .../src/services/legacy-controller-lease.ts | 12 ++-- .../native-provider-capacity-retry.ts | 2 +- server/src/services/recovery/service.ts | 41 ++++++++++--- server/src/services/recovery/work.ts | 58 +++++++++++++++++++ .../services/run-dispatch-notifications.ts | 4 +- ...eduled-retry-cancellation-notifications.ts | 2 +- server/src/services/task-watchdog-work.ts | 36 ++++++++++++ server/src/services/task-watchdogs.ts | 46 +++++++++++---- 23 files changed, 233 insertions(+), 62 deletions(-) create mode 100644 server/src/services/issue-recovery-notifications.ts create mode 100644 server/src/services/recovery/work.ts create mode 100644 server/src/services/task-watchdog-work.ts diff --git a/server/src/modules/wake-queue/adapters/postgres.ts b/server/src/modules/wake-queue/adapters/postgres.ts index c773b903dec..1df4420615f 100644 --- a/server/src/modules/wake-queue/adapters/postgres.ts +++ b/server/src/modules/wake-queue/adapters/postgres.ts @@ -638,7 +638,7 @@ function buildTransaction(tx: Db, deps: WakeQueuePostgresAdapterDeps, db: Db, ru .returning() .then((rows) => rows[0]); - await notifyRunDispatch(tx); + await notifyRunDispatch(tx, { companyId }); const queuedRun = await tx .insert(heartbeatRuns) .values({ @@ -717,7 +717,7 @@ function buildTransaction(tx: Db, deps: WakeQueuePostgresAdapterDeps, db: Db, ru .returning() .then((rows) => rows[0]); - await notifyRunDispatch(tx); + await notifyRunDispatch(tx, { companyId }); const queuedRun = await tx .insert(heartbeatRuns) .values({ diff --git a/server/src/services/agent-configuration.ts b/server/src/services/agent-configuration.ts index 88a2c636bdb..dc10efaa082 100644 --- a/server/src/services/agent-configuration.ts +++ b/server/src/services/agent-configuration.ts @@ -84,7 +84,7 @@ export async function prepareAgentConfiguration(db: Db, existing: AgentConfigura existing.adapterConfig, ); } - await notifyRunDispatch(db); + await notifyRunDispatch(db, { companyId: existing.companyId }); if (normalizedPatch.runtimeConfig !== undefined || normalizedPatch.reportsTo !== undefined) { await notifyDeliveryWork(db, DELIVERY_QUEUES.agentTimers); } diff --git a/server/src/services/agent-lifecycle.ts b/server/src/services/agent-lifecycle.ts index 700c6ba8ffa..59357fcc5de 100644 --- a/server/src/services/agent-lifecycle.ts +++ b/server/src/services/agent-lifecycle.ts @@ -40,7 +40,7 @@ export function createAgentLifecycleEffects(_hooks: BudgetServiceHooks = {}): li notifyWork: tx => notifyDeliveryWork(tx, DELIVERY_QUEUES.agentLifecycle), async beforeStatusChange(tx, companyId, agentId, before, after) { if (before !== after) { - await notifyRunDispatch(tx); + await notifyRunDispatch(tx, { companyId }); await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers); } if (!isAgentStatusInvokable(before) && isAgentStatusInvokable(after)) { diff --git a/server/src/services/budgets.ts b/server/src/services/budgets.ts index 86d00c4d108..186253b19bb 100644 --- a/server/src/services/budgets.ts +++ b/server/src/services/budgets.ts @@ -288,7 +288,7 @@ export function budgetServiceInTransaction(db: Db, publications: ActivityPublica } if (policy.scopeType === "project") { - await notifyRunDispatch(db); + await notifyRunDispatch(db, { companyId: policy.companyId }); await db .update(projects) .set({ @@ -302,7 +302,7 @@ export function budgetServiceInTransaction(db: Db, publications: ActivityPublica const scope = await resolveScopeRecord(db, "company", policy.scopeId); if (scope.paused) return; - await notifyRunDispatch(db); + await notifyRunDispatch(db, { companyId: policy.companyId }); await notifyCompanyRetentionWork(db, policy.companyId); await notifyDeliveryWork(db, DELIVERY_QUEUES.agentTimers); await notifyDeliveryWork(db, DELIVERY_QUEUES.routineSchedules); @@ -349,7 +349,7 @@ export function budgetServiceInTransaction(db: Db, publications: ActivityPublica } if (policy.scopeType === "project") { - await notifyRunDispatch(db); + await notifyRunDispatch(db, { companyId: policy.companyId }); await db .update(projects) .set({ @@ -364,7 +364,7 @@ export function budgetServiceInTransaction(db: Db, publications: ActivityPublica await notifyDeliveryWork(db, DELIVERY_QUEUES.skillSourceSync); const scope = await resolveScopeRecord(db, "company", policy.scopeId); if (scope.pauseReason !== "budget") return; - await notifyRunDispatch(db); + await notifyRunDispatch(db, { companyId: policy.companyId }); await notifyCompanyRetentionWork(db, policy.companyId); await notifyDeliveryWork(db, DELIVERY_QUEUES.agentTimers); await notifyDeliveryWork(db, DELIVERY_QUEUES.routineSchedules); diff --git a/server/src/services/chat-run-milestone-notifications.ts b/server/src/services/chat-run-milestone-notifications.ts index 159055c10d3..524936395b5 100644 --- a/server/src/services/chat-run-milestone-notifications.ts +++ b/server/src/services/chat-run-milestone-notifications.ts @@ -12,7 +12,7 @@ import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notificatio export type ChatRunMilestoneScope = { companyId: string; issueId?: string; runId?: string }; export async function notifyChatRunMilestones(tx: object, scope: ChatRunMilestoneScope) { - if (!scope.issueId && !scope.runId) await notifyRunDispatch(tx); + if (!scope.issueId && !scope.runId) await notifyRunDispatch(tx, scope); if (scope.issueId || scope.runId) { await notifyIssueReviewState(tx, { companyId: scope.companyId, issueId: scope.issueId, runId: scope.issueId ? undefined : scope.runId }); diff --git a/server/src/services/companies.ts b/server/src/services/companies.ts index cc10ffa3bb8..a9a8f59998c 100644 --- a/server/src/services/companies.ts +++ b/server/src/services/companies.ts @@ -333,7 +333,7 @@ export function companyService(db: Db, _budgetHooks: BudgetServiceHooks = {}) { const renamedPrefix = await resolveRenamedIssuePrefix(tx, id, companyPatch); if (companyPatch.status !== undefined && companyPatch.status !== existing.status) { - await notifyRunDispatch(tx); + await notifyRunDispatch(tx, { companyId: id }); await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers); await notifyDeliveryWork(tx, DELIVERY_QUEUES.skillSourceSync); await notifyDeliveryWork(tx, DELIVERY_QUEUES.routineSchedules); @@ -490,7 +490,7 @@ export function companyService(db: Db, _budgetHooks: BudgetServiceHooks = {}) { const wasAlreadyArchived = existing.status === "archived"; if (!wasAlreadyArchived) { - await notifyRunDispatch(tx); + await notifyRunDispatch(tx, { companyId: id }); await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers); await notifyDeliveryWork(tx, DELIVERY_QUEUES.skillSourceSync); await notifyDeliveryWork(tx, DELIVERY_QUEUES.routineSchedules); diff --git a/server/src/services/delivery-work-notifications.ts b/server/src/services/delivery-work-notifications.ts index 7388bdb8861..1d9ac79f520 100644 --- a/server/src/services/delivery-work-notifications.ts +++ b/server/src/services/delivery-work-notifications.ts @@ -2,6 +2,10 @@ import { signalDatabaseWork, subscribeDatabaseWork } from "@paperclipai/db"; /** Topics for existing durable queues; register intent before their writes. */ export const DELIVERY_QUEUES = { + strandedAssignments: "stranded-assignments", + resolvedDependencies: "resolved-dependencies", + taskWatchdogs: "task-watchdogs", + staleIssueLocks: "stale-issue-locks", scheduledRetryDispatch: "scheduled-retry-dispatch", queuedRunDispatch: "queued-run-dispatch", runEvidenceMigration: "run-evidence-migration", diff --git a/server/src/services/heartbeat.ts b/server/src/services/heartbeat.ts index 0772b920023..8c73880ce57 100644 --- a/server/src/services/heartbeat.ts +++ b/server/src/services/heartbeat.ts @@ -1,3 +1,4 @@ +import { registerIssueRecoveryWork } from "./recovery/work.js"; import { registerRunDispatchWork } from "./heartbeat/dispatch-work.js"; import { externalConversationSettlementWork } from "./external-conversation-settlement-work.js"; import { nativeRecoveryNoticeWork } from "./native-runtime/native-recovery-notice-work.js"; @@ -1611,15 +1612,15 @@ export function heartbeatService( void reportRunFailure(db, updated, failureReport); } - async function reconcileStrandedAssignedIssues(opts?: { skipExternalConversationSettlement?: boolean }) { + async function reconcileStrandedAssignedIssues(opts?: Parameters[0]) { return recovery.reconcileStrandedAssignedIssues({ - skipExternalConversationSettlement: opts?.skipExternalConversationSettlement, + ...opts, issueCreatedAtGte: await getWorktreeExecutionCutoff(), }); } - async function sweepStaleIssueLocks() { - return recovery.sweepStaleIssueLocks(); + async function sweepStaleIssueLocks(opts?: Parameters[0]) { + return recovery.sweepStaleIssueLocks(opts); } function issueIdFromRunContext(contextSnapshot: unknown) { @@ -1649,10 +1650,7 @@ export function heartbeatService( }); } - async function reconcileTaskWatchdogs(opts?: { - companyId?: string | null; - runId?: string | null; - }) { + async function reconcileTaskWatchdogs(opts?: Parameters[0]) { return taskWatchdogs.reconcileTaskWatchdogs({ ...opts, issueCreatedAtGte: await getWorktreeExecutionCutoff(), @@ -1677,10 +1675,7 @@ export function heartbeatService( return recovery.buildRunOutputSilence(run, now); } - async function reconcileResolvedDependencyWakes(opts?: { - runId?: string | null; - companyId?: string | null; - }) { + async function reconcileResolvedDependencyWakes(opts?: Parameters[0]) { return recovery.reconcileResolvedDependencyWakeBackstop(opts); } @@ -4248,6 +4243,14 @@ export function heartbeatService( computeTaskDrain, applyTaskDrain, + registerIssueStateRecoveryWork: (coordinator: Parameters[1], canRun: () => boolean) => + registerIssueRecoveryWork(db, coordinator, { + stranded: scope => reconcileStrandedAssignedIssues({ ...scope, skipExternalConversationSettlement: true }), + dependencies: reconcileResolvedDependencyWakes, + locks: sweepStaleIssueLocks, + watchdogs: reconcileTaskWatchdogs, + schedulingSuppressed: async () => (await getSchedulingSuppression(true)).suppressed, + }, () => canRun() && readTaskDrain(new Date()) === null), registerRunDispatchWork: (coordinator: Parameters[0], canRun: () => boolean) => registerRunDispatchWork(coordinator, { promoteDueScheduledRetries, diff --git a/server/src/services/heartbeat/queue.ts b/server/src/services/heartbeat/queue.ts index 376530b2f0a..9b08eece201 100644 --- a/server/src/services/heartbeat/queue.ts +++ b/server/src/services/heartbeat/queue.ts @@ -4633,7 +4633,7 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende } const queueOutcome = await db.transaction(async (tx) => { - await notifyRunDispatch(tx); + await notifyRunDispatch(tx, { companyId: agent.companyId }); await tx.execute( sql`select id from agents where id = ${agentId} and company_id = ${agent.companyId} for update`, ); diff --git a/server/src/services/heartbeat/retries.ts b/server/src/services/heartbeat/retries.ts index 5f4df5ed74e..32333c0ebec 100644 --- a/server/src/services/heartbeat/retries.ts +++ b/server/src/services/heartbeat/retries.ts @@ -998,7 +998,7 @@ export function createHeartbeatRetries(db: Db, dependencies: HeartbeatRetryDepen retryContextSnapshot.previousRunId = continuation.previousRunId; } - await notifyRunDispatch(tx); + await notifyRunDispatch(tx, { companyId: run.companyId }); const wakeupRequest = await tx .insert(agentWakeupRequests) .values({ @@ -1756,7 +1756,7 @@ export function createHeartbeatRetries(db: Db, dependencies: HeartbeatRetryDepen }; const updated = await db.transaction(async (tx) => { - await notifyRunDispatch(tx); + await notifyRunDispatch(tx, { companyId: scheduled.run.companyId }); const row = await tx .update(heartbeatRuns) .set({ diff --git a/server/src/services/heartbeat/run-lifecycle.ts b/server/src/services/heartbeat/run-lifecycle.ts index 07bb54dfdf7..e44906e72fb 100644 --- a/server/src/services/heartbeat/run-lifecycle.ts +++ b/server/src/services/heartbeat/run-lifecycle.ts @@ -2448,7 +2448,7 @@ export function createHeartbeatLifecycle(db: Db, dependencies: HeartbeatLifecycl .returning() .then((rows) => rows[0]); - await notifyRunDispatch(tx); + await notifyRunDispatch(tx, { companyId: run.companyId }); const queuedRun = await tx .insert(heartbeatRuns) .values({ diff --git a/server/src/services/issue-comment-writes.ts b/server/src/services/issue-comment-writes.ts index e24e78e9855..a968ecb1fec 100644 --- a/server/src/services/issue-comment-writes.ts +++ b/server/src/services/issue-comment-writes.ts @@ -1,3 +1,4 @@ +import { notifyIssueRecovery } from "./issue-recovery-notifications.js"; import type { PgInsertValue } from "drizzle-orm/pg-core"; import { issueComments, type Db } from "@paperclipai/db"; import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js"; @@ -6,6 +7,7 @@ type WriteDb = Db | Parameters[0]>[0]; export function insertIssueComment(db: WriteDb, input: PgInsertValue & { companyId: string }) { return db.transaction(async tx => { + await notifyIssueRecovery(tx, { companyId: input.companyId }); await notifyDeliveryWork(tx, DELIVERY_QUEUES.dotEvents, `company:${input.companyId}`); return tx.insert(issueComments).values(input).returning(); }); diff --git a/server/src/services/issue-recovery-notifications.ts b/server/src/services/issue-recovery-notifications.ts new file mode 100644 index 00000000000..db04ca801ba --- /dev/null +++ b/server/src/services/issue-recovery-notifications.ts @@ -0,0 +1,11 @@ +import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js"; + +export type IssueRecoveryScope = { companyId: string }; + +export async function notifyIssueRecovery(transaction: object, scope?: IssueRecoveryScope) { + const hint = scope ? JSON.stringify(scope) : "*"; + for (const queue of [DELIVERY_QUEUES.strandedAssignments, DELIVERY_QUEUES.resolvedDependencies, + DELIVERY_QUEUES.taskWatchdogs, DELIVERY_QUEUES.staleIssueLocks]) { + await notifyDeliveryWork(transaction, queue, hint); + } +} diff --git a/server/src/services/issue-review-state-notifications.ts b/server/src/services/issue-review-state-notifications.ts index b769be64056..d9d4d756816 100644 --- a/server/src/services/issue-review-state-notifications.ts +++ b/server/src/services/issue-review-state-notifications.ts @@ -5,7 +5,7 @@ import { notifyNativeReviewResponse } from "./native-review-response-notificatio import { notifyNativeStatusReassessment } from "./native-status-reassessment-notifications.js"; export async function notifyIssueReviewState(tx: object, scope: { companyId: string; issueId?: string; runId?: string; agentId?: string }) { - await notifyRunDispatch(tx); + await notifyRunDispatch(tx, scope); await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors); await notifyExternalConversationSettlement(tx, scope); await notifyNativeReviewResponse(tx, scope); diff --git a/server/src/services/issues.ts b/server/src/services/issues.ts index a7c8bb55023..575648ccc37 100644 --- a/server/src/services/issues.ts +++ b/server/src/services/issues.ts @@ -10606,7 +10606,7 @@ export function issueService(db: Db) { : values.parentId ? `issue:${values.parentId}` : undefined); } if (values.monitorNextCheckAt) await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors); - await notifyRunDispatch(tx); + await notifyRunDispatch(tx, { companyId: values.companyId }); await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers); const [issue] = await tx.insert(issues).values(values).returning(); await attachOwnedDraftImages(tx, issue, issue.description, { userId: issueData.createdByUserId, agentId: issueData.createdByAgentId }); @@ -11494,8 +11494,10 @@ export function issueService(db: Db) { await notifyTerminalWorkspaceWork(tx, `issue:${id}`); } if (patch.status !== undefined || patch.assigneeAgentId !== undefined || patch.assigneeUserId !== undefined - || patch.hiddenAt !== undefined || patch.conversationAgentId !== undefined) { - await notifyRunDispatch(tx); + || patch.hiddenAt !== undefined || patch.conversationAgentId !== undefined || patch.parentId !== undefined + || patch.executionState !== undefined || patch.executionRunId !== undefined || patch.checkoutRunId !== undefined + || blockedByIssueIds !== undefined) { + await notifyRunDispatch(tx, { companyId: receiptExisting.companyId }); await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers); } if (patch.monitorNextCheckAt !== undefined || (receiptExisting.monitorNextCheckAt && @@ -11960,7 +11962,7 @@ export function issueService(db: Db) { } await notifyDecisionTargetIssueRemoval(tx, id); await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors); - await notifyRunDispatch(tx); + await notifyRunDispatch(tx, owner ? { companyId: owner.companyId } : undefined); await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers); removedIssue = await tx .delete(issues) @@ -12089,7 +12091,7 @@ export function issueService(db: Db) { : isNull(issues.executionRunId); const updated = await db.transaction(async tx => { await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors); - await notifyRunDispatch(tx); + await notifyRunDispatch(tx, { companyId: issueCompany.companyId }); await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers); return tx .update(issues) @@ -12219,7 +12221,7 @@ export function issueService(db: Db) { const staleExecutionRunId = current.executionRunId; const adopted = await db.transaction(async tx => { await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors); - await notifyRunDispatch(tx); + await notifyRunDispatch(tx, { companyId: issueCompany.companyId }); await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers); return tx .update(issues) @@ -12475,7 +12477,7 @@ export function issueService(db: Db) { await notifyTerminalWorkspaceWork(tx, `issue:${id}`); await notifyNativeStatusReassessment(tx, { companyId: existing.companyId, issueId: id }); await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors); - await notifyRunDispatch(tx); + await notifyRunDispatch(tx, { companyId: existing.companyId }); await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers); const updated = await tx .update(issues) @@ -12536,7 +12538,7 @@ export function issueService(db: Db) { await notifyUnsafeWorkspaceExportRecovery(tx, { companyId: existing.companyId, issueId: id }); } await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors); - await notifyRunDispatch(tx); + await notifyRunDispatch(tx, { companyId: existing.companyId }); await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers); const updated = await tx .update(issues) diff --git a/server/src/services/legacy-controller-lease.ts b/server/src/services/legacy-controller-lease.ts index 55a8439debc..22fa078f459 100644 --- a/server/src/services/legacy-controller-lease.ts +++ b/server/src/services/legacy-controller-lease.ts @@ -77,14 +77,18 @@ async function renewOnLivenessPool( }); } -export async function hasLiveLegacyController(db: Db, run: Run): Promise { - if (run.runtimeMode === "native" || !run.controllerBootId) return false; - const [owner] = await db.select({ id: heartbeatRuns.id }).from(heartbeatRuns).where(and( +export async function liveLegacyControllerExpiresAt(db: Db, run: Run): Promise { + if (run.runtimeMode === "native" || !run.controllerBootId) return null; + const [owner] = await db.select({ expiresAt: heartbeatRuns.controllerLeaseExpiresAt }).from(heartbeatRuns).where(and( eq(heartbeatRuns.id, run.id), eq(heartbeatRuns.companyId, run.companyId), eq(heartbeatRuns.status, "running"), gt(heartbeatRuns.controllerLeaseExpiresAt, sql`clock_timestamp()`), )); - return Boolean(owner); + return owner?.expiresAt ?? null; +} + +export async function hasLiveLegacyController(db: Db, run: Run): Promise { + return (await liveLegacyControllerExpiresAt(db, run)) !== null; } export type RevokedLegacyControllerState = Pick; diff --git a/server/src/services/native-runtime/native-provider-capacity-retry.ts b/server/src/services/native-runtime/native-provider-capacity-retry.ts index 7d007c383dd..83314ff9c51 100644 --- a/server/src/services/native-runtime/native-provider-capacity-retry.ts +++ b/server/src/services/native-runtime/native-provider-capacity-retry.ts @@ -50,7 +50,7 @@ export async function materializeNativeProviderCapacityRetry(input: { scheduledRetryAt: dueAt.toISOString(), executionRetryAccounting: accountingForScheduledRetry(run, NATIVE_PROVIDER_OVERLOADED_CODE, attempt), }; - await notifyRunDispatch(input.tx); + await notifyRunDispatch(input.tx, { companyId: run.companyId }); const [wake] = await input.tx.insert(agentWakeupRequests).values({ companyId: run.companyId, agentId: run.agentId, source: "automation", triggerDetail: "system", reason: NATIVE_PROVIDER_OVERLOADED_CODE, status: "queued", runId: retryId, diff --git a/server/src/services/recovery/service.ts b/server/src/services/recovery/service.ts index 4393e0ad5cf..7888b3e23fb 100644 --- a/server/src/services/recovery/service.ts +++ b/server/src/services/recovery/service.ts @@ -27,7 +27,7 @@ import { decideLegacyContinuation, legacyDispositionEpisode, legacyDispositionFingerprint, LEGACY_DISPOSITION_REPAIR_INSTRUCTION, type LegacyDispositionEpisode, } from "./legacy-continuation.js"; -import { CONTROLLER_LEASE_LOST_FAILURE_CODE, hasLiveLegacyController } from "../legacy-controller-lease.js"; +import { CONTROLLER_LEASE_LOST_FAILURE_CODE, hasLiveLegacyController, liveLegacyControllerExpiresAt } from "../legacy-controller-lease.js"; import { isCompletedOnboardingHandoffWake } from "../chat-completion-delivery.js"; import { instanceSettingsService } from "../instance-settings.js"; import { isWaitingConversation, settleConversationTurnForRun, deliverConversationComments } from "../agent-conversations.js"; @@ -245,6 +245,7 @@ type ResolvedDependencyWakeBackstopOptions = { runId?: string | null; companyId?: string | null; blockerIssueId?: string | null; + issueId?: string; source?: ResolvedDependencyWakeBackstopSource; }; @@ -2717,7 +2718,7 @@ export function recoveryService( const now = new Date(); const retryAt = readProviderQuotaRetryAt(input.latestRun, now); return database.transaction(async (tx) => { - await notifyRunDispatch(tx); + await notifyRunDispatch(tx, { companyId: input.issue.companyId }); const wakeup = await tx .insert(agentWakeupRequests) .values({ @@ -3318,7 +3319,7 @@ export function recoveryService( created = Boolean(enqueuedRun); } else { scheduledRun = await db.transaction(async (tx) => { - await notifyRunDispatch(tx); + await notifyRunDispatch(tx, { companyId: input.issue.companyId }); const wakeup = await tx .insert(agentWakeupRequests) .values({ @@ -4526,6 +4527,9 @@ export function recoveryService( } async function reconcileStrandedAssignedIssues(opts?: { + companyId?: string; + issueId?: string; + onDeferredDeadline?: (at: number) => void; issueCreatedAtGte?: Date | null; skipExternalConversationSettlement?: boolean; }) { @@ -4534,6 +4538,8 @@ export function recoveryService( .from(issues) .where( and( + opts?.companyId ? eq(issues.companyId, opts.companyId) : undefined, + opts?.issueId ? eq(issues.id, opts.issueId) : undefined, isNull(issues.assigneeUserId), inArray(issues.status, ["todo", "in_progress", "in_review"]), or( @@ -5714,6 +5720,7 @@ export function recoveryService( classification.baseBackoffMs * Math.pow(2, Math.max(0, consecutive - 1)); if (elapsed < requiredDelay) { + opts?.onDeferredDeadline?.(latestFinishedAt.getTime() + requiredDelay); result.skipped += 1; continue; } @@ -5815,7 +5822,7 @@ export function recoveryService( source === "workspace.finalize" ? "workspace_finalize_reconciliation" : "issue_graph_liveness_reconciliation"; - const useCursor = !opts?.blockerIssueId; + const useCursor = !opts?.blockerIssueId && !opts?.issueId; const queryCandidates = (afterIssueId: string | null) => { const filters = [ @@ -5825,6 +5832,7 @@ export function recoveryService( sql`${issues.assigneeAgentId} is not null`, ]; if (opts?.companyId) filters.push(eq(issues.companyId, opts.companyId)); + if (opts?.issueId) filters.push(eq(issues.id, opts.issueId)); if (afterIssueId) filters.push(gt(issues.id, afterIssueId)); if (opts?.blockerIssueId) { @@ -6115,6 +6123,7 @@ export function recoveryService( // fallback below. Without it, a terminal issue named in the run context // snapshot would still terminalize the shared run and defeat the guard. runReferencedByActiveIssue?: boolean; + onDeferredDeadline?: (at: number) => void; }, ): Promise<{ terminalized: boolean; status: string }> { // Act only on a run in "running" status. A "queued" run has no process yet, @@ -6130,8 +6139,11 @@ export function recoveryService( // Another controller may own a sandbox run whose PID has no meaning on // this host. Its live lease owns both execution and finalization, even if // the issue is already terminal or this process has no in-memory handle. - if (await hasLiveLegacyController(db, run)) + const controllerDeadline = await liveLegacyControllerExpiresAt(db, run); + if (controllerDeadline) { + options?.onDeferredDeadline?.(controllerDeadline.getTime()); return { terminalized: false, status: run.status }; + } const pid = run.processPid ?? null; const processGroupId = run.processGroupId ?? null; @@ -6352,7 +6364,7 @@ export function recoveryService( // still claims to be live but can no longer reach a terminal status on its // own, so a stuck "running" run can no longer block the sweep. Idempotent and // safe: clears at most one row's worth of lock columns per candidate. - async function sweepStaleIssueLocks() { + async function sweepStaleIssueLocks(opts?: { companyId?: string; issueId?: string; onDeferredDeadline?: (at: number) => void }) { const result = { cleared: 0, issueIds: [] as string[], @@ -6369,7 +6381,9 @@ export function recoveryService( }) .from(issues) .where( - sql`(${issues.checkoutRunId} is not null or ${issues.executionRunId} is not null)`, + and(sql`(${issues.checkoutRunId} is not null or ${issues.executionRunId} is not null)`, + opts?.companyId ? eq(issues.companyId, opts.companyId) : undefined, + opts?.issueId ? eq(issues.id, opts.issueId) : undefined), ); const referencedRunIds = [ @@ -6384,7 +6398,8 @@ export function recoveryService( ? await db .select() .from(heartbeatRuns) - .where(inArray(heartbeatRuns.id, referencedRunIds)) + .where(and(inArray(heartbeatRuns.id, referencedRunIds), + opts?.companyId ? eq(heartbeatRuns.companyId, opts.companyId) : undefined)) : []; const runStatusById = new Map(); for (const row of runRows) runStatusById.set(row.id, row.status); @@ -6395,8 +6410,15 @@ export function recoveryService( // the same run id in a stale lock column. The terminal reference alone must // not terminalize a run that an active issue still owns, so exclude these // runs from the issue-terminal authority below. + const ownershipReferences = opts?.issueId && referencedRunIds.length > 0 + ? await db.select({ companyId: issues.companyId, status: issues.status, + checkoutRunId: issues.checkoutRunId, executionRunId: issues.executionRunId }).from(issues).where(and( + opts.companyId ? eq(issues.companyId, opts.companyId) : undefined, + or(inArray(issues.checkoutRunId, referencedRunIds), inArray(issues.executionRunId, referencedRunIds)), + )) + : candidates; const runIdsReferencedByActiveIssue = new Set(); - for (const issue of candidates) { + for (const issue of ownershipReferences) { if (issue.status === "done" || issue.status === "cancelled") continue; for (const runId of [issue.checkoutRunId, issue.executionRunId]) { if (runId) runIdsReferencedByActiveIssue.add(runId); @@ -6437,6 +6459,7 @@ export function recoveryService( referencingIssueTerminalStatus: issueTerminalStatusByRunId.get(row.id) ?? null, runReferencedByActiveIssue: runIdsReferencedByActiveIssue.has(row.id), + onDeferredDeadline: opts?.onDeferredDeadline, }); runStatusById.set(row.id, outcome.status); if (outcome.terminalized) result.terminalizedRunIds.push(row.id); diff --git a/server/src/services/recovery/work.ts b/server/src/services/recovery/work.ts new file mode 100644 index 00000000000..93679e80423 --- /dev/null +++ b/server/src/services/recovery/work.ts @@ -0,0 +1,58 @@ +import { registerTaskWatchdogWork } from "../task-watchdog-work.js"; +import { and, eq, inArray, isNotNull, isNull, or } from "drizzle-orm"; +import { issues, type Db } from "@paperclipai/db"; +import type { createDeliveryWorkCoordinator } from "../delivery-work-coordinator.js"; +import { DELIVERY_QUEUES, type DeliveryQueue } from "../delivery-work-notifications.js"; +import type { IssueRecoveryScope } from "../issue-recovery-notifications.js"; +import { registerScopedDeadlineWork } from "../scoped-deadline-work.js"; + +type Entry = { id: string; companyId: string; dueAt: number }; +type Scope = { companyId: string; issueId: string; onDeferredDeadline: (at: number) => void }; + +export function registerIssueRecoveryWork( + db: Db, + coordinator: ReturnType, + owner: { + stranded(scope: Scope): Promise; + dependencies(scope: Scope): Promise<{ enqueueFailed: number }>; + locks(scope: Scope): Promise; + watchdogs(scope: { companyId: string; watchdogId: string; onDeferredDeadline: (at: number) => void }): Promise<{ incomplete: number }>; + schedulingSuppressed(): Promise; + }, + canRun: () => boolean, +) { + function register(queue: DeliveryQueue, statuses: string[] | null, check: (scope: Scope) => Promise) { + return registerScopedDeadlineWork(coordinator, { + queue, + decodeScope: hint => hint === "*" ? undefined : JSON.parse(hint) as IssueRecoveryScope, + matches: (entry, scope) => !scope || entry.companyId === scope.companyId, + load: async scope => { + const rows = await db.select({ id: issues.id, companyId: issues.companyId }).from(issues).where(and( + scope ? eq(issues.companyId, scope.companyId) : undefined, + statuses ? inArray(issues.status, statuses) : or(isNotNull(issues.checkoutRunId), isNotNull(issues.executionRunId)), + statuses ? isNull(issues.hiddenAt) : undefined, + )); + return rows.map(row => ({ ...row, dueAt: Date.now() })); + }, + check: async entry => { + if (await owner.schedulingSuppressed()) return null; + let next: number | null = null; + await check({ companyId: entry.companyId, issueId: entry.id, + onDeferredDeadline: at => { next = next === null ? at : Math.min(next, at); } }); + return next === null ? null : { ...entry, dueAt: next }; + }, + }, canRun); + } + const assignments = register(DELIVERY_QUEUES.strandedAssignments, ["todo", "in_progress", "in_review"], owner.stranded); + const dependencies = register(DELIVERY_QUEUES.resolvedDependencies, ["blocked"], async scope => { + const result = await owner.dependencies(scope); + if (result.enqueueFailed > 0) throw new Error("Resolved dependency wake remains undelivered"); + }); + const locks = register(DELIVERY_QUEUES.staleIssueLocks, null, owner.locks); + const watchdogs = registerTaskWatchdogWork(db, coordinator, { + reconcile: owner.watchdogs, enabled: async () => !(await owner.schedulingSuppressed()), + }, canRun); + return { ready: Promise.all([assignments.ready, dependencies.ready, locks.ready, watchdogs.ready]), + wake() { assignments.invalidate(); dependencies.invalidate(); locks.invalidate(); watchdogs.invalidate(); }, + async stop() { await Promise.all([assignments.stop(), dependencies.stop(), locks.stop(), watchdogs.stop()]); } }; +} diff --git a/server/src/services/run-dispatch-notifications.ts b/server/src/services/run-dispatch-notifications.ts index 9b1bb175894..a7a3382872a 100644 --- a/server/src/services/run-dispatch-notifications.ts +++ b/server/src/services/run-dispatch-notifications.ts @@ -1,6 +1,8 @@ +import { notifyIssueRecovery, type IssueRecoveryScope } from "./issue-recovery-notifications.js"; import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js"; -export async function notifyRunDispatch(transaction: object) { +export async function notifyRunDispatch(transaction: object, scope?: IssueRecoveryScope) { + await notifyIssueRecovery(transaction, scope); await notifyDeliveryWork(transaction, DELIVERY_QUEUES.scheduledRetryDispatch); await notifyDeliveryWork(transaction, DELIVERY_QUEUES.queuedRunDispatch); } diff --git a/server/src/services/scheduled-retry-cancellation-notifications.ts b/server/src/services/scheduled-retry-cancellation-notifications.ts index c5548996607..24e57de66cb 100644 --- a/server/src/services/scheduled-retry-cancellation-notifications.ts +++ b/server/src/services/scheduled-retry-cancellation-notifications.ts @@ -4,6 +4,6 @@ import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notificatio export type ScheduledRetryCancellationScope = { companyId: string; runId?: string }; export async function notifyScheduledRetryCancellation(transaction: object, scope?: ScheduledRetryCancellationScope) { - await notifyRunDispatch(transaction); + await notifyRunDispatch(transaction, scope); await notifyDeliveryWork(transaction, DELIVERY_QUEUES.scheduledRetryCancellation, scope ? JSON.stringify(scope) : "*"); } diff --git a/server/src/services/task-watchdog-work.ts b/server/src/services/task-watchdog-work.ts new file mode 100644 index 00000000000..091c8c6595f --- /dev/null +++ b/server/src/services/task-watchdog-work.ts @@ -0,0 +1,36 @@ +import { and, eq } from "drizzle-orm"; +import { issueWatchdogs, type Db } from "@paperclipai/db"; +import type { createDeliveryWorkCoordinator } from "./delivery-work-coordinator.js"; +import { DELIVERY_QUEUES } from "./delivery-work-notifications.js"; +import type { IssueRecoveryScope } from "./issue-recovery-notifications.js"; +import { registerScopedDeadlineWork } from "./scoped-deadline-work.js"; + +type Entry = { id: string; companyId: string; dueAt: number }; + +export function registerTaskWatchdogWork( + db: Db, + coordinator: ReturnType, + owner: { + enabled(): Promise; + reconcile(scope: { companyId: string; watchdogId: string; onDeferredDeadline: (at: number) => void }): Promise<{ incomplete: number }>; + }, + canRun: () => boolean, +) { + return registerScopedDeadlineWork(coordinator, { + queue: DELIVERY_QUEUES.taskWatchdogs, + decodeScope: hint => hint === "*" ? undefined : JSON.parse(hint) as IssueRecoveryScope, + matches: (entry, scope) => !scope || entry.companyId === scope.companyId, + load: async scope => (await db.select({ id: issueWatchdogs.id, companyId: issueWatchdogs.companyId }) + .from(issueWatchdogs).where(and(eq(issueWatchdogs.status, "active"), + scope ? eq(issueWatchdogs.companyId, scope.companyId) : undefined))) + .map(row => ({ ...row, dueAt: Date.now() })), + check: async entry => { + if (!(await owner.enabled())) return null; + let next: number | null = null; + const result = await owner.reconcile({ companyId: entry.companyId, watchdogId: entry.id, + onDeferredDeadline: at => { next = next === null ? at : Math.min(next, at); } }); + if (result.incomplete > 0) throw new Error("Task watchdog subtree remains incomplete"); + return next === null ? null : { ...entry, dueAt: next }; + }, + }, canRun); +} diff --git a/server/src/services/task-watchdogs.ts b/server/src/services/task-watchdogs.ts index cf8d92d39df..6f94e5520a7 100644 --- a/server/src/services/task-watchdogs.ts +++ b/server/src/services/task-watchdogs.ts @@ -1,3 +1,4 @@ +import { notifyIssueRecovery } from "./issue-recovery-notifications.js"; import { createHash } from "node:crypto"; import { and, asc, eq, gte, inArray, isNull, or, sql } from "drizzle-orm"; import type { Db } from "@paperclipai/db"; @@ -831,7 +832,7 @@ async function updateIssueWatchdogRow( return updated; } -export async function upsertIssueWatchdogForIssue( +async function upsertIssueWatchdogInTransaction( dbOrTx: any, companyId: string, issueId: string, @@ -852,7 +853,7 @@ export async function upsertIssueWatchdogForIssue( return { watchdog: toIssueWatchdog(updated), created: false }; } - const insertResult: { row: IssueWatchdogRow; created: boolean } = await dbOrTx + const insertResult: { row: IssueWatchdogRow; created: boolean } = await dbOrTx.transaction(async (tx: Db) => tx .insert(issueWatchdogs) .values({ companyId, @@ -870,7 +871,7 @@ export async function upsertIssueWatchdogForIssue( updatedAt: now, }) .returning() - .then((rows: IssueWatchdogRow[]) => ({ row: rows[0], created: true })) + .then((rows: IssueWatchdogRow[]) => ({ row: rows[0], created: true }))) .catch(async (error: unknown) => { if (!isIssueWatchdogUniqueConflict(error)) throw error; const winner = await dbOrTx @@ -885,6 +886,18 @@ export async function upsertIssueWatchdogForIssue( return { watchdog: toIssueWatchdog(insertResult.row), created: insertResult.created }; } +export function upsertIssueWatchdogForIssue( + dbOrTx: Db | Parameters[0]>[0], + companyId: string, + issueId: string, + input: IssueWatchdogUpsertInput, +): Promise<{ watchdog: IssueWatchdog; created: boolean }> { + return dbOrTx.transaction(async tx => { + await notifyIssueRecovery(tx, { companyId }); + return upsertIssueWatchdogInTransaction(tx, companyId, issueId, input); + }); +} + export function taskWatchdogService(db: Db, deps: TaskWatchdogServiceDeps = {}) { const issuesSvc = issueService(db); @@ -1478,7 +1491,7 @@ export function taskWatchdogService(db: Db, deps: TaskWatchdogServiceDeps = {}) return created; } - async function evaluateWatchdog(row: IssueWatchdogRow, opts: { runId?: string | null } = {}) { + async function evaluateWatchdog(row: IssueWatchdogRow, opts: { runId?: string | null; onDeferredDeadline?: (at: number) => void } = {}) { const watchdog = await markTerminalWatchdogIssueReviewed(row, opts); const sourceIssue = await db .select() @@ -1491,6 +1504,13 @@ export function taskWatchdogService(db: Db, deps: TaskWatchdogServiceDeps = {}) const input = await collectClassifierInput(watchdog.companyId, watchdog); const classification = classifyTaskWatchdogSubtree(input); + if (classification.state === "pending_first_run") { + const pending = new Set(classification.pendingIssueIds); + for (const issue of input.issues) if (pending.has(issue.id)) { + const createdAt = toEpochMs(issue.createdAt); + if (createdAt !== null) opts.onDeferredDeadline?.(createdAt + TASK_WATCHDOG_FIRST_RUN_GRACE_MS); + } + } if (classification.state !== "stopped") { return { state: classification.state, reason: classification.reason, classification }; } @@ -1610,13 +1630,14 @@ export function taskWatchdogService(db: Db, deps: TaskWatchdogServiceDeps = {}) }; } - async function listActiveWatchdogsForCompany(companyId?: string | null) { + async function listActiveWatchdogsForCompany(companyId?: string | null, watchdogId?: string) { return db .select() .from(issueWatchdogs) .where(and( eq(issueWatchdogs.status, "active"), ...(companyId ? [eq(issueWatchdogs.companyId, companyId)] : []), + watchdogId ? eq(issueWatchdogs.id, watchdogId) : undefined, )); } @@ -1746,14 +1767,16 @@ export function taskWatchdogService(db: Db, deps: TaskWatchdogServiceDeps = {}) issueId: string, actor: ActorFields = {}, ): Promise => { - await assertWatchedIssue(db, companyId, issueId); - const existing = await db + return db.transaction(async tx => { + await assertWatchedIssue(tx, companyId, issueId); + const existing = await tx .select() .from(issueWatchdogs) .where(and(eq(issueWatchdogs.companyId, companyId), eq(issueWatchdogs.issueId, issueId))) .then((rows) => rows[0] ?? null); if (!existing || existing.status === "disabled") return null; - const [updated] = await db + await notifyIssueRecovery(tx, { companyId }); + const [updated] = await tx .update(issueWatchdogs) .set({ status: "disabled", @@ -1765,14 +1788,17 @@ export function taskWatchdogService(db: Db, deps: TaskWatchdogServiceDeps = {}) .where(eq(issueWatchdogs.id, existing.id)) .returning(); return toIssueWatchdog(updated); + }); }, reconcileTaskWatchdogs: async (opts: { companyId?: string | null; runId?: string | null; + watchdogId?: string; + onDeferredDeadline?: (at: number) => void; issueCreatedAtGte?: Date | null; } = {}) => { - let rows = await listActiveWatchdogsForCompany(opts.companyId ?? null); + let rows = await listActiveWatchdogsForCompany(opts.companyId ?? null, opts.watchdogId); if (opts.issueCreatedAtGte) { const watchdogIssueIds = [...new Set(rows.map((row) => row.issueId))]; const eligibleIssueIds = new Set( @@ -1802,7 +1828,7 @@ export function taskWatchdogService(db: Db, deps: TaskWatchdogServiceDeps = {}) }; for (const row of rows) { result.checked += 1; - const evaluated = await evaluateWatchdog(row, { runId: opts.runId ?? null }); + const evaluated = await evaluateWatchdog(row, { runId: opts.runId ?? null, onDeferredDeadline: opts.onDeferredDeadline }); if (evaluated.state === "triggered") { result.triggered += 1; result.watchdogIssueIds.push(evaluated.watchdogIssueId); From 1b12f8f732bd1c6d848797c145b1768f5b76e3a1 Mon Sep 17 00:00:00 2001 From: Dotta Date: Fri, 30 Oct 2026 16:54:51 -0500 Subject: [PATCH 13/70] test(skills): verify Cloud CLI reads through shared object storage Use the real S3 provider with a controlled HTTP fixture and a stable test key. Preserve read-only/no-seeding behavior and verify corrupt, missing, and restored object outcomes under managed Cloud policy. Co-Authored-By: Paperclip --- .../__tests__/managed-skill-content.test.ts | 90 ++++++++++++++++--- 1 file changed, 77 insertions(+), 13 deletions(-) diff --git a/server/src/__tests__/managed-skill-content.test.ts b/server/src/__tests__/managed-skill-content.test.ts index cd4ddb8135f..59566d743e3 100644 --- a/server/src/__tests__/managed-skill-content.test.ts +++ b/server/src/__tests__/managed-skill-content.test.ts @@ -1,3 +1,5 @@ +import { createServer } from "node:http"; +import { createS3StorageProvider } from "../storage/s3-provider.js"; import { skillInstallationMaintenance } from "../services/skill-installation-maintenance.js"; import { removeRuntimeAssetTree } from "../services/native-runtime/runtime-asset-retention.js"; import { eq } from "drizzle-orm"; @@ -79,26 +81,88 @@ describe.skipIf(!support.supported)("managed skills use committed versions", () it("loads persisted Cloud configuration for read-only maintenance without seeding it", async () => { const companyId = await company(); - await companySkillService(db).createLocalSkill(companyId, { name: "Cloud CLI", markdown: "# Cloud CLI" }); + const objects = new Map(); + const server = createServer((request, response) => { + void (async () => { + const key = new URL(request.url!, "http://fixture").pathname; + if (request.method === "PUT") { + const chunks: Buffer[] = []; + for await (const chunk of request) chunks.push(Buffer.from(chunk)); + let body = Buffer.concat(chunks); + if (request.headers["content-encoding"]?.includes("aws-chunked")) { + const decoded: Buffer[] = []; + let offset = 0; + while (offset < body.length) { + const end = body.indexOf("\r\n", offset); + const size = Number.parseInt(body.subarray(offset, end).toString().split(";")[0]!, 16); + if (size === 0) break; + decoded.push(body.subarray(end + 2, end + 2 + size)); offset = end + 2 + size + 2; + } + body = Buffer.concat(decoded); + } + objects.set(key, body); response.writeHead(200, { etag: '"fixture"' }).end(); return; + } + const body = objects.get(key); + if (!body) { response.writeHead(404).end(); return; } + response.writeHead(200, { "content-length": body.length, "content-type": "application/octet-stream", etag: '"fixture"' }); + response.end(request.method === "HEAD" ? undefined : body); + })().catch(() => { response.writeHead(500).end(); }); + }); + await new Promise(resolve => { server.listen(0, "127.0.0.1", resolve); }); + const address = server.address(); if (!address || typeof address === "string") throw new Error("Expected fixture address"); + const endpoint = `http://127.0.0.1:${address.port}`; + const credentials = { AWS_ACCESS_KEY_ID: "fixture-access-key", AWS_SECRET_ACCESS_KEY: "fixture-secret-key", AWS_SESSION_TOKEN: "" }; + const previousCredentials = Object.fromEntries(Object.keys(credentials).map(key => [key, process.env[key]])); + Object.assign(process.env, credentials); + vi.mocked(storageModule.getStorageService).mockReturnValue(createStorageService(createS3StorageProvider({ + bucket: "managed-skill-fixture", region: "us-east-1", endpoint, forcePathStyle: true, + }))); const key = "cloud-runtime-config:v1"; - const run = (phase: string) => promisify(execFile)(process.execPath, [ - "--import", fileURLToPath(import.meta.resolve("tsx")), - fileURLToPath(new URL("../../scripts/migrate-skill-packages.ts", import.meta.url)), - `--phase=${phase}`, `--company=${companyId}`, "--limit=50", - ], { env: { ...process.env, DATABASE_URL: database.connectionString, - PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN: "disposable-cloud-cli-fixture", - PAPERCLIP_STORAGE_PROVIDER: "local_disk", PAPERCLIP_STORAGE_LOCAL_DIR: path.join(root, "objects") }, - timeout: 60_000, maxBuffer: 1024 * 1024 }); - await expect(run("eligibility")).rejects.toThrow("Initialize Cloud runtime configuration"); - expect(await db.select().from(instanceSettings).where(eq(instanceSettings.singletonKey, key))).toHaveLength(0); - await db.insert(instanceSettings).values({ singletonKey: key, general: { version: 1, revision: 0, config: null } }); try { + await companySkillService(db).createLocalSkill(companyId, { name: "Cloud CLI", markdown: "# Cloud CLI" }); + const run = (phase: string) => promisify(execFile)(process.execPath, [ + "--import", fileURLToPath(import.meta.resolve("tsx")), + fileURLToPath(new URL("../../scripts/migrate-skill-packages.ts", import.meta.url)), + `--phase=${phase}`, `--company=${companyId}`, "--limit=50", + ], { env: { ...process.env, ...credentials, DATABASE_URL: database.connectionString, + PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN: "disposable-cloud-cli-fixture", + PAPERCLIP_SECRETS_MASTER_KEY: "a".repeat(64), PAPERCLIP_STORAGE_PROVIDER: "s3", + PAPERCLIP_STORAGE_S3_BUCKET: "managed-skill-fixture", PAPERCLIP_STORAGE_S3_REGION: "us-east-1", + PAPERCLIP_STORAGE_S3_ENDPOINT: endpoint, PAPERCLIP_STORAGE_S3_FORCE_PATH_STYLE: "true", PAPERCLIP_STORAGE_S3_PREFIX: "" }, + timeout: 60_000, maxBuffer: 1024 * 1024 }); + await expect(run("eligibility")).rejects.toThrow("Initialize Cloud runtime configuration"); + expect(await db.select().from(instanceSettings).where(eq(instanceSettings.singletonKey, key))).toHaveLength(0); + await db.insert(instanceSettings).values({ singletonKey: key, general: { version: 1, revision: 0, config: null } }); const eligibility = JSON.parse((await run("eligibility")).stdout.trim().split("\n").at(-1)!); expect(eligibility.results).toEqual([expect.objectContaining({ status: "version_backed" })]); + const before = [...objects.entries()].map(([key, body]) => [key, Buffer.from(body)] as const); const verification = JSON.parse((await run("verification")).stdout.trim().split("\n").at(-1)!); - expect(verification.results).toEqual([expect.objectContaining({ status: "local_storage" })]); + expect(verification.results).toEqual([expect.objectContaining({ status: "remote_storage" })]); + expect([...objects.entries()]).toEqual(before); + const archiveKey = [...objects.keys()].find(key => key.endsWith("/files.tar.gz"))!; + const archive = objects.get(archiveKey)!; + const corrupted = Buffer.from(archive); corrupted[0] = corrupted[0]! ^ 1; + objects.set(archiveKey, corrupted); + const corrupt = JSON.parse((await run("verification")).stdout.trim().split("\n").at(-1)!); + expect(corrupt.results).toEqual([expect.objectContaining({ status: "corrupt_package" })]); + objects.set(archiveKey, archive); + const manifestKey = [...objects.keys()].find(key => key.endsWith("/manifest.json"))!; + const manifest = objects.get(manifestKey)!; objects.delete(manifestKey); + const unavailable = JSON.parse((await run("verification")).stdout.trim().split("\n").at(-1)!); + expect(unavailable.results).toEqual([expect.objectContaining({ status: "unavailable_or_corrupt_package" })]); + objects.set(manifestKey, manifest); + const recovered = JSON.parse((await run("verification")).stdout.trim().split("\n").at(-1)!); + expect(recovered.results).toEqual([expect.objectContaining({ status: "remote_storage" })]); + expect([...objects.entries()].sort()).toEqual([...before].sort()); + expect(await db.select().from(instanceSettings).where(eq(instanceSettings.singletonKey, key))) + .toEqual([expect.objectContaining({ general: { version: 1, revision: 0, config: null } })]); } finally { await db.delete(instanceSettings).where(eq(instanceSettings.singletonKey, key)); + vi.mocked(storageModule.getStorageService).mockReturnValue(createStorageService(createLocalDiskStorageProvider(path.join(root, "objects")))); + for (const [key, value] of Object.entries(previousCredentials)) { + if (value === undefined) delete process.env[key]; else process.env[key] = value; + } + server.closeAllConnections(); await new Promise(resolve => { server.close(() => resolve()); }); } }, 180_000); From 51c554275e8786c39c186e008c5d83c0e3ba1b2a Mon Sep 17 00:00:00 2001 From: Dotta Date: Fri, 30 Oct 2026 16:55:32 -0500 Subject: [PATCH 14/70] docs: record bounded HTTP and Cloud fixture verification Co-Authored-By: Paperclip --- .../2026-10-30-merged-pr-confirmations-sleep.md | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/doc/plans/2026-10-30-merged-pr-confirmations-sleep.md b/doc/plans/2026-10-30-merged-pr-confirmations-sleep.md index 7b8699456fb..1c691c19509 100644 --- a/doc/plans/2026-10-30-merged-pr-confirmations-sleep.md +++ b/doc/plans/2026-10-30-merged-pr-confirmations-sleep.md @@ -91,9 +91,18 @@ The original head passed full-workspace typecheck and build. A later required Linux CI shard failed the unchanged managed-skill maintenance CLI fixture in `managed-skill-content.test.ts`: it returned `unavailable_or_corrupt_package` where `local_storage` was expected. Both the isolated case and the complete -unchanged 17-test file pass against fresh local PostgreSQL. The Linux cause is -not established; this is retained as an unresolved CI failure, not called a flake. -The current-head required CI result remains the completion gate. +unchanged 17-test file pass against fresh local PostgreSQL. The original Linux cause is +not established; that failure is retained and is not called a flake. + +After merging current master, the same fixture failed locally because the new +managed-Cloud policy from PR #18350 correctly refuses local disk. A separate +test-only prerequisite commit changes that fixture to the real S3 provider with +a controlled HTTP endpoint, valid fixture credentials, and a stable test key. +It verifies remote reads without seeding or mutating runtime configuration, +rejects corrupt and missing objects, and recovers after restoring canonical bytes. +The full existing 17-test file passes with these failure controls. Production +configuration validation remains unchanged. Current-head required CI remains +the completion gate. ## Existing limitations outside this migration From 9466bca1ad7100e8fded2c5935117d8405579e49 Mon Sep 17 00:00:00 2001 From: Dotta Date: Fri, 30 Oct 2026 16:54:51 -0500 Subject: [PATCH 15/70] test(skills): verify Cloud CLI reads through shared object storage Use the real S3 provider with a controlled HTTP fixture and a stable test key. Preserve read-only/no-seeding behavior and verify corrupt, missing, and restored object outcomes under managed Cloud policy. Co-Authored-By: Paperclip --- .../__tests__/managed-skill-content.test.ts | 90 ++++++++++++++++--- 1 file changed, 77 insertions(+), 13 deletions(-) diff --git a/server/src/__tests__/managed-skill-content.test.ts b/server/src/__tests__/managed-skill-content.test.ts index cd4ddb8135f..59566d743e3 100644 --- a/server/src/__tests__/managed-skill-content.test.ts +++ b/server/src/__tests__/managed-skill-content.test.ts @@ -1,3 +1,5 @@ +import { createServer } from "node:http"; +import { createS3StorageProvider } from "../storage/s3-provider.js"; import { skillInstallationMaintenance } from "../services/skill-installation-maintenance.js"; import { removeRuntimeAssetTree } from "../services/native-runtime/runtime-asset-retention.js"; import { eq } from "drizzle-orm"; @@ -79,26 +81,88 @@ describe.skipIf(!support.supported)("managed skills use committed versions", () it("loads persisted Cloud configuration for read-only maintenance without seeding it", async () => { const companyId = await company(); - await companySkillService(db).createLocalSkill(companyId, { name: "Cloud CLI", markdown: "# Cloud CLI" }); + const objects = new Map(); + const server = createServer((request, response) => { + void (async () => { + const key = new URL(request.url!, "http://fixture").pathname; + if (request.method === "PUT") { + const chunks: Buffer[] = []; + for await (const chunk of request) chunks.push(Buffer.from(chunk)); + let body = Buffer.concat(chunks); + if (request.headers["content-encoding"]?.includes("aws-chunked")) { + const decoded: Buffer[] = []; + let offset = 0; + while (offset < body.length) { + const end = body.indexOf("\r\n", offset); + const size = Number.parseInt(body.subarray(offset, end).toString().split(";")[0]!, 16); + if (size === 0) break; + decoded.push(body.subarray(end + 2, end + 2 + size)); offset = end + 2 + size + 2; + } + body = Buffer.concat(decoded); + } + objects.set(key, body); response.writeHead(200, { etag: '"fixture"' }).end(); return; + } + const body = objects.get(key); + if (!body) { response.writeHead(404).end(); return; } + response.writeHead(200, { "content-length": body.length, "content-type": "application/octet-stream", etag: '"fixture"' }); + response.end(request.method === "HEAD" ? undefined : body); + })().catch(() => { response.writeHead(500).end(); }); + }); + await new Promise(resolve => { server.listen(0, "127.0.0.1", resolve); }); + const address = server.address(); if (!address || typeof address === "string") throw new Error("Expected fixture address"); + const endpoint = `http://127.0.0.1:${address.port}`; + const credentials = { AWS_ACCESS_KEY_ID: "fixture-access-key", AWS_SECRET_ACCESS_KEY: "fixture-secret-key", AWS_SESSION_TOKEN: "" }; + const previousCredentials = Object.fromEntries(Object.keys(credentials).map(key => [key, process.env[key]])); + Object.assign(process.env, credentials); + vi.mocked(storageModule.getStorageService).mockReturnValue(createStorageService(createS3StorageProvider({ + bucket: "managed-skill-fixture", region: "us-east-1", endpoint, forcePathStyle: true, + }))); const key = "cloud-runtime-config:v1"; - const run = (phase: string) => promisify(execFile)(process.execPath, [ - "--import", fileURLToPath(import.meta.resolve("tsx")), - fileURLToPath(new URL("../../scripts/migrate-skill-packages.ts", import.meta.url)), - `--phase=${phase}`, `--company=${companyId}`, "--limit=50", - ], { env: { ...process.env, DATABASE_URL: database.connectionString, - PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN: "disposable-cloud-cli-fixture", - PAPERCLIP_STORAGE_PROVIDER: "local_disk", PAPERCLIP_STORAGE_LOCAL_DIR: path.join(root, "objects") }, - timeout: 60_000, maxBuffer: 1024 * 1024 }); - await expect(run("eligibility")).rejects.toThrow("Initialize Cloud runtime configuration"); - expect(await db.select().from(instanceSettings).where(eq(instanceSettings.singletonKey, key))).toHaveLength(0); - await db.insert(instanceSettings).values({ singletonKey: key, general: { version: 1, revision: 0, config: null } }); try { + await companySkillService(db).createLocalSkill(companyId, { name: "Cloud CLI", markdown: "# Cloud CLI" }); + const run = (phase: string) => promisify(execFile)(process.execPath, [ + "--import", fileURLToPath(import.meta.resolve("tsx")), + fileURLToPath(new URL("../../scripts/migrate-skill-packages.ts", import.meta.url)), + `--phase=${phase}`, `--company=${companyId}`, "--limit=50", + ], { env: { ...process.env, ...credentials, DATABASE_URL: database.connectionString, + PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN: "disposable-cloud-cli-fixture", + PAPERCLIP_SECRETS_MASTER_KEY: "a".repeat(64), PAPERCLIP_STORAGE_PROVIDER: "s3", + PAPERCLIP_STORAGE_S3_BUCKET: "managed-skill-fixture", PAPERCLIP_STORAGE_S3_REGION: "us-east-1", + PAPERCLIP_STORAGE_S3_ENDPOINT: endpoint, PAPERCLIP_STORAGE_S3_FORCE_PATH_STYLE: "true", PAPERCLIP_STORAGE_S3_PREFIX: "" }, + timeout: 60_000, maxBuffer: 1024 * 1024 }); + await expect(run("eligibility")).rejects.toThrow("Initialize Cloud runtime configuration"); + expect(await db.select().from(instanceSettings).where(eq(instanceSettings.singletonKey, key))).toHaveLength(0); + await db.insert(instanceSettings).values({ singletonKey: key, general: { version: 1, revision: 0, config: null } }); const eligibility = JSON.parse((await run("eligibility")).stdout.trim().split("\n").at(-1)!); expect(eligibility.results).toEqual([expect.objectContaining({ status: "version_backed" })]); + const before = [...objects.entries()].map(([key, body]) => [key, Buffer.from(body)] as const); const verification = JSON.parse((await run("verification")).stdout.trim().split("\n").at(-1)!); - expect(verification.results).toEqual([expect.objectContaining({ status: "local_storage" })]); + expect(verification.results).toEqual([expect.objectContaining({ status: "remote_storage" })]); + expect([...objects.entries()]).toEqual(before); + const archiveKey = [...objects.keys()].find(key => key.endsWith("/files.tar.gz"))!; + const archive = objects.get(archiveKey)!; + const corrupted = Buffer.from(archive); corrupted[0] = corrupted[0]! ^ 1; + objects.set(archiveKey, corrupted); + const corrupt = JSON.parse((await run("verification")).stdout.trim().split("\n").at(-1)!); + expect(corrupt.results).toEqual([expect.objectContaining({ status: "corrupt_package" })]); + objects.set(archiveKey, archive); + const manifestKey = [...objects.keys()].find(key => key.endsWith("/manifest.json"))!; + const manifest = objects.get(manifestKey)!; objects.delete(manifestKey); + const unavailable = JSON.parse((await run("verification")).stdout.trim().split("\n").at(-1)!); + expect(unavailable.results).toEqual([expect.objectContaining({ status: "unavailable_or_corrupt_package" })]); + objects.set(manifestKey, manifest); + const recovered = JSON.parse((await run("verification")).stdout.trim().split("\n").at(-1)!); + expect(recovered.results).toEqual([expect.objectContaining({ status: "remote_storage" })]); + expect([...objects.entries()].sort()).toEqual([...before].sort()); + expect(await db.select().from(instanceSettings).where(eq(instanceSettings.singletonKey, key))) + .toEqual([expect.objectContaining({ general: { version: 1, revision: 0, config: null } })]); } finally { await db.delete(instanceSettings).where(eq(instanceSettings.singletonKey, key)); + vi.mocked(storageModule.getStorageService).mockReturnValue(createStorageService(createLocalDiskStorageProvider(path.join(root, "objects")))); + for (const [key, value] of Object.entries(previousCredentials)) { + if (value === undefined) delete process.env[key]; else process.env[key] = value; + } + server.closeAllConnections(); await new Promise(resolve => { server.close(() => resolve()); }); } }, 180_000); From bd5a104c418494ad50b39204e2875647b8e31367 Mon Sep 17 00:00:00 2001 From: Dotta Date: Fri, 30 Oct 2026 16:59:00 -0500 Subject: [PATCH 16/70] fix(agents): revalidate error-clear authority before write intent Co-Authored-By: Paperclip --- .../agent-lifecycle/adapters/records.ts | 23 ++++++++----------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/server/src/modules/agent-lifecycle/adapters/records.ts b/server/src/modules/agent-lifecycle/adapters/records.ts index db842941836..96892987b01 100644 --- a/server/src/modules/agent-lifecycle/adapters/records.ts +++ b/server/src/modules/agent-lifecycle/adapters/records.ts @@ -81,22 +81,19 @@ export function agentRecords(db: Db, effects: LifecycleEffects) { } const updated = await effects.transaction(db, existing.companyId, async tx => { + const [current] = await tx.select({ status: agents.status, lifecycleState: agents.lifecycleState }) + .from(agents).where(and(eq(agents.id, id), eq(agents.companyId, existing.companyId))).for("no key update"); + if (current?.status !== "error" || current.lifecycleState !== "ready") { + throw conflict("Only agents in error status can have their error cleared"); + } await effects.beforeStatusChange(tx, existing.companyId, id, "error", "idle"); - return tx.update(agents).set({ - status: "idle", - pauseReason: null, - pausedAt: null, - errorReason: null, - updatedAt: new Date(), - }) - .where(and(eq(agents.id, id), eq(agents.status, "error"), eq(agents.lifecycleState, "ready"))) - .returning() - .then((rows) => rows[0] ?? null); + const [saved] = await tx.update(agents).set({ + status: "idle", pauseReason: null, pausedAt: null, errorReason: null, updatedAt: new Date(), + }).where(and(eq(agents.id, id), eq(agents.status, "error"), eq(agents.lifecycleState, "ready"))).returning(); + if (!saved) throw conflict("Only agents in error status can have their error cleared"); + return saved; }); - if (!updated) { - throw conflict("Only agents in error status can have their error cleared"); - } return getById(updated.id); }, From 9e530436ac7cd1da94aa6e67b756cc2b3a6b66ba Mon Sep 17 00:00:00 2001 From: Dotta Date: Fri, 30 Oct 2026 16:59:00 -0500 Subject: [PATCH 17/70] fix(agents): revalidate error-clear authority before write intent Co-Authored-By: Paperclip --- .../agent-lifecycle/adapters/records.ts | 23 ++++++++----------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/server/src/modules/agent-lifecycle/adapters/records.ts b/server/src/modules/agent-lifecycle/adapters/records.ts index db842941836..96892987b01 100644 --- a/server/src/modules/agent-lifecycle/adapters/records.ts +++ b/server/src/modules/agent-lifecycle/adapters/records.ts @@ -81,22 +81,19 @@ export function agentRecords(db: Db, effects: LifecycleEffects) { } const updated = await effects.transaction(db, existing.companyId, async tx => { + const [current] = await tx.select({ status: agents.status, lifecycleState: agents.lifecycleState }) + .from(agents).where(and(eq(agents.id, id), eq(agents.companyId, existing.companyId))).for("no key update"); + if (current?.status !== "error" || current.lifecycleState !== "ready") { + throw conflict("Only agents in error status can have their error cleared"); + } await effects.beforeStatusChange(tx, existing.companyId, id, "error", "idle"); - return tx.update(agents).set({ - status: "idle", - pauseReason: null, - pausedAt: null, - errorReason: null, - updatedAt: new Date(), - }) - .where(and(eq(agents.id, id), eq(agents.status, "error"), eq(agents.lifecycleState, "ready"))) - .returning() - .then((rows) => rows[0] ?? null); + const [saved] = await tx.update(agents).set({ + status: "idle", pauseReason: null, pausedAt: null, errorReason: null, updatedAt: new Date(), + }).where(and(eq(agents.id, id), eq(agents.status, "error"), eq(agents.lifecycleState, "ready"))).returning(); + if (!saved) throw conflict("Only agents in error status can have their error cleared"); + return saved; }); - if (!updated) { - throw conflict("Only agents in error status can have their error cleared"); - } return getById(updated.id); }, From 0136ff7708d775ff7d9600dbf72cf4c7d263516b Mon Sep 17 00:00:00 2001 From: Dotta Date: Fri, 30 Oct 2026 16:54:51 -0500 Subject: [PATCH 18/70] test(skills): verify Cloud CLI reads through shared object storage Use the real S3 provider with a controlled HTTP fixture and a stable test key. Preserve read-only/no-seeding behavior and verify corrupt, missing, and restored object outcomes under managed Cloud policy. Co-Authored-By: Paperclip --- .../__tests__/managed-skill-content.test.ts | 90 ++++++++++++++++--- 1 file changed, 77 insertions(+), 13 deletions(-) diff --git a/server/src/__tests__/managed-skill-content.test.ts b/server/src/__tests__/managed-skill-content.test.ts index cd4ddb8135f..59566d743e3 100644 --- a/server/src/__tests__/managed-skill-content.test.ts +++ b/server/src/__tests__/managed-skill-content.test.ts @@ -1,3 +1,5 @@ +import { createServer } from "node:http"; +import { createS3StorageProvider } from "../storage/s3-provider.js"; import { skillInstallationMaintenance } from "../services/skill-installation-maintenance.js"; import { removeRuntimeAssetTree } from "../services/native-runtime/runtime-asset-retention.js"; import { eq } from "drizzle-orm"; @@ -79,26 +81,88 @@ describe.skipIf(!support.supported)("managed skills use committed versions", () it("loads persisted Cloud configuration for read-only maintenance without seeding it", async () => { const companyId = await company(); - await companySkillService(db).createLocalSkill(companyId, { name: "Cloud CLI", markdown: "# Cloud CLI" }); + const objects = new Map(); + const server = createServer((request, response) => { + void (async () => { + const key = new URL(request.url!, "http://fixture").pathname; + if (request.method === "PUT") { + const chunks: Buffer[] = []; + for await (const chunk of request) chunks.push(Buffer.from(chunk)); + let body = Buffer.concat(chunks); + if (request.headers["content-encoding"]?.includes("aws-chunked")) { + const decoded: Buffer[] = []; + let offset = 0; + while (offset < body.length) { + const end = body.indexOf("\r\n", offset); + const size = Number.parseInt(body.subarray(offset, end).toString().split(";")[0]!, 16); + if (size === 0) break; + decoded.push(body.subarray(end + 2, end + 2 + size)); offset = end + 2 + size + 2; + } + body = Buffer.concat(decoded); + } + objects.set(key, body); response.writeHead(200, { etag: '"fixture"' }).end(); return; + } + const body = objects.get(key); + if (!body) { response.writeHead(404).end(); return; } + response.writeHead(200, { "content-length": body.length, "content-type": "application/octet-stream", etag: '"fixture"' }); + response.end(request.method === "HEAD" ? undefined : body); + })().catch(() => { response.writeHead(500).end(); }); + }); + await new Promise(resolve => { server.listen(0, "127.0.0.1", resolve); }); + const address = server.address(); if (!address || typeof address === "string") throw new Error("Expected fixture address"); + const endpoint = `http://127.0.0.1:${address.port}`; + const credentials = { AWS_ACCESS_KEY_ID: "fixture-access-key", AWS_SECRET_ACCESS_KEY: "fixture-secret-key", AWS_SESSION_TOKEN: "" }; + const previousCredentials = Object.fromEntries(Object.keys(credentials).map(key => [key, process.env[key]])); + Object.assign(process.env, credentials); + vi.mocked(storageModule.getStorageService).mockReturnValue(createStorageService(createS3StorageProvider({ + bucket: "managed-skill-fixture", region: "us-east-1", endpoint, forcePathStyle: true, + }))); const key = "cloud-runtime-config:v1"; - const run = (phase: string) => promisify(execFile)(process.execPath, [ - "--import", fileURLToPath(import.meta.resolve("tsx")), - fileURLToPath(new URL("../../scripts/migrate-skill-packages.ts", import.meta.url)), - `--phase=${phase}`, `--company=${companyId}`, "--limit=50", - ], { env: { ...process.env, DATABASE_URL: database.connectionString, - PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN: "disposable-cloud-cli-fixture", - PAPERCLIP_STORAGE_PROVIDER: "local_disk", PAPERCLIP_STORAGE_LOCAL_DIR: path.join(root, "objects") }, - timeout: 60_000, maxBuffer: 1024 * 1024 }); - await expect(run("eligibility")).rejects.toThrow("Initialize Cloud runtime configuration"); - expect(await db.select().from(instanceSettings).where(eq(instanceSettings.singletonKey, key))).toHaveLength(0); - await db.insert(instanceSettings).values({ singletonKey: key, general: { version: 1, revision: 0, config: null } }); try { + await companySkillService(db).createLocalSkill(companyId, { name: "Cloud CLI", markdown: "# Cloud CLI" }); + const run = (phase: string) => promisify(execFile)(process.execPath, [ + "--import", fileURLToPath(import.meta.resolve("tsx")), + fileURLToPath(new URL("../../scripts/migrate-skill-packages.ts", import.meta.url)), + `--phase=${phase}`, `--company=${companyId}`, "--limit=50", + ], { env: { ...process.env, ...credentials, DATABASE_URL: database.connectionString, + PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN: "disposable-cloud-cli-fixture", + PAPERCLIP_SECRETS_MASTER_KEY: "a".repeat(64), PAPERCLIP_STORAGE_PROVIDER: "s3", + PAPERCLIP_STORAGE_S3_BUCKET: "managed-skill-fixture", PAPERCLIP_STORAGE_S3_REGION: "us-east-1", + PAPERCLIP_STORAGE_S3_ENDPOINT: endpoint, PAPERCLIP_STORAGE_S3_FORCE_PATH_STYLE: "true", PAPERCLIP_STORAGE_S3_PREFIX: "" }, + timeout: 60_000, maxBuffer: 1024 * 1024 }); + await expect(run("eligibility")).rejects.toThrow("Initialize Cloud runtime configuration"); + expect(await db.select().from(instanceSettings).where(eq(instanceSettings.singletonKey, key))).toHaveLength(0); + await db.insert(instanceSettings).values({ singletonKey: key, general: { version: 1, revision: 0, config: null } }); const eligibility = JSON.parse((await run("eligibility")).stdout.trim().split("\n").at(-1)!); expect(eligibility.results).toEqual([expect.objectContaining({ status: "version_backed" })]); + const before = [...objects.entries()].map(([key, body]) => [key, Buffer.from(body)] as const); const verification = JSON.parse((await run("verification")).stdout.trim().split("\n").at(-1)!); - expect(verification.results).toEqual([expect.objectContaining({ status: "local_storage" })]); + expect(verification.results).toEqual([expect.objectContaining({ status: "remote_storage" })]); + expect([...objects.entries()]).toEqual(before); + const archiveKey = [...objects.keys()].find(key => key.endsWith("/files.tar.gz"))!; + const archive = objects.get(archiveKey)!; + const corrupted = Buffer.from(archive); corrupted[0] = corrupted[0]! ^ 1; + objects.set(archiveKey, corrupted); + const corrupt = JSON.parse((await run("verification")).stdout.trim().split("\n").at(-1)!); + expect(corrupt.results).toEqual([expect.objectContaining({ status: "corrupt_package" })]); + objects.set(archiveKey, archive); + const manifestKey = [...objects.keys()].find(key => key.endsWith("/manifest.json"))!; + const manifest = objects.get(manifestKey)!; objects.delete(manifestKey); + const unavailable = JSON.parse((await run("verification")).stdout.trim().split("\n").at(-1)!); + expect(unavailable.results).toEqual([expect.objectContaining({ status: "unavailable_or_corrupt_package" })]); + objects.set(manifestKey, manifest); + const recovered = JSON.parse((await run("verification")).stdout.trim().split("\n").at(-1)!); + expect(recovered.results).toEqual([expect.objectContaining({ status: "remote_storage" })]); + expect([...objects.entries()].sort()).toEqual([...before].sort()); + expect(await db.select().from(instanceSettings).where(eq(instanceSettings.singletonKey, key))) + .toEqual([expect.objectContaining({ general: { version: 1, revision: 0, config: null } })]); } finally { await db.delete(instanceSettings).where(eq(instanceSettings.singletonKey, key)); + vi.mocked(storageModule.getStorageService).mockReturnValue(createStorageService(createLocalDiskStorageProvider(path.join(root, "objects")))); + for (const [key, value] of Object.entries(previousCredentials)) { + if (value === undefined) delete process.env[key]; else process.env[key] = value; + } + server.closeAllConnections(); await new Promise(resolve => { server.close(() => resolve()); }); } }, 180_000); From 2bdda5dcf2bd5bd332de4fc43cffdee1ffe3dbdf Mon Sep 17 00:00:00 2001 From: Dotta Date: Fri, 30 Oct 2026 16:10:20 -0500 Subject: [PATCH 19/70] fix: release settled deadline work holds during drain Preserve dirty generations with an in-memory resume deadline while allowing settled transaction holds to end during drain. Keep unresolved database intent fenced by the coordinator. Co-Authored-By: Paperclip --- server/src/__tests__/deadline-work.test.ts | 85 +++++++++++++++++++++- server/src/services/deadline-work.ts | 13 ++-- 2 files changed, 92 insertions(+), 6 deletions(-) diff --git a/server/src/__tests__/deadline-work.test.ts b/server/src/__tests__/deadline-work.test.ts index 7b852977345..7657a7e9d99 100644 --- a/server/src/__tests__/deadline-work.test.ts +++ b/server/src/__tests__/deadline-work.test.ts @@ -1,4 +1,5 @@ -import { installDatabaseWorkSignals } from "../../../packages/db/src/work-signals.js"; +import { idleWorkSnapshot } from "../services/task-admission.js"; +import { installDatabaseWorkSignals, signalDatabaseWork, databaseWorkPending } from "../../../packages/db/src/work-signals.js"; import { afterEach, expect, it, vi } from "vitest"; import { createDeliveryWorkCoordinator } from "../services/delivery-work-coordinator.js"; import { registerDeadlineWork } from "../services/deadline-work.js"; @@ -39,3 +40,85 @@ it.each([ expect(run).toHaveBeenCalledTimes(2); expect(onError).not.toHaveBeenCalled(); }); + +it("releases a settled first-write hold during drain and retains a probe-free resume wake", async () => { + vi.useFakeTimers(); + let draining = false; + const owner = installDatabaseWorkSignals({ + execute: vi.fn(async () => [{ status: "committed" }]), + transaction: async (callback: (tx: object) => Promise) => callback({ execute: async () => [{ xid: "1" }] }), + }); + const run = vi.fn(async () => {}), next = vi.fn(async () => null); + coordinator = createDeliveryWorkCoordinator({ owner, canRun: () => !draining, canReconcile: () => true, onError: vi.fn() }); + const worker = registerDeadlineWork(coordinator, DELIVERY_QUEUES.agentTimers, { run, next, canRun: () => !draining, retryMs: 30_000 }); + await worker.ready; + draining = true; + const before = idleWorkSnapshot().active; + await owner.transaction(async tx => { await signalDatabaseWork(tx, DELIVERY_QUEUES.agentTimers); }); + await vi.advanceTimersByTimeAsync(0); + expect(idleWorkSnapshot().active).toBe(before); + expect(coordinator.nextWakeAt()).not.toBeNull(); + await vi.advanceTimersByTimeAsync(3 * 30_000); + expect(run).toHaveBeenCalledOnce(); expect(next).toHaveBeenCalledOnce(); expect(owner.execute).not.toHaveBeenCalled(); + draining = false; + await vi.advanceTimersByTimeAsync(30_000); + expect(run).toHaveBeenCalledTimes(2); expect(coordinator.nextWakeAt()).toBeNull(); +}); + +it("retains failed work through drain and preserves a future domain deadline", async () => { + vi.useFakeTimers(); + let draining = false; + const owner = installDatabaseWorkSignals({ execute: vi.fn(async () => []) }); + const run = vi.fn(async () => { if (run.mock.calls.length === 1) { draining = true; throw new Error("retry debt"); } }); + const deadline = Date.now() + 600_000; + const next = vi.fn(async () => deadline); + const onError = vi.fn(); + coordinator = createDeliveryWorkCoordinator({ owner, canRun: () => !draining, canReconcile: () => true, onError }); + const worker = registerDeadlineWork(coordinator, DELIVERY_QUEUES.agentTimers, { run, next, canRun: () => !draining, retryMs: 30_000 }); + await worker.ready; + await vi.advanceTimersByTimeAsync(90_000); + expect(run).toHaveBeenCalledOnce(); expect(next).not.toHaveBeenCalled(); expect(owner.execute).not.toHaveBeenCalled(); + draining = false; await vi.advanceTimersByTimeAsync(30_000); + expect(run).toHaveBeenCalledTimes(2); expect(coordinator.nextWakeAt()).toBe(deadline); + draining = true; await vi.advanceTimersByTimeAsync(30_000); + expect(coordinator.nextWakeAt()).toBe(deadline); expect(onError).toHaveBeenCalledOnce(); +}); + +it("does no domain probes in warm standby and reconstructs after admission opens", async () => { + vi.useFakeTimers(); + let standby = true; + const owner = installDatabaseWorkSignals({ execute: vi.fn(async () => []) }); + const run = vi.fn(async () => {}), next = vi.fn(async () => null); + coordinator = createDeliveryWorkCoordinator({ owner, canRun: () => !standby, canReconcile: () => !standby, onError: vi.fn() }); + const worker = registerDeadlineWork(coordinator, DELIVERY_QUEUES.agentTimers, { run, next, canRun: () => !standby, retryMs: 30_000 }); + await worker.ready; await vi.advanceTimersByTimeAsync(90_000); + expect(run).not.toHaveBeenCalled(); expect(next).not.toHaveBeenCalled(); expect(owner.execute).not.toHaveBeenCalled(); + standby = false; await vi.advanceTimersByTimeAsync(30_000); + expect(run).toHaveBeenCalledOnce(); expect(coordinator.nextWakeAt()).toBeNull(); +}); + +it("does not release an ambiguous database intent while dispatch is drained", async () => { + vi.useFakeTimers(); + let draining = false, committed = false; + const owner = installDatabaseWorkSignals({ + execute: vi.fn(async () => [{ status: committed ? "committed" : "in progress" }]), + transaction: async (callback: (tx: object) => Promise) => { + await callback({ execute: async () => [{ xid: "2" }] }); + throw new Error("lost acknowledgement"); + }, + }); + coordinator = createDeliveryWorkCoordinator({ owner, canRun: () => !draining, canReconcile: () => true, onError: vi.fn() }); + const run = vi.fn(async () => {}); + const worker = registerDeadlineWork(coordinator, DELIVERY_QUEUES.agentTimers, { run, next: async () => null, canRun: () => !draining, retryMs: 30_000 }); + await worker.ready; draining = true; + const before = idleWorkSnapshot().active; + await expect(owner.transaction(async tx => { await signalDatabaseWork(tx, DELIVERY_QUEUES.agentTimers); })).rejects.toThrow("lost acknowledgement"); + await vi.advanceTimersByTimeAsync(0); + expect(idleWorkSnapshot().active).toBe(before + 1); + expect(databaseWorkPending(owner, DELIVERY_QUEUES.agentTimers)).toBe(true); + committed = true; await vi.advanceTimersByTimeAsync(30_000); + expect(databaseWorkPending(owner, DELIVERY_QUEUES.agentTimers)).toBe(false); + expect(idleWorkSnapshot().active).toBe(before); + draining = false; await vi.advanceTimersByTimeAsync(30_000); + expect(run).toHaveBeenCalledTimes(2); expect(coordinator.nextWakeAt()).toBeNull(); +}); diff --git a/server/src/services/deadline-work.ts b/server/src/services/deadline-work.ts index 306f340d9fc..15ba76abdae 100644 --- a/server/src/services/deadline-work.ts +++ b/server/src/services/deadline-work.ts @@ -26,7 +26,7 @@ export function registerDeadlineWork( pausePassiveMaintenance: true, onSettled: hints => { requested++; task.onSettled?.(hints); }, run: async signal => { - if (!task.canRun()) { requested++; next = null; return; } + if (!task.canRun()) { requested++; return; } if (Date.now() < earliest) { next = earliest; return; } if (Date.now() < coalesceUntil && (next === null || next > Date.now())) { next = next === null ? coalesceUntil : Math.min(next, coalesceUntil); @@ -39,8 +39,6 @@ export function registerDeadlineWork( if (signal.aborted || !task.canRun()) { next = null; return; } const deadline = await task.next(); - - if (!task.canRun()) { next = null; return; } earliest = deadline === null || deadline > Date.now() ? (task.minimumSpacingMs ? Date.now() + task.minimumSpacingMs : 0) @@ -49,8 +47,13 @@ export function registerDeadlineWork( coalesceUntil = Date.now() + (task.notificationCoalesceMs ?? 0); reconciled = generation; }, - hasPending: async () => requested !== reconciled, - nextRunAt: () => next, + // A settled producer must release its admission hold during drain. Keep + // the unreconciled generation and an in-memory resume wake, without asking + // the domain to probe its database while dispatch is closed. + hasPending: async () => task.canRun() && requested !== reconciled, + nextRunAt: () => !task.canRun() && requested !== reconciled + ? Math.min(next ?? Infinity, Date.now() + task.retryMs) + : next, }); return { ...worker, wake: () => { requested++; worker.wake(); } }; } From b1104c6dd7d2ae7ec4ce6a1ab23a652bd2fb2988 Mon Sep 17 00:00:00 2001 From: Dotta Date: Fri, 30 Oct 2026 17:04:31 -0500 Subject: [PATCH 20/70] fix(server): scope run retention hints to affected companies Co-Authored-By: Paperclip --- server/src/__tests__/execution-control-work.test.ts | 4 ++-- server/src/services/execution-control-notifications.ts | 9 ++++++--- .../src/services/execution-control-reconciliation.ts | 2 +- server/src/services/execution-recovery-resolution.ts | 4 ++-- server/src/services/explicit-native-continuation.ts | 2 +- server/src/services/heartbeat.ts | 2 +- server/src/services/heartbeat/run-lifecycle.ts | 2 +- server/src/services/heartbeat/runtime-execution.ts | 4 ++-- server/src/services/issue-recovery-actions.ts | 10 +++++----- server/src/services/legacy-execution-recovery.ts | 2 +- .../native-runtime/native-failed-workspace-recovery.ts | 2 +- .../services/native-runtime/native-restart-recovery.ts | 2 +- .../services/native-runtime/native-run-finalizer.ts | 4 ++-- .../services/native-runtime/native-run-projection.ts | 2 +- .../services/native-runtime/native-safe-replacement.ts | 4 ++-- .../services/native-runtime/native-session-executor.ts | 6 +++--- .../services/native-runtime/native-superseded-retry.ts | 2 +- server/src/services/system-recovery.ts | 2 +- server/src/services/task-retry.ts | 2 +- 19 files changed, 35 insertions(+), 32 deletions(-) diff --git a/server/src/__tests__/execution-control-work.test.ts b/server/src/__tests__/execution-control-work.test.ts index 0046d8574aa..14cd9456eb5 100644 --- a/server/src/__tests__/execution-control-work.test.ts +++ b/server/src/__tests__/execution-control-work.test.ts @@ -383,11 +383,11 @@ describe.skipIf(!support.supported)("execution-control durable work", () => { const due = new Date(Date.now() + 120_000), earlier = new Date(Date.now() + 60_000); for (const at of [due, earlier]) { await writeExecutionControl(db, DELIVERY_QUEUES.executionFinalization, tx => tx.update(heartbeatRuns) - .set({ executionControlDeadlineAt: at }).where(eq(heartbeatRuns.id, run.id))); + .set({ executionControlDeadlineAt: at }).where(eq(heartbeatRuns.id, run.id)), { companyId }); await settle(); expect(c.nextWakeAt()).toBe(at.getTime()); } await writeExecutionControl(db, DELIVERY_QUEUES.executionFinalization, tx => tx.update(heartbeatRuns) - .set({ executionControlDeadlineAt: null }).where(eq(heartbeatRuns.id, run.id))); + .set({ executionControlDeadlineAt: null }).where(eq(heartbeatRuns.id, run.id)), { companyId }); await quiet(c); }); it("delivers a reconciled continuation when marking and resolving commit together", async () => { diff --git a/server/src/services/execution-control-notifications.ts b/server/src/services/execution-control-notifications.ts index a96ca761ec2..40393216e2e 100644 --- a/server/src/services/execution-control-notifications.ts +++ b/server/src/services/execution-control-notifications.ts @@ -2,8 +2,10 @@ import { notifyAttentionRetentionWork } from "./retention-work-notifications.js" import type { Db } from "@paperclipai/db"; import { DELIVERY_QUEUES, notifyDeliveryWork, type DeliveryQueue } from "./delivery-work-notifications.js"; -export async function notifyExecutionRunWork(transaction: object) { - await notifyAttentionRetentionWork(transaction); +export type ExecutionWorkScope = { companyId: string; agentId?: string; runId?: string }; + +export async function notifyExecutionRunWork(transaction: object, scope?: ExecutionWorkScope) { + if (scope) await notifyAttentionRetentionWork(transaction, scope); for (const queue of [DELIVERY_QUEUES.executionStatus, DELIVERY_QUEUES.executionFinalization, DELIVERY_QUEUES.executionReplacement, DELIVERY_QUEUES.executionDisposition, DELIVERY_QUEUES.issueMonitors]) { await notifyDeliveryWork(transaction, queue); @@ -14,9 +16,10 @@ export function writeExecutionControl( db: Db | Parameters[0]>[0], queue: DeliveryQueue | readonly DeliveryQueue[], write: (tx: Db) => PromiseLike, + scope: ExecutionWorkScope | undefined, ): Promise { return db.transaction(async transaction => { - await notifyAttentionRetentionWork(transaction); + if (scope) await notifyAttentionRetentionWork(transaction, scope); await notifyDeliveryWork(transaction, DELIVERY_QUEUES.issueMonitors); for (const topic of typeof queue === "string" ? [queue] : queue) await notifyDeliveryWork(transaction, topic); return write(transaction as unknown as Db); diff --git a/server/src/services/execution-control-reconciliation.ts b/server/src/services/execution-control-reconciliation.ts index 2f2a435982d..621fd210cfc 100644 --- a/server/src/services/execution-control-reconciliation.ts +++ b/server/src/services/execution-control-reconciliation.ts @@ -196,7 +196,7 @@ export async function reconcileAbandonedExecutionControl( await notifyNativeEnvironmentSettlement(tx, { companyId: run.companyId, runId: run.id }); await notifyCancelledWorkspacePreservation(tx, { companyId: run.companyId, runId: run.id }); await notifyUnsafeWorkspaceExportRunChange(tx, run); - await notifyExecutionRunWork(tx); + await notifyExecutionRunWork(tx, { companyId: run.companyId }); const [updatedRun] = await tx .update(heartbeatRuns) .set({ diff --git a/server/src/services/execution-recovery-resolution.ts b/server/src/services/execution-recovery-resolution.ts index f3f5a3e4310..d56e7d926f1 100644 --- a/server/src/services/execution-recovery-resolution.ts +++ b/server/src/services/execution-recovery-resolution.ts @@ -546,7 +546,7 @@ export async function settleUnrecoverableExecutions( companyId: run.companyId, issueId: task.id, runId: run.id, }).limit(1).for("update"); if (!hold) return; - await notifyExecutionRunWork(tx); + await notifyExecutionRunWork(tx, { companyId: run.companyId }); const note = "Exact session recovery stopped. The recorded workspace recovery holds this execution without replaying its actions."; await tx.update(issueRecoveryActions).set({ status: "cancelled", outcome: "cancelled", resolvedAt: now, updatedAt: now, wakePolicy: null, monitorPolicy: null, @@ -654,7 +654,7 @@ export async function settleUnrecoverableExecutions( replay: "not_authorized", }, }); - await notifyExecutionRunWork(tx); + await notifyExecutionRunWork(tx, { companyId: run.companyId }); await tx .update(heartbeatRuns) .set({ executionStatusDeliveryId: randomUUID() }) diff --git a/server/src/services/explicit-native-continuation.ts b/server/src/services/explicit-native-continuation.ts index 074ffc3bdf5..084a25e457f 100644 --- a/server/src/services/explicit-native-continuation.ts +++ b/server/src/services/explicit-native-continuation.ts @@ -475,7 +475,7 @@ export async function admitExplicitNativeContinuation(input: { }).where(and(eq(agentWakeupRequests.companyId, companyId), eq(agentWakeupRequests.id, input.queuedCommentInterruptId), eq(agentWakeupRequests.agentId, agentId), eq(agentWakeupRequests.status, "deferred_issue_execution"))); if (cancelledStartupIds.size) await db.transaction(async tx => { - await notifyExecutionRunWork(tx); + await notifyExecutionRunWork(tx, { companyId, agentId }); for (const runId of cancelledStartupIds) { await tx.update(nativeRunFinalizations).set({ phase: "terminal_failure", failureCode: "native_startup_cancelled", nextAttemptAt: null, diff --git a/server/src/services/heartbeat.ts b/server/src/services/heartbeat.ts index eb655fdef25..a4cec7b81b0 100644 --- a/server/src/services/heartbeat.ts +++ b/server/src/services/heartbeat.ts @@ -3724,7 +3724,7 @@ export function heartbeatService( await writeExecutionControl(db, DELIVERY_QUEUES.executionFinalization, tx => tx .update(heartbeatRuns) .set({ executionControlDeadlineAt: null }) - .where(eq(heartbeatRuns.id, run.id))); + .where(eq(heartbeatRuns.id, run.id)), { companyId: run.companyId }); } nativeOwnershipHeld = nativeOwnershipHeld || diff --git a/server/src/services/heartbeat/run-lifecycle.ts b/server/src/services/heartbeat/run-lifecycle.ts index 00a27c1faf2..9b9eaad3061 100644 --- a/server/src/services/heartbeat/run-lifecycle.ts +++ b/server/src/services/heartbeat/run-lifecycle.ts @@ -1044,7 +1044,7 @@ export function createHeartbeatLifecycle(db: Db, dependencies: HeartbeatLifecycl if (previous && isHeartbeatRunTerminalStatus(status)) await notifyRetainedNativeSessionCleanup(tx, { companyId: previous.companyId, runId: previous.id }); if (isHeartbeatRunTerminalStatus(status)) await notifyDeliveryWork(tx, DELIVERY_QUEUES.accounting, companyId); - await notifyExecutionRunWork(tx); + await notifyExecutionRunWork(tx, companyId ? { companyId } : undefined); if (previous) await notifyChatRunMilestones(tx, { companyId: previous.companyId, runId: previous.id, issueId: previous.nativeIssueId ?? previous.issueId ?? readNonEmptyString(parseObject(previous.contextSnapshot).issueId) ?? undefined }); return write(tx); diff --git a/server/src/services/heartbeat/runtime-execution.ts b/server/src/services/heartbeat/runtime-execution.ts index b5b64abc948..bdbbfa07752 100644 --- a/server/src/services/heartbeat/runtime-execution.ts +++ b/server/src/services/heartbeat/runtime-execution.ts @@ -966,7 +966,7 @@ export async function executeHeartbeatRuntime(db: Db, input: HeartbeatRuntimeExe eq(heartbeatRuns.id, run.id), eq(heartbeatRuns.status, "running"), ), - )); + ), { companyId: run.companyId }); const workspaceFinalizeStatus = hasWorkspaceRestoreFailure(adapterResult.resultJson) ? "failed" : "succeeded"; await recordWorkspaceFinalize(workspaceFinalizeStatus); if (adapterResult.nativeFinalization) { @@ -1042,7 +1042,7 @@ export async function executeHeartbeatRuntime(db: Db, input: HeartbeatRuntimeExe eq(heartbeatRuns.id, run.id), eq(heartbeatRuns.status, "running"), ), - )); + ), { companyId: run.companyId }); if ( issueRef && context.resumeSessionGoalHeartbeat === true && diff --git a/server/src/services/issue-recovery-actions.ts b/server/src/services/issue-recovery-actions.ts index 355e0bc4399..a1ccb8e8e7a 100644 --- a/server/src/services/issue-recovery-actions.ts +++ b/server/src/services/issue-recovery-actions.ts @@ -394,7 +394,7 @@ export function issueRecoveryActionService(db: DbOrTransaction) { const [updated] = await writeExecutionControl(db, [DELIVERY_QUEUES.executionDisposition, DELIVERY_QUEUES.executionContinuation], tx => tx.update(issueRecoveryActions).set({ nextAction: input.nextAction, updatedAt: now }) .where(and(eq(issueRecoveryActions.id, existing.id), inArray(issueRecoveryActions.status, [...ACTIVE_RECOVERY_ACTION_STATUSES]), - sql`${issueRecoveryActions.evidence}->'workspaceOwnerStop' is null`)).returning()); + sql`${issueRecoveryActions.evidence}->'workspaceOwnerStop' is null`)).returning(), { companyId: input.companyId }); if (updated) return toReadModel(updated); } return existing; @@ -469,7 +469,7 @@ export function issueRecoveryActionService(db: DbOrTransaction) { ), ) .returning(); - }); + }, { companyId: input.companyId }); if (!exhausted) { return retryUpsertSourceScoped(input, retryCount); } @@ -541,7 +541,7 @@ export function issueRecoveryActionService(db: DbOrTransaction) { ), ) .returning(); - }); + }, { companyId: input.companyId }); if (!updated) { return retryUpsertSourceScoped(input, retryCount); } @@ -556,7 +556,7 @@ export function issueRecoveryActionService(db: DbOrTransaction) { .insert(issueRecoveryActions) .values(buildInsertValues(input, ownerType, now)) .returning(); - }); + }, { companyId: input.companyId }); return toReadModel(created!); } catch (error) { if (!isUniqueRecoveryActionConflict(error)) throw error; @@ -610,7 +610,7 @@ export function issueRecoveryActionService(db: DbOrTransaction) { }) .where(and(...predicates)) .returning(); - }); + }, { companyId: input.companyId }); return updated ? toReadModel(updated) : null; } diff --git a/server/src/services/legacy-execution-recovery.ts b/server/src/services/legacy-execution-recovery.ts index 427704de63f..c462cdf1fca 100644 --- a/server/src/services/legacy-execution-recovery.ts +++ b/server/src/services/legacy-execution-recovery.ts @@ -135,7 +135,7 @@ export async function terminalizeLegacyExecution(input: { }; if (issueId) await notifyUnsafeWorkspaceExportRecovery(tx, { companyId: current.companyId, issueId }); await notifyInstructionCollection(tx, { companyId: current.companyId, runId: current.id }); - await notifyExecutionRunWork(tx); + await notifyExecutionRunWork(tx, { companyId: current.companyId }); let [updated] = await tx .update(heartbeatRuns) .set({ diff --git a/server/src/services/native-runtime/native-failed-workspace-recovery.ts b/server/src/services/native-runtime/native-failed-workspace-recovery.ts index 5c3268f6afc..a1c6c914b33 100644 --- a/server/src/services/native-runtime/native-failed-workspace-recovery.ts +++ b/server/src/services/native-runtime/native-failed-workspace-recovery.ts @@ -65,7 +65,7 @@ export async function preserveNativeFailedWorkspace(db: Db, selected: Pick>'{systemRecovery,revision}' = ${String(previous.revision)}`)) .returning({ id: issueRecoveryActions.id }); return Boolean(updated); - }); + }, { companyId: source.companyId }); } export async function loadSystemRecovery(db: Db, source: WorkspaceRecoverySourceIdentity) { diff --git a/server/src/services/task-retry.ts b/server/src/services/task-retry.ts index e9f0bd4edbf..17075c34eb9 100644 --- a/server/src/services/task-retry.ts +++ b/server/src/services/task-retry.ts @@ -334,7 +334,7 @@ export async function requestTaskRetry(input: { details: { requestId: active?.id ?? id, sourceRunId: latest.id, attachedToExistingWork: Boolean(active || busy) } }); return { requestId: active?.id ?? id, accepted: true, stale: false, observedRunId: latest.id, chatActionId: intent.chatActionId }; - }); + }, { companyId: input.companyId }); } /** Chat owns admission and authorization. The task receipt follows that From 15cc12dba67c1f2e72a1317cc8f081e24147492a Mon Sep 17 00:00:00 2001 From: Dotta Date: Fri, 30 Oct 2026 17:06:23 -0500 Subject: [PATCH 21/70] fix(heartbeat): scope and batch dispatch wakeups by owner Co-Authored-By: Paperclip --- doc/plans/2026-10-30-run-dispatch-sleep.md | 2 +- .../heartbeat-run-dispatch-lifecycle.test.ts | 34 ++++++++++--- .../heartbeat-run-dispatch-work.test.ts | 50 ++++++++++++++++--- .../modules/run-dispatch/adapters/postgres.ts | 2 + .../modules/wake-queue/adapters/postgres.ts | 4 +- server/src/services/agent-configuration.ts | 2 +- server/src/services/agent-lifecycle.ts | 2 +- server/src/services/budgets.ts | 8 +-- .../chat-run-milestone-notifications.ts | 2 +- server/src/services/companies.ts | 4 +- .../execution-control-notifications.ts | 6 +-- server/src/services/heartbeat.ts | 4 +- .../src/services/heartbeat/dispatch-work.ts | 25 ++++++++-- server/src/services/heartbeat/queue.ts | 5 +- server/src/services/heartbeat/retries.ts | 4 +- .../src/services/heartbeat/run-lifecycle.ts | 4 +- .../issue-review-state-notifications.ts | 2 +- server/src/services/issues.ts | 14 +++--- .../native-provider-capacity-retry.ts | 2 +- server/src/services/recovery/service.ts | 4 +- .../services/run-dispatch-notifications.ts | 9 ++-- ...eduled-retry-cancellation-notifications.ts | 2 +- 22 files changed, 136 insertions(+), 55 deletions(-) diff --git a/doc/plans/2026-10-30-run-dispatch-sleep.md b/doc/plans/2026-10-30-run-dispatch-sleep.md index 82dd8692043..e5cd0a9e014 100644 --- a/doc/plans/2026-10-30-run-dispatch-sleep.md +++ b/doc/plans/2026-10-30-run-dispatch-sleep.md @@ -12,7 +12,7 @@ invalidate retries notify the retry worker in their transaction. Promotion uses the existing gate and compare-and-set, and its committed write wakes ordinary queued dispatch. A queued run behind occupied capacity waits for the owning run's lifecycle transition. Configuration and authority changes also wake it. -An idle queue has no recurring read or timer-driven dispatch attempt. +An idle queue has no recurring read or timer-driven dispatch attempt. Notifications batch for one second and carry the changed company to queued dispatch. Known retry deadlines still preempt batching. Cancellation-owned retries stay with their cancellation worker and do not retain a promotion deadline. The supported writer contract is service writes through the shared database connection scope, including caller-owned and nested transactions. Existing diff --git a/server/src/__tests__/heartbeat-run-dispatch-lifecycle.test.ts b/server/src/__tests__/heartbeat-run-dispatch-lifecycle.test.ts index 3d6566e543d..b6fbb7353de 100644 --- a/server/src/__tests__/heartbeat-run-dispatch-lifecycle.test.ts +++ b/server/src/__tests__/heartbeat-run-dispatch-lifecycle.test.ts @@ -1,3 +1,4 @@ +import { idleWorkSnapshot } from "../services/task-admission.js"; import { PgDialect } from "drizzle-orm/pg-core"; import { afterEach, describe, expect, it, vi } from "vitest"; import { installDatabaseWorkSignals } from "../../../packages/db/src/work-signals.js"; @@ -28,7 +29,7 @@ function fixture(input: { due?: number; suppressed?: boolean; draining?: boolean if (state.due !== null && state.due <= Date.now()) state.due = null; }); const next = vi.fn(async () => state.due); - const dispatch = vi.fn(async () => {}); + const dispatch = vi.fn(async (_scope?: { companyId: string }) => {}); const coordinator = createDeliveryWorkCoordinator({ owner: db, canRun: () => !state.draining && !state.standby, canReconcile: () => !state.standby, onError: vi.fn() }); const worker = registerRunDispatchWork(coordinator, { @@ -37,8 +38,8 @@ function fixture(input: { due?: number; suppressed?: boolean; draining?: boolean }, () => !state.draining && !state.standby); cleanup.push(() => coordinator.stop()); return { db, state, promote, next, dispatch, coordinator, worker, - notify: () => db.transaction(tx => notifyRunDispatch(tx)), - settle: () => vi.advanceTimersByTimeAsync(0) }; + notify: (scope?: { companyId: string }) => db.transaction(tx => notifyRunDispatch(tx, scope)), + settle: () => vi.advanceTimersByTimeAsync(1_000) }; } describe("run dispatch obligations", () => { @@ -67,20 +68,39 @@ describe("run dispatch obligations", () => { const before = f.next.mock.calls.length; await vi.advanceTimersByTimeAsync(20 * 60_000); expect(f.next).toHaveBeenCalledTimes(before); }); + it("batches repeated tenant writes and passes only their company to queued dispatch", async () => { + const f = fixture(); await f.worker.ready; + for (let index = 0; index < 100; index++) { + await f.notify({ companyId: "company-a" }); + await vi.advanceTimersByTimeAsync(10); + } + expect(f.promote.mock.calls.length).toBeLessThanOrEqual(2); + expect(f.dispatch.mock.calls.length).toBeLessThanOrEqual(2); + await f.settle(); + expect(f.dispatch).toHaveBeenLastCalledWith({ companyId: "company-a" }); + expect(f.dispatch.mock.calls.slice(1).every(([scope]) => scope?.companyId === "company-a")).toBe(true); + expect(f.coordinator.nextWakeAt()).toBeNull(); + const count = f.dispatch.mock.calls.length; + await vi.advanceTimersByTimeAsync(20 * 60_000); expect(f.dispatch).toHaveBeenCalledTimes(count); + }); it("remembers the first producer arriving during drain without owner SQL", async () => { const f = fixture(); await f.worker.ready; f.state.draining = true; await f.notify(); await f.settle(); await vi.advanceTimersByTimeAsync(90_000); expect(f.promote).toHaveBeenCalledOnce(); expect(f.next).toHaveBeenCalledOnce(); + expect(idleWorkSnapshot().active).toBe(0); f.state.draining = false; await vi.advanceTimersByTimeAsync(30_000); expect(f.promote).toHaveBeenCalledTimes(2); expect(f.dispatch).toHaveBeenCalledTimes(2); expect(f.coordinator.nextWakeAt()).toBeNull(); }); - it("retains failed work across drain and settles after recovery", async () => { - const f = fixture(); f.state.fail = true; await f.worker.ready; + it("releases a failed producer hold during drain and retains recovery for resume", async () => { + const f = fixture(); await f.worker.ready; + f.state.fail = true; await f.notify(); await f.settle(); + expect(f.promote).toHaveBeenCalledTimes(2); f.state.draining = true; await vi.advanceTimersByTimeAsync(90_000); - expect(f.promote).toHaveBeenCalledOnce(); expect(f.next).not.toHaveBeenCalled(); + expect(f.promote).toHaveBeenCalledTimes(2); expect(f.next).toHaveBeenCalledOnce(); + expect(idleWorkSnapshot().active).toBe(0); f.state.fail = false; f.state.draining = false; await vi.advanceTimersByTimeAsync(30_000); - expect(f.promote).toHaveBeenCalledTimes(2); expect(f.coordinator.nextWakeAt()).toBeNull(); + expect(f.promote).toHaveBeenCalledTimes(3); expect(f.coordinator.nextWakeAt()).toBeNull(); }); it("preserves a notification during a nonoverlapping pass", async () => { const f = fixture(); await f.worker.ready; diff --git a/server/src/__tests__/heartbeat-run-dispatch-work.test.ts b/server/src/__tests__/heartbeat-run-dispatch-work.test.ts index 4a616887479..bcc8ed23972 100644 --- a/server/src/__tests__/heartbeat-run-dispatch-work.test.ts +++ b/server/src/__tests__/heartbeat-run-dispatch-work.test.ts @@ -1,3 +1,4 @@ +import { idleWorkSnapshot } from "../services/task-admission.js"; import { randomUUID } from "node:crypto"; import { and, eq, sql } from "drizzle-orm"; import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; @@ -76,9 +77,11 @@ describe.skipIf(!support.supported)("run dispatch production wiring (controlled return db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, id)).then(rows => rows[0]); } async function completed(id: string) { - await vi.waitFor(async () => expect((await status(id))?.status).toBe("succeeded"), { timeout: 10_000 }); + await vi.waitFor(async () => expect(["running", "succeeded"]).toContain((await status(id))?.status), { timeout: 10_000 }); await heartbeat.drainActiveRunExecutions(); - expect((await status(id)).stdoutExcerpt).toContain("dispatch-proof"); + const completed = await status(id); + expect(completed.status).toBe("succeeded"); + expect(completed.stdoutExcerpt).toContain("dispatch-proof"); } it("keeps fresh empty, configured idle and retained terminal history quiet", async () => { await start(); await quiet(); @@ -87,10 +90,10 @@ describe.skipIf(!support.supported)("run dispatch production wiring (controlled }); it("ordinary retry creation wakes the worker, respects the exact deadline, dispatches and settles", async () => { const f = await fixture(); await start(); await quiet(); - const result = await heartbeat.scheduleBoundedRetry(f.runId, { delayMs: 2_000, random: () => 0 }); + const result = await heartbeat.scheduleBoundedRetry(f.runId, { delayMs: 4_000, random: () => 0 }); expect(result.outcome).toBe("scheduled"); if (result.outcome !== "scheduled") throw new Error("Expected retry creation"); - await vi.waitFor(() => expect(coordinator!.nextWakeAt()).toBe(result.dueAt.getTime())); + await vi.waitFor(() => expect(coordinator!.nextWakeAt()).toBe(result.dueAt.getTime()), { timeout: 5_000 }); expect((await status(result.run.id)).status).toBe("scheduled_retry"); await new Promise(resolve => setTimeout(resolve, 100)); expect(Date.now()).toBeLessThan(result.dueAt.getTime()); @@ -110,7 +113,7 @@ describe.skipIf(!support.supported)("run dispatch production wiring (controlled const f = await fixture(); await start(); const scheduled = await heartbeat.scheduleBoundedRetry(f.runId, { delayMs: 600_000 }); if (scheduled.outcome !== "scheduled") throw new Error("Expected retry creation"); - await vi.waitFor(() => expect(coordinator!.nextWakeAt()).toBe(scheduled.dueAt.getTime())); + await vi.waitFor(() => expect(coordinator!.nextWakeAt()).toBe(scheduled.dueAt.getTime()), { timeout: 5_000 }); await heartbeat.cancelRun(scheduled.run.id, "operator stop", { suppressImmediateRecovery: true }); await quiet(); expect((await status(scheduled.run.id)).status).toBe("cancelled"); }); @@ -118,7 +121,14 @@ describe.skipIf(!support.supported)("run dispatch production wiring (controlled const f = await fixture(); await start(); await quiet(); draining = true; const scheduled = await heartbeat.scheduleBoundedRetry(f.runId, { delayMs: 200 }); if (scheduled.outcome !== "scheduled") throw new Error("Expected retry creation"); - await new Promise(resolve => setTimeout(resolve, 400)); + await vi.waitFor(() => expect(idleWorkSnapshot().active).toBe(0)); + const count = reads.mock.calls.length; + vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"] }); + try { + await vi.advanceTimersByTimeAsync(90_000); + expect(reads).toHaveBeenCalledTimes(count); + expect(idleWorkSnapshot().active).toBe(0); + } finally { vi.useRealTimers(); } expect((await status(scheduled.run.id)).status).toBe("scheduled_retry"); draining = false; worker!.wake(); await completed(scheduled.run.id); await quiet(); @@ -151,13 +161,39 @@ describe.skipIf(!support.supported)("run dispatch production wiring (controlled await blocker.select().from(agents).where(eq(agents.id, f.agentId)).for("update"); writing = agentService(db).clearError(f.agentId); void writing.catch(() => {}); - await vi.waitFor(() => expect(databaseWorkPending(db, DELIVERY_QUEUES.queuedRunDispatch)).toBe(true)); + await new Promise(resolve => setTimeout(resolve, 100)); + expect(databaseWorkPending(db, DELIVERY_QUEUES.queuedRunDispatch)).toBe(false); expect((await status(queued.id)).status).toBe("queued"); expect((await db.select().from(agents).where(eq(agents.id, f.agentId)))[0].status).toBe("error"); }); } finally { await writing; } await completed(queued.id); await quiet(); }); + it("leaves cancellation-owned retries quiet even when their old promotion time is overdue", async () => { + const f = await fixture(); + const scheduled = await heartbeat.scheduleBoundedRetry(f.runId, { now: new Date(Date.now() - 60_000), delayMs: 1_000 }); + if (scheduled.outcome !== "scheduled") throw new Error("Expected retry creation"); + await db.update(heartbeatRuns).set({ resultJson: { startupCancellation: { requestedAt: new Date().toISOString() }, + cancellation: { expected: true, reason: "operator stop", source: "operator" } } }).where(eq(heartbeatRuns.id, scheduled.run.id)); + await start(); await quiet(); + expect((await status(scheduled.run.id)).status).toBe("scheduled_retry"); + expect(await heartbeat.promoteDueScheduledRetries()).toEqual({ promoted: 0, runIds: [] }); + await quiet(); + }); + it("rejects an error clear without publishing work when the lifecycle is no longer ready", async () => { + const f = await fixture(); await start(); await quiet(); + await db.update(agents).set({ status: "error", lifecycleState: "paused" }).where(eq(agents.id, f.agentId)); + vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"] }); + try { + await expect(agentService(db).clearError(f.agentId)).rejects.toThrow("Only agents in error status"); + expect(coordinator!.nextWakeAt()).toBeNull(); + const count = reads.mock.calls.length; + await vi.advanceTimersByTimeAsync(60_000); + expect(reads).toHaveBeenCalledTimes(count); + } finally { vi.useRealTimers(); } + expect((await db.select().from(agents).where(eq(agents.id, f.agentId)))[0].status).toBe("error"); + await quiet(); + }); it("keeps a foreign company's future retry intact while concurrent workers claim one due retry", async () => { const selected = await fixture(), foreign = await fixture(); await start(); const later = await heartbeat.scheduleBoundedRetry(foreign.runId, { delayMs: 600_000 }); diff --git a/server/src/modules/run-dispatch/adapters/postgres.ts b/server/src/modules/run-dispatch/adapters/postgres.ts index 63559e7701d..b05f94e1a4b 100644 --- a/server/src/modules/run-dispatch/adapters/postgres.ts +++ b/server/src/modules/run-dispatch/adapters/postgres.ts @@ -514,6 +514,7 @@ export function createPostgresRunDispatchAdapter( async function nextScheduledRetryAt(input: { cutoff: Date | null }): Promise { const [row] = await db.select({ at: heartbeatRuns.scheduledRetryAt }).from(heartbeatRuns) .where(and(eq(heartbeatRuns.status, "scheduled_retry"), + sql`jsonb_typeof(${heartbeatRuns.resultJson}->'startupCancellation') is distinct from 'object'`, input.cutoff ? gte(heartbeatRuns.createdAt, input.cutoff) : undefined)) .orderBy(asc(heartbeatRuns.scheduledRetryAt)).limit(1); return row?.at?.getTime() ?? null; @@ -526,6 +527,7 @@ export function createPostgresRunDispatchAdapter( .where( and( eq(heartbeatRuns.status, "scheduled_retry"), + sql`jsonb_typeof(${heartbeatRuns.resultJson}->'startupCancellation') is distinct from 'object'`, lte(heartbeatRuns.scheduledRetryAt, input.now), input.cutoff ? gte(heartbeatRuns.createdAt, input.cutoff) : undefined, ), diff --git a/server/src/modules/wake-queue/adapters/postgres.ts b/server/src/modules/wake-queue/adapters/postgres.ts index c773b903dec..1df4420615f 100644 --- a/server/src/modules/wake-queue/adapters/postgres.ts +++ b/server/src/modules/wake-queue/adapters/postgres.ts @@ -638,7 +638,7 @@ function buildTransaction(tx: Db, deps: WakeQueuePostgresAdapterDeps, db: Db, ru .returning() .then((rows) => rows[0]); - await notifyRunDispatch(tx); + await notifyRunDispatch(tx, { companyId }); const queuedRun = await tx .insert(heartbeatRuns) .values({ @@ -717,7 +717,7 @@ function buildTransaction(tx: Db, deps: WakeQueuePostgresAdapterDeps, db: Db, ru .returning() .then((rows) => rows[0]); - await notifyRunDispatch(tx); + await notifyRunDispatch(tx, { companyId }); const queuedRun = await tx .insert(heartbeatRuns) .values({ diff --git a/server/src/services/agent-configuration.ts b/server/src/services/agent-configuration.ts index 88a2c636bdb..dc10efaa082 100644 --- a/server/src/services/agent-configuration.ts +++ b/server/src/services/agent-configuration.ts @@ -84,7 +84,7 @@ export async function prepareAgentConfiguration(db: Db, existing: AgentConfigura existing.adapterConfig, ); } - await notifyRunDispatch(db); + await notifyRunDispatch(db, { companyId: existing.companyId }); if (normalizedPatch.runtimeConfig !== undefined || normalizedPatch.reportsTo !== undefined) { await notifyDeliveryWork(db, DELIVERY_QUEUES.agentTimers); } diff --git a/server/src/services/agent-lifecycle.ts b/server/src/services/agent-lifecycle.ts index 700c6ba8ffa..59357fcc5de 100644 --- a/server/src/services/agent-lifecycle.ts +++ b/server/src/services/agent-lifecycle.ts @@ -40,7 +40,7 @@ export function createAgentLifecycleEffects(_hooks: BudgetServiceHooks = {}): li notifyWork: tx => notifyDeliveryWork(tx, DELIVERY_QUEUES.agentLifecycle), async beforeStatusChange(tx, companyId, agentId, before, after) { if (before !== after) { - await notifyRunDispatch(tx); + await notifyRunDispatch(tx, { companyId }); await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers); } if (!isAgentStatusInvokable(before) && isAgentStatusInvokable(after)) { diff --git a/server/src/services/budgets.ts b/server/src/services/budgets.ts index 86d00c4d108..186253b19bb 100644 --- a/server/src/services/budgets.ts +++ b/server/src/services/budgets.ts @@ -288,7 +288,7 @@ export function budgetServiceInTransaction(db: Db, publications: ActivityPublica } if (policy.scopeType === "project") { - await notifyRunDispatch(db); + await notifyRunDispatch(db, { companyId: policy.companyId }); await db .update(projects) .set({ @@ -302,7 +302,7 @@ export function budgetServiceInTransaction(db: Db, publications: ActivityPublica const scope = await resolveScopeRecord(db, "company", policy.scopeId); if (scope.paused) return; - await notifyRunDispatch(db); + await notifyRunDispatch(db, { companyId: policy.companyId }); await notifyCompanyRetentionWork(db, policy.companyId); await notifyDeliveryWork(db, DELIVERY_QUEUES.agentTimers); await notifyDeliveryWork(db, DELIVERY_QUEUES.routineSchedules); @@ -349,7 +349,7 @@ export function budgetServiceInTransaction(db: Db, publications: ActivityPublica } if (policy.scopeType === "project") { - await notifyRunDispatch(db); + await notifyRunDispatch(db, { companyId: policy.companyId }); await db .update(projects) .set({ @@ -364,7 +364,7 @@ export function budgetServiceInTransaction(db: Db, publications: ActivityPublica await notifyDeliveryWork(db, DELIVERY_QUEUES.skillSourceSync); const scope = await resolveScopeRecord(db, "company", policy.scopeId); if (scope.pauseReason !== "budget") return; - await notifyRunDispatch(db); + await notifyRunDispatch(db, { companyId: policy.companyId }); await notifyCompanyRetentionWork(db, policy.companyId); await notifyDeliveryWork(db, DELIVERY_QUEUES.agentTimers); await notifyDeliveryWork(db, DELIVERY_QUEUES.routineSchedules); diff --git a/server/src/services/chat-run-milestone-notifications.ts b/server/src/services/chat-run-milestone-notifications.ts index 159055c10d3..524936395b5 100644 --- a/server/src/services/chat-run-milestone-notifications.ts +++ b/server/src/services/chat-run-milestone-notifications.ts @@ -12,7 +12,7 @@ import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notificatio export type ChatRunMilestoneScope = { companyId: string; issueId?: string; runId?: string }; export async function notifyChatRunMilestones(tx: object, scope: ChatRunMilestoneScope) { - if (!scope.issueId && !scope.runId) await notifyRunDispatch(tx); + if (!scope.issueId && !scope.runId) await notifyRunDispatch(tx, scope); if (scope.issueId || scope.runId) { await notifyIssueReviewState(tx, { companyId: scope.companyId, issueId: scope.issueId, runId: scope.issueId ? undefined : scope.runId }); diff --git a/server/src/services/companies.ts b/server/src/services/companies.ts index cc10ffa3bb8..a9a8f59998c 100644 --- a/server/src/services/companies.ts +++ b/server/src/services/companies.ts @@ -333,7 +333,7 @@ export function companyService(db: Db, _budgetHooks: BudgetServiceHooks = {}) { const renamedPrefix = await resolveRenamedIssuePrefix(tx, id, companyPatch); if (companyPatch.status !== undefined && companyPatch.status !== existing.status) { - await notifyRunDispatch(tx); + await notifyRunDispatch(tx, { companyId: id }); await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers); await notifyDeliveryWork(tx, DELIVERY_QUEUES.skillSourceSync); await notifyDeliveryWork(tx, DELIVERY_QUEUES.routineSchedules); @@ -490,7 +490,7 @@ export function companyService(db: Db, _budgetHooks: BudgetServiceHooks = {}) { const wasAlreadyArchived = existing.status === "archived"; if (!wasAlreadyArchived) { - await notifyRunDispatch(tx); + await notifyRunDispatch(tx, { companyId: id }); await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers); await notifyDeliveryWork(tx, DELIVERY_QUEUES.skillSourceSync); await notifyDeliveryWork(tx, DELIVERY_QUEUES.routineSchedules); diff --git a/server/src/services/execution-control-notifications.ts b/server/src/services/execution-control-notifications.ts index 6fef650fe42..39b1d7765c6 100644 --- a/server/src/services/execution-control-notifications.ts +++ b/server/src/services/execution-control-notifications.ts @@ -1,9 +1,9 @@ -import { notifyRunDispatch } from "./run-dispatch-notifications.js"; +import { notifyRunDispatch, type RunDispatchScope } from "./run-dispatch-notifications.js"; import type { Db } from "@paperclipai/db"; import { DELIVERY_QUEUES, notifyDeliveryWork, type DeliveryQueue } from "./delivery-work-notifications.js"; -export async function notifyExecutionRunWork(transaction: object) { - await notifyRunDispatch(transaction); +export async function notifyExecutionRunWork(transaction: object, scope?: RunDispatchScope) { + await notifyRunDispatch(transaction, scope); for (const queue of [DELIVERY_QUEUES.executionStatus, DELIVERY_QUEUES.executionFinalization, DELIVERY_QUEUES.executionReplacement, DELIVERY_QUEUES.executionDisposition, DELIVERY_QUEUES.issueMonitors]) { await notifyDeliveryWork(transaction, queue); diff --git a/server/src/services/heartbeat.ts b/server/src/services/heartbeat.ts index 0772b920023..bbce8ba8f5b 100644 --- a/server/src/services/heartbeat.ts +++ b/server/src/services/heartbeat.ts @@ -1335,8 +1335,8 @@ export function heartbeatService( return heartbeatQueue.dispatchPendingNativeStatusWakeups(...args); } - function resumeQueuedRunsAfterDrain() { - const promise = heartbeatQueue.resumeQueuedRunsAfterDrain(); + function resumeQueuedRunsAfterDrain(...args: Parameters) { + const promise = heartbeatQueue.resumeQueuedRunsAfterDrain(...args); activeWakeupPromises.add(promise); void promise.then( () => { activeWakeupPromises.delete(promise); }, diff --git a/server/src/services/heartbeat/dispatch-work.ts b/server/src/services/heartbeat/dispatch-work.ts index 3291008dd9a..5b1f728688a 100644 --- a/server/src/services/heartbeat/dispatch-work.ts +++ b/server/src/services/heartbeat/dispatch-work.ts @@ -1,3 +1,4 @@ +import type { RunDispatchScope } from "../run-dispatch-notifications.js"; import type { createDeliveryWorkCoordinator } from "../delivery-work-coordinator.js"; import { registerDeadlineWork } from "../deadline-work.js"; import { DELIVERY_QUEUES } from "../delivery-work-notifications.js"; @@ -7,7 +8,7 @@ export function registerRunDispatchWork( owner: { promoteDueScheduledRetries(): Promise; nextScheduledRetryAt(): Promise; - resumeQueuedRunsAfterDrain(): Promise; + resumeQueuedRunsAfterDrain(scope?: RunDispatchScope): Promise; schedulingSuppressed(): Promise; }, canRun: () => boolean, @@ -16,6 +17,7 @@ export function registerRunDispatchWork( const retries = registerDeadlineWork(coordinator, DELIVERY_QUEUES.scheduledRetryDispatch, { retryMs: 30_000, continuationMs: 1_000, + notificationCoalesceMs: 1_000, canRun, run: async signal => { suppressed = await owner.schedulingSuppressed(); @@ -23,17 +25,34 @@ export function registerRunDispatchWork( }, next: () => suppressed ? Promise.resolve(null) : owner.nextScheduledRetryAt(), }); + const scopes = new Set(["*"]); const queue = registerDeadlineWork(coordinator, DELIVERY_QUEUES.queuedRunDispatch, { retryMs: 30_000, + notificationCoalesceMs: 1_000, canRun, + onSettled: hints => { for (const hint of hints ?? ["*"]) scopes.add(hint); }, run: async signal => { - if (!signal.aborted && canRun()) await owner.resumeQueuedRunsAfterDrain(); + const requested = scopes.has("*") ? ["*"] : [...scopes]; + if (requested[0] === "*") scopes.clear(); + else for (const hint of requested) scopes.delete(hint); + for (let index = 0; index < requested.length; index++) { + if (signal.aborted || !canRun()) { + for (const hint of requested.slice(index)) scopes.add(hint); + return; + } + try { + await owner.resumeQueuedRunsAfterDrain(requested[index] === "*" ? undefined : JSON.parse(requested[index]) as RunDispatchScope); + } catch (error) { + for (const hint of requested.slice(index)) scopes.add(hint); + throw error; + } + } }, next: async () => null, }); return { ready: Promise.all([retries.ready, queue.ready]), - wake() { retries.wake(); queue.wake(); }, + wake() { scopes.add("*"); retries.wake(); queue.wake(); }, async stop() { await Promise.all([retries.stop(), queue.stop()]); }, }; } diff --git a/server/src/services/heartbeat/queue.ts b/server/src/services/heartbeat/queue.ts index 376530b2f0a..64ca0b84ffd 100644 --- a/server/src/services/heartbeat/queue.ts +++ b/server/src/services/heartbeat/queue.ts @@ -1973,7 +1973,7 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende // Release admission promptly without first scanning unrelated recovery work. // Every run still passes the ordinary per-agent claim and dispatch gates. - async function resumeQueuedRunsAfterDrain() { + async function resumeQueuedRunsAfterDrain(scope?: { companyId: string }) { if ((await getSchedulingSuppression()).suppressed) return; const cutoff = await getWorktreeExecutionCutoff(); const queuedRuns = await db @@ -1983,6 +1983,7 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende .where( and( eq(heartbeatRuns.status, "queued"), + scope ? eq(heartbeatRuns.companyId, scope.companyId) : undefined, eq(companies.status, "active"), cutoff ? gte(heartbeatRuns.createdAt, cutoff) : undefined, ), @@ -4633,7 +4634,7 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende } const queueOutcome = await db.transaction(async (tx) => { - await notifyRunDispatch(tx); + await notifyRunDispatch(tx, { companyId: agent.companyId }); await tx.execute( sql`select id from agents where id = ${agentId} and company_id = ${agent.companyId} for update`, ); diff --git a/server/src/services/heartbeat/retries.ts b/server/src/services/heartbeat/retries.ts index 5f4df5ed74e..32333c0ebec 100644 --- a/server/src/services/heartbeat/retries.ts +++ b/server/src/services/heartbeat/retries.ts @@ -998,7 +998,7 @@ export function createHeartbeatRetries(db: Db, dependencies: HeartbeatRetryDepen retryContextSnapshot.previousRunId = continuation.previousRunId; } - await notifyRunDispatch(tx); + await notifyRunDispatch(tx, { companyId: run.companyId }); const wakeupRequest = await tx .insert(agentWakeupRequests) .values({ @@ -1756,7 +1756,7 @@ export function createHeartbeatRetries(db: Db, dependencies: HeartbeatRetryDepen }; const updated = await db.transaction(async (tx) => { - await notifyRunDispatch(tx); + await notifyRunDispatch(tx, { companyId: scheduled.run.companyId }); const row = await tx .update(heartbeatRuns) .set({ diff --git a/server/src/services/heartbeat/run-lifecycle.ts b/server/src/services/heartbeat/run-lifecycle.ts index 07bb54dfdf7..9d3b30367d2 100644 --- a/server/src/services/heartbeat/run-lifecycle.ts +++ b/server/src/services/heartbeat/run-lifecycle.ts @@ -1045,7 +1045,7 @@ export function createHeartbeatLifecycle(db: Db, dependencies: HeartbeatLifecycl if (previous && isHeartbeatRunTerminalStatus(status)) await notifyRetainedNativeSessionCleanup(tx, { companyId: previous.companyId, runId: previous.id }); if (isHeartbeatRunTerminalStatus(status)) await notifyDeliveryWork(tx, DELIVERY_QUEUES.accounting, companyId); - await notifyExecutionRunWork(tx); + await notifyExecutionRunWork(tx, companyId ? { companyId } : undefined); if (previous) await notifyChatRunMilestones(tx, { companyId: previous.companyId, runId: previous.id, issueId: previous.nativeIssueId ?? previous.issueId ?? readNonEmptyString(parseObject(previous.contextSnapshot).issueId) ?? undefined }); return write(tx); @@ -2448,7 +2448,7 @@ export function createHeartbeatLifecycle(db: Db, dependencies: HeartbeatLifecycl .returning() .then((rows) => rows[0]); - await notifyRunDispatch(tx); + await notifyRunDispatch(tx, { companyId: run.companyId }); const queuedRun = await tx .insert(heartbeatRuns) .values({ diff --git a/server/src/services/issue-review-state-notifications.ts b/server/src/services/issue-review-state-notifications.ts index b769be64056..d9d4d756816 100644 --- a/server/src/services/issue-review-state-notifications.ts +++ b/server/src/services/issue-review-state-notifications.ts @@ -5,7 +5,7 @@ import { notifyNativeReviewResponse } from "./native-review-response-notificatio import { notifyNativeStatusReassessment } from "./native-status-reassessment-notifications.js"; export async function notifyIssueReviewState(tx: object, scope: { companyId: string; issueId?: string; runId?: string; agentId?: string }) { - await notifyRunDispatch(tx); + await notifyRunDispatch(tx, scope); await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors); await notifyExternalConversationSettlement(tx, scope); await notifyNativeReviewResponse(tx, scope); diff --git a/server/src/services/issues.ts b/server/src/services/issues.ts index a7c8bb55023..ade48bf0cc2 100644 --- a/server/src/services/issues.ts +++ b/server/src/services/issues.ts @@ -10606,7 +10606,7 @@ export function issueService(db: Db) { : values.parentId ? `issue:${values.parentId}` : undefined); } if (values.monitorNextCheckAt) await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors); - await notifyRunDispatch(tx); + await notifyRunDispatch(tx, { companyId: values.companyId }); await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers); const [issue] = await tx.insert(issues).values(values).returning(); await attachOwnedDraftImages(tx, issue, issue.description, { userId: issueData.createdByUserId, agentId: issueData.createdByAgentId }); @@ -11495,7 +11495,7 @@ export function issueService(db: Db) { } if (patch.status !== undefined || patch.assigneeAgentId !== undefined || patch.assigneeUserId !== undefined || patch.hiddenAt !== undefined || patch.conversationAgentId !== undefined) { - await notifyRunDispatch(tx); + await notifyRunDispatch(tx, { companyId: receiptExisting.companyId }); await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers); } if (patch.monitorNextCheckAt !== undefined || (receiptExisting.monitorNextCheckAt && @@ -11960,7 +11960,7 @@ export function issueService(db: Db) { } await notifyDecisionTargetIssueRemoval(tx, id); await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors); - await notifyRunDispatch(tx); + await notifyRunDispatch(tx, owner ? { companyId: owner.companyId } : undefined); await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers); removedIssue = await tx .delete(issues) @@ -12089,7 +12089,7 @@ export function issueService(db: Db) { : isNull(issues.executionRunId); const updated = await db.transaction(async tx => { await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors); - await notifyRunDispatch(tx); + await notifyRunDispatch(tx, { companyId: issueCompany.companyId }); await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers); return tx .update(issues) @@ -12219,7 +12219,7 @@ export function issueService(db: Db) { const staleExecutionRunId = current.executionRunId; const adopted = await db.transaction(async tx => { await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors); - await notifyRunDispatch(tx); + await notifyRunDispatch(tx, { companyId: issueCompany.companyId }); await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers); return tx .update(issues) @@ -12475,7 +12475,7 @@ export function issueService(db: Db) { await notifyTerminalWorkspaceWork(tx, `issue:${id}`); await notifyNativeStatusReassessment(tx, { companyId: existing.companyId, issueId: id }); await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors); - await notifyRunDispatch(tx); + await notifyRunDispatch(tx, { companyId: existing.companyId }); await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers); const updated = await tx .update(issues) @@ -12536,7 +12536,7 @@ export function issueService(db: Db) { await notifyUnsafeWorkspaceExportRecovery(tx, { companyId: existing.companyId, issueId: id }); } await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors); - await notifyRunDispatch(tx); + await notifyRunDispatch(tx, { companyId: existing.companyId }); await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers); const updated = await tx .update(issues) diff --git a/server/src/services/native-runtime/native-provider-capacity-retry.ts b/server/src/services/native-runtime/native-provider-capacity-retry.ts index 7d007c383dd..83314ff9c51 100644 --- a/server/src/services/native-runtime/native-provider-capacity-retry.ts +++ b/server/src/services/native-runtime/native-provider-capacity-retry.ts @@ -50,7 +50,7 @@ export async function materializeNativeProviderCapacityRetry(input: { scheduledRetryAt: dueAt.toISOString(), executionRetryAccounting: accountingForScheduledRetry(run, NATIVE_PROVIDER_OVERLOADED_CODE, attempt), }; - await notifyRunDispatch(input.tx); + await notifyRunDispatch(input.tx, { companyId: run.companyId }); const [wake] = await input.tx.insert(agentWakeupRequests).values({ companyId: run.companyId, agentId: run.agentId, source: "automation", triggerDetail: "system", reason: NATIVE_PROVIDER_OVERLOADED_CODE, status: "queued", runId: retryId, diff --git a/server/src/services/recovery/service.ts b/server/src/services/recovery/service.ts index 4393e0ad5cf..fffc2699dbf 100644 --- a/server/src/services/recovery/service.ts +++ b/server/src/services/recovery/service.ts @@ -2717,7 +2717,7 @@ export function recoveryService( const now = new Date(); const retryAt = readProviderQuotaRetryAt(input.latestRun, now); return database.transaction(async (tx) => { - await notifyRunDispatch(tx); + await notifyRunDispatch(tx, { companyId: input.issue.companyId }); const wakeup = await tx .insert(agentWakeupRequests) .values({ @@ -3318,7 +3318,7 @@ export function recoveryService( created = Boolean(enqueuedRun); } else { scheduledRun = await db.transaction(async (tx) => { - await notifyRunDispatch(tx); + await notifyRunDispatch(tx, { companyId: input.issue.companyId }); const wakeup = await tx .insert(agentWakeupRequests) .values({ diff --git a/server/src/services/run-dispatch-notifications.ts b/server/src/services/run-dispatch-notifications.ts index 9b1bb175894..2e04e6e7531 100644 --- a/server/src/services/run-dispatch-notifications.ts +++ b/server/src/services/run-dispatch-notifications.ts @@ -1,6 +1,9 @@ import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js"; -export async function notifyRunDispatch(transaction: object) { - await notifyDeliveryWork(transaction, DELIVERY_QUEUES.scheduledRetryDispatch); - await notifyDeliveryWork(transaction, DELIVERY_QUEUES.queuedRunDispatch); +export type RunDispatchScope = { companyId: string }; + +export async function notifyRunDispatch(transaction: object, scope?: RunDispatchScope) { + const hint = scope ? JSON.stringify(scope) : "*"; + await notifyDeliveryWork(transaction, DELIVERY_QUEUES.scheduledRetryDispatch, hint); + await notifyDeliveryWork(transaction, DELIVERY_QUEUES.queuedRunDispatch, hint); } diff --git a/server/src/services/scheduled-retry-cancellation-notifications.ts b/server/src/services/scheduled-retry-cancellation-notifications.ts index c5548996607..24e57de66cb 100644 --- a/server/src/services/scheduled-retry-cancellation-notifications.ts +++ b/server/src/services/scheduled-retry-cancellation-notifications.ts @@ -4,6 +4,6 @@ import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notificatio export type ScheduledRetryCancellationScope = { companyId: string; runId?: string }; export async function notifyScheduledRetryCancellation(transaction: object, scope?: ScheduledRetryCancellationScope) { - await notifyRunDispatch(transaction); + await notifyRunDispatch(transaction, scope); await notifyDeliveryWork(transaction, DELIVERY_QUEUES.scheduledRetryCancellation, scope ? JSON.stringify(scope) : "*"); } From 2955e0e22e253da31e698f600e166f6b5db41fc9 Mon Sep 17 00:00:00 2001 From: Dotta Date: Fri, 30 Oct 2026 17:04:31 -0500 Subject: [PATCH 22/70] fix(server): scope run retention hints to affected companies Co-Authored-By: Paperclip --- server/src/__tests__/execution-control-work.test.ts | 4 ++-- .../src/services/execution-control-notifications.ts | 11 +++++++---- .../src/services/execution-control-reconciliation.ts | 2 +- server/src/services/execution-recovery-resolution.ts | 4 ++-- server/src/services/explicit-native-continuation.ts | 2 +- server/src/services/heartbeat.ts | 2 +- server/src/services/heartbeat/runtime-execution.ts | 4 ++-- server/src/services/issue-recovery-actions.ts | 10 +++++----- server/src/services/legacy-execution-recovery.ts | 2 +- .../native-failed-workspace-recovery.ts | 2 +- .../native-runtime/native-restart-recovery.ts | 2 +- .../services/native-runtime/native-run-finalizer.ts | 4 ++-- .../services/native-runtime/native-run-projection.ts | 2 +- .../native-runtime/native-safe-replacement.ts | 4 ++-- .../native-runtime/native-session-executor.ts | 6 +++--- .../native-runtime/native-superseded-retry.ts | 2 +- server/src/services/system-recovery.ts | 2 +- server/src/services/task-retry.ts | 2 +- 18 files changed, 35 insertions(+), 32 deletions(-) diff --git a/server/src/__tests__/execution-control-work.test.ts b/server/src/__tests__/execution-control-work.test.ts index 0046d8574aa..14cd9456eb5 100644 --- a/server/src/__tests__/execution-control-work.test.ts +++ b/server/src/__tests__/execution-control-work.test.ts @@ -383,11 +383,11 @@ describe.skipIf(!support.supported)("execution-control durable work", () => { const due = new Date(Date.now() + 120_000), earlier = new Date(Date.now() + 60_000); for (const at of [due, earlier]) { await writeExecutionControl(db, DELIVERY_QUEUES.executionFinalization, tx => tx.update(heartbeatRuns) - .set({ executionControlDeadlineAt: at }).where(eq(heartbeatRuns.id, run.id))); + .set({ executionControlDeadlineAt: at }).where(eq(heartbeatRuns.id, run.id)), { companyId }); await settle(); expect(c.nextWakeAt()).toBe(at.getTime()); } await writeExecutionControl(db, DELIVERY_QUEUES.executionFinalization, tx => tx.update(heartbeatRuns) - .set({ executionControlDeadlineAt: null }).where(eq(heartbeatRuns.id, run.id))); + .set({ executionControlDeadlineAt: null }).where(eq(heartbeatRuns.id, run.id)), { companyId }); await quiet(c); }); it("delivers a reconciled continuation when marking and resolving commit together", async () => { diff --git a/server/src/services/execution-control-notifications.ts b/server/src/services/execution-control-notifications.ts index 39b1d7765c6..1839176769f 100644 --- a/server/src/services/execution-control-notifications.ts +++ b/server/src/services/execution-control-notifications.ts @@ -1,9 +1,11 @@ -import { notifyRunDispatch, type RunDispatchScope } from "./run-dispatch-notifications.js"; +import { notifyRunDispatch } from "./run-dispatch-notifications.js"; import type { Db } from "@paperclipai/db"; import { DELIVERY_QUEUES, notifyDeliveryWork, type DeliveryQueue } from "./delivery-work-notifications.js"; -export async function notifyExecutionRunWork(transaction: object, scope?: RunDispatchScope) { - await notifyRunDispatch(transaction, scope); +export type ExecutionWorkScope = { companyId: string; agentId?: string; runId?: string }; + +export async function notifyExecutionRunWork(transaction: object, scope?: ExecutionWorkScope) { + if (scope) await notifyRunDispatch(transaction, scope); for (const queue of [DELIVERY_QUEUES.executionStatus, DELIVERY_QUEUES.executionFinalization, DELIVERY_QUEUES.executionReplacement, DELIVERY_QUEUES.executionDisposition, DELIVERY_QUEUES.issueMonitors]) { await notifyDeliveryWork(transaction, queue); @@ -14,9 +16,10 @@ export function writeExecutionControl( db: Db | Parameters[0]>[0], queue: DeliveryQueue | readonly DeliveryQueue[], write: (tx: Db) => PromiseLike, + scope: ExecutionWorkScope | undefined, ): Promise { return db.transaction(async transaction => { - await notifyRunDispatch(transaction); + await notifyRunDispatch(transaction, scope); await notifyDeliveryWork(transaction, DELIVERY_QUEUES.issueMonitors); for (const topic of typeof queue === "string" ? [queue] : queue) await notifyDeliveryWork(transaction, topic); return write(transaction as unknown as Db); diff --git a/server/src/services/execution-control-reconciliation.ts b/server/src/services/execution-control-reconciliation.ts index 2f2a435982d..621fd210cfc 100644 --- a/server/src/services/execution-control-reconciliation.ts +++ b/server/src/services/execution-control-reconciliation.ts @@ -196,7 +196,7 @@ export async function reconcileAbandonedExecutionControl( await notifyNativeEnvironmentSettlement(tx, { companyId: run.companyId, runId: run.id }); await notifyCancelledWorkspacePreservation(tx, { companyId: run.companyId, runId: run.id }); await notifyUnsafeWorkspaceExportRunChange(tx, run); - await notifyExecutionRunWork(tx); + await notifyExecutionRunWork(tx, { companyId: run.companyId }); const [updatedRun] = await tx .update(heartbeatRuns) .set({ diff --git a/server/src/services/execution-recovery-resolution.ts b/server/src/services/execution-recovery-resolution.ts index f3f5a3e4310..d56e7d926f1 100644 --- a/server/src/services/execution-recovery-resolution.ts +++ b/server/src/services/execution-recovery-resolution.ts @@ -546,7 +546,7 @@ export async function settleUnrecoverableExecutions( companyId: run.companyId, issueId: task.id, runId: run.id, }).limit(1).for("update"); if (!hold) return; - await notifyExecutionRunWork(tx); + await notifyExecutionRunWork(tx, { companyId: run.companyId }); const note = "Exact session recovery stopped. The recorded workspace recovery holds this execution without replaying its actions."; await tx.update(issueRecoveryActions).set({ status: "cancelled", outcome: "cancelled", resolvedAt: now, updatedAt: now, wakePolicy: null, monitorPolicy: null, @@ -654,7 +654,7 @@ export async function settleUnrecoverableExecutions( replay: "not_authorized", }, }); - await notifyExecutionRunWork(tx); + await notifyExecutionRunWork(tx, { companyId: run.companyId }); await tx .update(heartbeatRuns) .set({ executionStatusDeliveryId: randomUUID() }) diff --git a/server/src/services/explicit-native-continuation.ts b/server/src/services/explicit-native-continuation.ts index 074ffc3bdf5..084a25e457f 100644 --- a/server/src/services/explicit-native-continuation.ts +++ b/server/src/services/explicit-native-continuation.ts @@ -475,7 +475,7 @@ export async function admitExplicitNativeContinuation(input: { }).where(and(eq(agentWakeupRequests.companyId, companyId), eq(agentWakeupRequests.id, input.queuedCommentInterruptId), eq(agentWakeupRequests.agentId, agentId), eq(agentWakeupRequests.status, "deferred_issue_execution"))); if (cancelledStartupIds.size) await db.transaction(async tx => { - await notifyExecutionRunWork(tx); + await notifyExecutionRunWork(tx, { companyId, agentId }); for (const runId of cancelledStartupIds) { await tx.update(nativeRunFinalizations).set({ phase: "terminal_failure", failureCode: "native_startup_cancelled", nextAttemptAt: null, diff --git a/server/src/services/heartbeat.ts b/server/src/services/heartbeat.ts index bbce8ba8f5b..2160f451b23 100644 --- a/server/src/services/heartbeat.ts +++ b/server/src/services/heartbeat.ts @@ -3726,7 +3726,7 @@ export function heartbeatService( await writeExecutionControl(db, DELIVERY_QUEUES.executionFinalization, tx => tx .update(heartbeatRuns) .set({ executionControlDeadlineAt: null }) - .where(eq(heartbeatRuns.id, run.id))); + .where(eq(heartbeatRuns.id, run.id)), { companyId: run.companyId }); } nativeOwnershipHeld = nativeOwnershipHeld || diff --git a/server/src/services/heartbeat/runtime-execution.ts b/server/src/services/heartbeat/runtime-execution.ts index b5b64abc948..bdbbfa07752 100644 --- a/server/src/services/heartbeat/runtime-execution.ts +++ b/server/src/services/heartbeat/runtime-execution.ts @@ -966,7 +966,7 @@ export async function executeHeartbeatRuntime(db: Db, input: HeartbeatRuntimeExe eq(heartbeatRuns.id, run.id), eq(heartbeatRuns.status, "running"), ), - )); + ), { companyId: run.companyId }); const workspaceFinalizeStatus = hasWorkspaceRestoreFailure(adapterResult.resultJson) ? "failed" : "succeeded"; await recordWorkspaceFinalize(workspaceFinalizeStatus); if (adapterResult.nativeFinalization) { @@ -1042,7 +1042,7 @@ export async function executeHeartbeatRuntime(db: Db, input: HeartbeatRuntimeExe eq(heartbeatRuns.id, run.id), eq(heartbeatRuns.status, "running"), ), - )); + ), { companyId: run.companyId }); if ( issueRef && context.resumeSessionGoalHeartbeat === true && diff --git a/server/src/services/issue-recovery-actions.ts b/server/src/services/issue-recovery-actions.ts index 5fb28bcb6d8..48fb001c0b4 100644 --- a/server/src/services/issue-recovery-actions.ts +++ b/server/src/services/issue-recovery-actions.ts @@ -392,7 +392,7 @@ export function issueRecoveryActionService(db: DbOrTransaction) { const [updated] = await writeExecutionControl(db, [DELIVERY_QUEUES.executionDisposition, DELIVERY_QUEUES.executionContinuation], tx => tx.update(issueRecoveryActions).set({ nextAction: input.nextAction, updatedAt: now }) .where(and(eq(issueRecoveryActions.id, existing.id), inArray(issueRecoveryActions.status, [...ACTIVE_RECOVERY_ACTION_STATUSES]), - sql`${issueRecoveryActions.evidence}->'workspaceOwnerStop' is null`)).returning()); + sql`${issueRecoveryActions.evidence}->'workspaceOwnerStop' is null`)).returning(), { companyId: input.companyId }); if (updated) return toReadModel(updated); } return existing; @@ -467,7 +467,7 @@ export function issueRecoveryActionService(db: DbOrTransaction) { ), ) .returning(); - }); + }, { companyId: input.companyId }); if (!exhausted) { return retryUpsertSourceScoped(input, retryCount); } @@ -539,7 +539,7 @@ export function issueRecoveryActionService(db: DbOrTransaction) { ), ) .returning(); - }); + }, { companyId: input.companyId }); if (!updated) { return retryUpsertSourceScoped(input, retryCount); } @@ -554,7 +554,7 @@ export function issueRecoveryActionService(db: DbOrTransaction) { .insert(issueRecoveryActions) .values(buildInsertValues(input, ownerType, now)) .returning(); - }); + }, { companyId: input.companyId }); return toReadModel(created!); } catch (error) { if (!isUniqueRecoveryActionConflict(error)) throw error; @@ -608,7 +608,7 @@ export function issueRecoveryActionService(db: DbOrTransaction) { }) .where(and(...predicates)) .returning(); - }); + }, { companyId: input.companyId }); return updated ? toReadModel(updated) : null; } diff --git a/server/src/services/legacy-execution-recovery.ts b/server/src/services/legacy-execution-recovery.ts index 427704de63f..c462cdf1fca 100644 --- a/server/src/services/legacy-execution-recovery.ts +++ b/server/src/services/legacy-execution-recovery.ts @@ -135,7 +135,7 @@ export async function terminalizeLegacyExecution(input: { }; if (issueId) await notifyUnsafeWorkspaceExportRecovery(tx, { companyId: current.companyId, issueId }); await notifyInstructionCollection(tx, { companyId: current.companyId, runId: current.id }); - await notifyExecutionRunWork(tx); + await notifyExecutionRunWork(tx, { companyId: current.companyId }); let [updated] = await tx .update(heartbeatRuns) .set({ diff --git a/server/src/services/native-runtime/native-failed-workspace-recovery.ts b/server/src/services/native-runtime/native-failed-workspace-recovery.ts index 5c3268f6afc..a1c6c914b33 100644 --- a/server/src/services/native-runtime/native-failed-workspace-recovery.ts +++ b/server/src/services/native-runtime/native-failed-workspace-recovery.ts @@ -65,7 +65,7 @@ export async function preserveNativeFailedWorkspace(db: Db, selected: Pick>'{systemRecovery,revision}' = ${String(previous.revision)}`)) .returning({ id: issueRecoveryActions.id }); return Boolean(updated); - }); + }, { companyId: source.companyId }); } export async function loadSystemRecovery(db: Db, source: WorkspaceRecoverySourceIdentity) { diff --git a/server/src/services/task-retry.ts b/server/src/services/task-retry.ts index e9f0bd4edbf..17075c34eb9 100644 --- a/server/src/services/task-retry.ts +++ b/server/src/services/task-retry.ts @@ -334,7 +334,7 @@ export async function requestTaskRetry(input: { details: { requestId: active?.id ?? id, sourceRunId: latest.id, attachedToExistingWork: Boolean(active || busy) } }); return { requestId: active?.id ?? id, accepted: true, stale: false, observedRunId: latest.id, chatActionId: intent.chatActionId }; - }); + }, { companyId: input.companyId }); } /** Chat owns admission and authorization. The task receipt follows that From 7b0da01e3a6b6fc04a4eb887fc6da752febd35f1 Mon Sep 17 00:00:00 2001 From: Dotta Date: Fri, 30 Oct 2026 17:07:07 -0500 Subject: [PATCH 23/70] test(server): preserve UUID entropy in restore company prefixes The 60-company retained-history fixture collided on Rad9867 under companies_issue_prefix_idx in CI job 124331069055. Use the complete generated company UUID for its unique prefix while preserving all retention assertions. Co-Authored-By: Paperclip --- server/src/__tests__/git-restore-point-ledger.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/__tests__/git-restore-point-ledger.test.ts b/server/src/__tests__/git-restore-point-ledger.test.ts index 8b2e894dc44..0f30dea022b 100644 --- a/server/src/__tests__/git-restore-point-ledger.test.ts +++ b/server/src/__tests__/git-restore-point-ledger.test.ts @@ -73,7 +73,7 @@ describeEmbeddedPostgres("git restore point retention", () => { } = {}) { const companyId = input.companyId ?? randomUUID(); if (!input.companyId) { - await db.insert(companies).values({ id: companyId, name: "Restore", issuePrefix: `R${companyId.slice(0, 6)}` }); + await db.insert(companies).values({ id: companyId, name: "Restore", issuePrefix: `R${companyId.replaceAll("-", "")}` }); } const issueStatus = input.issueStatus ?? "done"; const closedAt = new Date(now.getTime() - (input.closedDaysAgo ?? 10) * DAY_MS); From 6fe6d6d29cbefd041435c6282d6b2dd7507140c9 Mon Sep 17 00:00:00 2001 From: Dotta Date: Fri, 30 Oct 2026 17:10:15 -0500 Subject: [PATCH 24/70] fix(server): bound retention reconstruction and preserve producer transactions Batch review and confirmation eligibility per company, scope decision and budget hints, and keep child blocker relation intents inside their owning transaction. Cover ordinary child, budget and execution producers plus full candidate-page query bounds; update startup/import fixtures for source-driven retention. Co-Authored-By: Paperclip --- .../2026-10-30-attention-retention-sleep.md | 4 +- .../attention-retention-work.test.ts | 75 +++++++++++- .../src/__tests__/company-portability.test.ts | 1 + .../decision-effect-recovery-work.test.ts | 4 +- .../src/__tests__/decisions-service.test.ts | 2 +- .../server-startup-feedback-export.test.ts | 18 ++- server/src/services/attention-retention.ts | 113 ++++++++++++------ .../services/attention-source-eligibility.ts | 4 + server/src/services/attention.ts | 7 +- server/src/services/budgets.ts | 21 ++-- server/src/services/decision-expiry.ts | 2 +- .../services/decision-work-notifications.ts | 14 ++- server/src/services/decisions.ts | 25 ++-- server/src/services/heartbeat-run-events.ts | 6 +- server/src/services/issues.ts | 47 ++++---- 15 files changed, 242 insertions(+), 101 deletions(-) diff --git a/doc/plans/2026-10-30-attention-retention-sleep.md b/doc/plans/2026-10-30-attention-retention-sleep.md index c0e4e168bd0..1b71dc71261 100644 --- a/doc/plans/2026-10-30-attention-retention-sleep.md +++ b/doc/plans/2026-10-30-attention-retention-sleep.md @@ -25,7 +25,7 @@ The feed and worker share confirmation collapse, newer-run probes, issue visibil ## Existing mutation boundaries reactivate work -Committed hints identify a company, source kind, and source ID where those values are known. Retention's own writes use exact source hints; blocker retention changes invalidate the company/source because multiple blocked roots can project one terminal subject. Reconstruction pages and due batches remain bounded. +Committed hints identify a company, source kind, and source ID where those values are known. Retention's own writes use exact source hints; blocker retention changes invalidate the company/source because multiple blocked roots can project one terminal subject. Reconstruction pages and due batches remain bounded. Review attention and interaction invokability are computed in company batches per candidate page; a full 100-candidate review page uses at most 20 statements and 100 confirmations use at most six. Ordinary execution, decision and budget mutations pass their affected company; exact decision sources are included. Producer coverage includes decision creation and changes; interaction creation, resolution, expiry and supersession; issue creation, updates, blocker relations, checkout, adoption and release; agent hire, configuration, lifecycle, error clearing and deletion; budget incidents and company pause/resume; exhausted-run events and retry-run creation; native status, review, finalization and reconstruction; external conversation transitions; and imported blocker relations. Existing service transactions retain outer commit ownership. Savepoint release does not publish work before outer commit. @@ -33,7 +33,7 @@ The writer contract follows the existing delivery workers: serving processes use ## Real PostgreSQL evidence proves quiet and reactivation -The new acceptance suite exercises all eight sources, including terminal blocker identity and date, newest confirmations, independent quality gates, unhealthy reporting chains, conversation visibility, maintained reviews and pending approvals, newer-run suppression, ordinary task and interaction producers, ordinary exhaustion-event ingress, and ordinary error clearing. +The new acceptance suite exercises all eight sources, including terminal blocker identity and date, newest confirmations, independent quality gates, unhealthy reporting chains, conversation visibility, maintained reviews and pending approvals, newer-run suppression, ordinary task and interaction producers, ordinary exhaustion-event ingress, and ordinary error clearing. Additional ordinary producer controls cover child creation with a parent-blocking relation, budget incident creation and resolution, and repeated company-scoped execution-control writes. It also checks keep/revive, moved and cancelled deadlines, company activation and isolation, retained terminal history, restart, nested outer commit and rollback, first writes during drain, retry across drain, atomic archive/outbox rollback, and notifications during reconstruction. A 225-item backlog uses three startup pages and source-specific self-invalidations. SQL is counted at the PostgreSQL driver boundary. diff --git a/server/src/__tests__/attention-retention-work.test.ts b/server/src/__tests__/attention-retention-work.test.ts index 22a966d5fef..889030d2c22 100644 --- a/server/src/__tests__/attention-retention-work.test.ts +++ b/server/src/__tests__/attention-retention-work.test.ts @@ -2,9 +2,12 @@ import { randomUUID } from "node:crypto"; import { and, eq, sql } from "drizzle-orm"; import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; import { authUsers, companyMemberships, agents, approvals, budgetIncidents, budgetPolicies, companies, createDb, applyPendingMigrations, decisionArchiveNotificationOutbox, - decisionRetention, decisions, heartbeatRunEvents, heartbeatRuns, issueApprovals, issueRecoveryActions, + decisionRetention, decisions, costEvents, heartbeatRunEvents, heartbeatRuns, issueApprovals, issueRecoveryActions, issueRelations, issueThreadInteractions, issues, type Db } from "@paperclipai/db"; import { attentionRetentionService, registerAttentionRetentionWork } from "../services/attention-retention.js"; +import { budgetService } from "../services/budgets.js"; +import { writeExecutionControl } from "../services/execution-control-notifications.js"; +import { DELIVERY_QUEUES } from "../services/delivery-work-notifications.js"; import { attentionService } from "../services/attention.js"; import { companyService } from "../services/companies.js"; import { createDeliveryWorkCoordinator } from "../services/delivery-work-coordinator.js"; @@ -135,6 +138,76 @@ const delay = DEFAULT_DECISION_ARCHIVE_DAYS * 86_400_000; await coordinator!.stop(); coordinator = undefined; await start(); await quiet(); }); + it("registers the ordinary createChild blocker path inside its relation transaction", async () => { + const parent = await issueService(db).create(companyId, { title: "Parent", status: "in_progress", assigneeAgentId: agentId, createdByAgentId: agentId }); + await start(); await quiet(); + const { issue: child, parentBlockerAdded } = await issueService(db).createChild(parent.id, { + title: "Child needs review", status: "in_review", assigneeUserId: "board", createdByAgentId: agentId, + blockParentUntilDone: true, + }); + expect(parentBlockerAdded).toBe(true); + expect(await db.select().from(issueRelations).where(and(eq(issueRelations.issueId, child.id), + eq(issueRelations.relatedIssueId, parent.id)))).toHaveLength(1); + await settle(); expect(coordinator!.nextWakeAt()).not.toBeNull(); + await vi.advanceTimersByTimeAsync(delay + 2000); await archived("review", child.id); await quiet(); + expect(await db.select().from(decisionArchiveNotificationOutbox).where(eq(decisionArchiveNotificationOutbox.sourceId, child.id))).toHaveLength(1); + }); + + it("batches a full page of maintained reviews and confirmations by company", async () => { + const rows = await db.insert(issues).values(Array.from({ length: 100 }, () => ({ companyId, + title: randomUUID(), status: "in_review", assigneeAgentId: agentId, updatedAt: old() }))).returning(); + await db.insert(issueThreadInteractions).values(rows.map(row => ({ companyId, issueId: row.id, + kind: "request_confirmation", addresseeAgentId: agentId, payload: { version: 1, prompt: "Agent verdict" } }))); + statements.length = 0; + const page = await attentionRetentionService(db).load({ companyId, sourceKind: "review" }); + expect(page.rows).toEqual([]); expect(page.complete).toBe(false); + expect(statements.length).toBeLessThanOrEqual(20); + expect(statements.filter(query => query.includes('from "agents"')).length).toBeLessThanOrEqual(1); + await db.update(agents).set({ status: "paused" }).where(eq(agents.id, agentId)); + statements.length = 0; + const interactions = await attentionRetentionService(db).load({ companyId, sourceKind: "issue_thread_interaction" }); + expect(interactions.rows).toHaveLength(100); expect(interactions.complete).toBe(false); + expect(statements.length).toBeLessThanOrEqual(6); + expect(statements.filter(query => query.includes('from "agents"'))).toHaveLength(1); + }); + + it("scopes ordinary execution-control writes to their company", async () => { + const otherCompany = randomUUID(); + await db.insert(companies).values({ id: otherCompany, name: "Other", issuePrefix: "OTHER" }); + await db.insert(issues).values(Array.from({ length: 125 }, () => ({ companyId: otherCompany, + title: randomUUID(), status: "in_review", assigneeUserId: "board", updatedAt: old() }))); + const [run] = await db.insert(heartbeatRuns).values({ companyId, agentId, status: "running" }).returning(); + const service = attentionRetentionService(db), load = vi.spyOn(service, "load"); + await start(service); await quiet(); load.mockClear(); + for (let count = 0; count < 3; count++) { + await writeExecutionControl(db, DELIVERY_QUEUES.executionFinalization, + tx => tx.update(heartbeatRuns).set({ updatedAt: new Date() }).where(eq(heartbeatRuns.id, run.id)), { companyId }); + await quiet(); + } + expect(load).toHaveBeenCalled(); + expect(load.mock.calls.every(([scope]) => scope?.companyId === companyId)).toBe(true); + }); + + it("registers and cancels budget deadlines through ordinary policy and incident producers", async () => { + const service = budgetService(db); + await service.upsertPolicy(companyId, { scopeType: "agent", scopeId: agentId, amount: 100, + warnPercent: 85, hardStopEnabled: false, notifyEnabled: true }, "board"); + const retention = attentionRetentionService(db), load = vi.spyOn(retention, "load"); + await start(retention); await quiet(); load.mockClear(); + const [event] = await db.insert(costEvents).values({ companyId, agentId, provider: "openai", biller: "openai", + billingType: "metered_api", model: "fixture", inputTokens: 100, outputTokens: 20, + costCents: 90, occurredAt: new Date() }).returning(); + await service.evaluateCostEvent(event); + await settle(); expect(coordinator!.nextWakeAt()).not.toBeNull(); + const [incident] = await db.select().from(budgetIncidents).where(eq(budgetIncidents.companyId, companyId)); + expect(incident).toMatchObject({ status: "open", thresholdType: "soft" }); + expect(await state("budget_alert", incident.id)).toBeNull(); + await service.upsertPolicy(companyId, { scopeType: "agent", scopeId: agentId, amount: 200 }, "board"); + await quiet(); + expect((await db.select().from(budgetIncidents).where(eq(budgetIncidents.id, incident.id)))[0].status).toBe("resolved"); + expect(load.mock.calls.every(([scope]) => scope?.companyId === companyId)).toBe(true); + }); + it("matches the existing feed across all eight remaining source kinds and exclusion variants", async () => { const origin = await insertIssue(); const review = await insertIssue("in_review", { assigneeUserId: "board" }); diff --git a/server/src/__tests__/company-portability.test.ts b/server/src/__tests__/company-portability.test.ts index fdf00b828af..ee7db993937 100644 --- a/server/src/__tests__/company-portability.test.ts +++ b/server/src/__tests__/company-portability.test.ts @@ -4184,6 +4184,7 @@ describe("company portability", () => { }), // eslint-disable-next-line @typescript-eslint/no-explicit-any } as any; + db.transaction = async (write: (tx: typeof db) => Promise) => write(db); return { db, insertedRelationValues, monitorUpdates }; } diff --git a/server/src/__tests__/decision-effect-recovery-work.test.ts b/server/src/__tests__/decision-effect-recovery-work.test.ts index 6c1a17d2b39..8a7d4e300c8 100644 --- a/server/src/__tests__/decision-effect-recovery-work.test.ts +++ b/server/src/__tests__/decision-effect-recovery-work.test.ts @@ -73,7 +73,7 @@ const support = externalUrl ? { supported: true } : await getEmbeddedPostgresTes const claim = (id: string, updatedAt = new Date(), owner = db) => writeDecisionChange(owner, async tx => tx.update(decisions).set({ status: "decided", executionStatus: "running", chosenOptionId: "yes", decidedByUserId: userId, updatedAt, inputValues: {}, metadata: { continuationPending: true }, - }).where(eq(decisions.id, id))); + }).where(eq(decisions.id, id)), { companyId }); async function start() { coordinator = createDeliveryWorkCoordinator({ owner: db, canRun: () => !draining, canReconcile: () => true, onError: error => errors.push(error) }); @@ -467,7 +467,7 @@ const support = externalUrl ? { supported: true } : await getEmbeddedPostgresTes } else { await writeDecisionChange(db, async tx => tx.update(decisions).set({ status: "decided", executionStatus: "running", chosenOptionId: "archive", decidedByUserId: userId, updatedAt: new Date(Date.now() - 120_000), - metadata: { ...row.metadata, continuationPending: true } }).where(eq(decisions.id, row.id))); + metadata: { ...row.metadata, continuationPending: true } }).where(eq(decisions.id, row.id)), { companyId }); } await start(); await settled(row.id, committed ? "failed" : "succeeded"); await quiet(); const [retained] = await db.select().from(decisionRetention); diff --git a/server/src/__tests__/decisions-service.test.ts b/server/src/__tests__/decisions-service.test.ts index 7fc2cc50c0d..999bde8d60e 100644 --- a/server/src/__tests__/decisions-service.test.ts +++ b/server/src/__tests__/decisions-service.test.ts @@ -793,7 +793,7 @@ describePg("decisionService", () => { expect((await service().get(id))?.metadata.continuationPending).toBe(false); } const markReady = (id: string, source = db) => writeDecisionChange(source, async tx => tx.update(decisions) - .set({ status: "cancelled", metadata: { continuationPending: true } }).where(eq(decisions.id, id))); + .set({ status: "cancelled", metadata: { continuationPending: true } }).where(eq(decisions.id, id)), { companyId }); it("coalesces sequential tenant writes without dropping pending recovery", async () => { const row = await createCommentDecision(); diff --git a/server/src/__tests__/server-startup-feedback-export.test.ts b/server/src/__tests__/server-startup-feedback-export.test.ts index bbb020c865d..bc72ad57f02 100644 --- a/server/src/__tests__/server-startup-feedback-export.test.ts +++ b/server/src/__tests__/server-startup-feedback-export.test.ts @@ -1,3 +1,4 @@ +import { attentionRetentionService } from "../services/attention-retention.js"; import { nativeHarnessBackupExpirationService } from "../services/native-runtime/native-harness-backup-expiration.js"; import { skillVersionRetentionService } from "../services/skill-version-retention.js"; import { approvalQueueSeedingService } from "../services/approval-queue-seeding-work.js"; @@ -377,6 +378,11 @@ vi.mock("../realtime/live-events-ws.js", () => ({ setupLiveEventsWebSocketServer: vi.fn(), })); +vi.mock("../services/attention-retention.js", async importOriginal => ({ + ...await importOriginal(), + attentionRetentionService: vi.fn(() => ({ load: vi.fn(async () => ({ rows: [], complete: true })), check: vi.fn() })), +})); + vi.mock("../services/join-request-retention.js", async importOriginal => ({ ...await importOriginal(), joinRequestRetentionService: vi.fn(() => ({ sweep: vi.fn(async () => ({ archived: 0 })), next: vi.fn(async () => null) })), @@ -974,7 +980,7 @@ describe("startServer feedback export wiring", () => { } }); - it.each([false, true])("registers archive delivery separately from retention sweeps (scheduler %s)", async enabled => { + it.each([false, true])("registers source retention separately from archive delivery without tick scans (scheduler %s)", async enabled => { loadConfigMock.mockReturnValue(buildTestConfig({ heartbeatSchedulerEnabled: enabled, heartbeatSchedulerIntervalMs: 30000 })); let tick: (() => void) | undefined; const timer = vi.spyOn(globalThis, "setInterval").mockImplementation(((callback: () => void, interval: number) => { @@ -986,12 +992,17 @@ describe("startServer feedback export wiring", () => { const registration = deliveryWorkRegisterMock.mock.calls.find(([queue]) => queue === DELIVERY_QUEUES.decisionArchiveDelivery); if (!enabled) { expect(registration).toBeUndefined(); + expect(attentionRetentionService).not.toHaveBeenCalled(); expect(decisionRetentionService).not.toHaveBeenCalled(); expect(tick).toBeDefined(); tick!(); await settleStartupHousekeeping(); expect(decisionRetentionService).not.toHaveBeenCalled(); return; } expect(registration).toBeDefined(); + expect(deliveryWorkRegisterMock.mock.calls.some(([queue]) => queue === DELIVERY_QUEUES.attentionRetention)).toBe(true); + expect(attentionRetentionService).toHaveBeenCalledOnce(); + const source = vi.mocked(attentionRetentionService).mock.results[0]!.value; + expect(source.load).toHaveBeenCalledOnce(); const retention = vi.mocked(decisionRetentionService).mock.results[0]!.value; expect(retention.deliverNotifications).toHaveBeenCalledOnce(); expect(retention.nextArchiveNotificationAt).toHaveBeenCalledOnce(); @@ -999,9 +1010,10 @@ describe("startServer feedback export wiring", () => { const companyReads = () => vi.mocked(db.select).mock.results.filter(({ value }) => vi.mocked(value.from).mock.calls.some(([table]) => table === companies)).length; const before = companyReads(); - expect(before).toBeGreaterThan(0); expect(tick).toBeDefined(); + expect(before).toBe(0); expect(tick).toBeDefined(); tick!(); await settleStartupHousekeeping(); - expect(companyReads()).toBeGreaterThan(before); + expect(companyReads()).toBe(before); + expect(source.load).toHaveBeenCalledOnce(); expect(retention.deliverNotifications).toHaveBeenCalledOnce(); await registration![1].run(new AbortController().signal); expect(retention.deliverNotifications).toHaveBeenCalledTimes(2); diff --git a/server/src/services/attention-retention.ts b/server/src/services/attention-retention.ts index 4622c4883bf..85e27a0d941 100644 --- a/server/src/services/attention-retention.ts +++ b/server/src/services/attention-retention.ts @@ -1,13 +1,14 @@ -import { and, eq, isNull, sql, type SQL } from "drizzle-orm"; +import { and, eq, inArray, isNull, sql, type SQL } from "drizzle-orm"; import { agents, decisionRetention, decisions, heartbeatRuns, issueApprovals, issueRecoveryActions, issueThreadInteractions, issues, approvals, type Db } from "@paperclipai/db"; import type { AttentionSourceKind } from "@paperclipai/shared"; import { evaluateAgentInvokability } from "./agent-invokability.js"; -import { collapsePendingConfirmationsToNewest, listFailedRunsWithNewerRun, readRunIssueId, requiresReviewAttention, terminalBlockerRequiresAttention } from "./attention-source-eligibility.js"; +import { HUMAN_RECOVERY_OWNER_TYPES, ATTENTION_BUDGET_WARNING_PERCENT, ATTENTION_AGENT_ERROR_STATUS, collapsePendingConfirmationsToNewest, listFailedRunsWithNewerRun, readRunIssueId, requiresReviewAttention, terminalBlockerRequiresAttention } from "./attention-source-eligibility.js"; import { decisionRetentionService, DEFAULT_DECISION_ARCHIVE_DAYS } from "./decision-retention.js"; import type { createDeliveryWorkCoordinator } from "./delivery-work-coordinator.js"; import { DELIVERY_QUEUES } from "./delivery-work-notifications.js"; import { attentionIssueCondition } from "./issue-visibility.js"; +import { RETRY_EXHAUSTION_MESSAGE_PREFIX } from "./heartbeat-run-events.js"; import { issueService } from "./issues.js"; import { parseIssueExecutionState } from "./issue-execution-policy.js"; import { isProspectiveBlockedTransition } from "./routable-blocked.js"; @@ -34,7 +35,7 @@ const sources: Record = { issue_thread_interaction: sql`select 'issue_thread_interaction', i.id::text, i.company_id, i.updated_at from issue_thread_interactions i where i.status = 'pending' and i.addressee_user_id is null`, recovery_action: sql`select 'recovery_action', r.id::text, r.company_id, r.updated_at - from issue_recovery_actions r where r.status in ('active', 'escalated') and r.owner_type in ('user', 'board')`, + from issue_recovery_actions r where r.status in ('active', 'escalated') and r.owner_type in (${sql.join(HUMAN_RECOVERY_OWNER_TYPES.map(owner => sql`${owner}`), sql`, `)})`, blocker_attention: sql`select 'blocker_attention', i.id::text, i.company_id, i.updated_at from issues i where i.status = 'blocked' and i.hidden_at is null and i.harness_kind is null`, review: sql`select 'review', i.id::text, i.company_id, i.updated_at @@ -42,11 +43,11 @@ const sources: Record = { failed_run: sql`select 'failed_run', r.id::text, r.company_id, coalesce(r.finished_at, r.updated_at, r.created_at) from heartbeat_runs r where r.status in ('failed', 'timed_out') and exists ( select 1 from heartbeat_run_events e where e.run_id = r.id and e.company_id = r.company_id - and e.event_type = 'lifecycle' and e.message like 'Bounded retry exhausted%')`, + and e.event_type = 'lifecycle' and e.message like ${`${RETRY_EXHAUSTION_MESSAGE_PREFIX}%`})`, budget_alert: sql`select 'budget_alert', b.id::text, b.company_id, b.updated_at from budget_incidents b where b.status = 'open' and (b.threshold_type = 'hard' or (b.amount_limit > 0 - and round(b.amount_observed::numeric / b.amount_limit * 10000) / 100 >= 85))`, - agent_error_alert: sql`select 'agent_error_alert', a.id::text, a.company_id, a.updated_at from agents a where a.status = 'error'`, + and round(b.amount_observed::numeric / b.amount_limit * 10000) / 100 >= ${ATTENTION_BUDGET_WARNING_PERCENT}))`, + agent_error_alert: sql`select 'agent_error_alert', a.id::text, a.company_id, a.updated_at from agents a where a.status = ${ATTENTION_AGENT_ERROR_STATUS}`, }; export function attentionRetentionService(db: Db) { @@ -74,7 +75,69 @@ export function attentionRetentionService(db: Db) { return row; } - async function eligible(database: Db, candidate: Candidate, variant?: "owner" | "terminal", lock = false): Promise { + async function interactionEligibility(database: Db, candidates: Candidate[]) { + const result = new Map(); + const byCompany = new Map(); + for (const candidate of candidates) { + if (candidate.sourceKind !== "issue_thread_interaction") continue; + const ids = byCompany.get(candidate.companyId) ?? []; + ids.push(candidate.sourceId); + byCompany.set(candidate.companyId, ids); + } + for (const [companyId, ids] of byCompany) { + const rows = await database.select().from(issueThreadInteractions).where(and( + eq(issueThreadInteractions.companyId, companyId), inArray(issueThreadInteractions.id, ids), + eq(issueThreadInteractions.status, "pending"), isNull(issueThreadInteractions.addresseeUserId))); + if (!rows.length) continue; + const visible = await database.select({ id: issues.id }).from(issues).where(and(eq(issues.companyId, companyId), + inArray(issues.id, rows.map(row => row.issueId)), attentionIssueCondition(true))); + const visibleIds = new Set(visible.map(row => row.id)); + const roster = await database.select().from(agents).where(eq(agents.companyId, companyId)); + const boardAddressed = (row: typeof issueThreadInteractions.$inferSelect) => !row.addresseeAgentId + || !evaluateAgentInvokability(roster.find(agent => agent.id === row.addresseeAgentId), roster).invokable; + const confirmationIssueIds = rows.filter(row => row.kind === "request_confirmation").map(row => row.issueId); + const siblings = confirmationIssueIds.length ? await database.select().from(issueThreadInteractions).where(and( + eq(issueThreadInteractions.companyId, companyId), inArray(issueThreadInteractions.issueId, confirmationIssueIds), + eq(issueThreadInteractions.status, "pending"), eq(issueThreadInteractions.kind, "request_confirmation"), + isNull(issueThreadInteractions.addresseeUserId))) : []; + const newest = new Set(collapsePendingConfirmationsToNewest(siblings.filter(boardAddressed)).map(row => row.id)); + for (const row of rows) result.set(row.id, visibleIds.has(row.issueId) && boardAddressed(row) + && (row.kind !== "request_confirmation" || newest.has(row.id))); + } + return result; + } + + async function reviewEligibility(database: Db, candidates: Candidate[]) { + const result = new Map(); + const byCompany = new Map(); + for (const candidate of candidates) { + if (candidate.sourceKind !== "review") continue; + const ids = byCompany.get(candidate.companyId) ?? []; + ids.push(candidate.sourceId); + byCompany.set(candidate.companyId, ids); + } + for (const [companyId, ids] of byCompany) { + const rows = await database.select().from(issues).where(and(eq(issues.companyId, companyId), + inArray(issues.id, ids), attentionIssueCondition(false))); + const pendingApprovals = await database.select({ issueId: issueApprovals.issueId }).from(issueApprovals) + .innerJoin(approvals, eq(approvals.id, issueApprovals.approvalId)).where(and( + eq(issueApprovals.companyId, companyId), eq(approvals.companyId, companyId), + inArray(issueApprovals.issueId, ids), eq(approvals.status, "pending"))); + const approved = new Set(pendingApprovals.map(row => row.issueId)); + const stalled = await issueService(database).listReviewAttention(companyId, rows); + for (const row of rows) { + const state = parseIssueExecutionState(row.executionState); + result.set(row.id, requiresReviewAttention({ + hasHumanParticipant: state?.status === "pending" && state.currentParticipant?.type === "user", + hasUserAssignee: Boolean(row.assigneeUserId), hasPendingApproval: approved.has(row.id), + stalled: stalled.get(row.id)?.state === "stalled", + })); + } + } + return result; + } + + async function eligible(database: Db, candidate: Candidate, variant?: "owner" | "terminal", lock = false, reviewAttention?: Map, interactionAttention?: Map): Promise { let sourceId = candidate.sourceId; let activityAt = candidate.activityAt; let dueAt = candidate.deadlineAt; @@ -83,38 +146,14 @@ export function attentionRetentionService(db: Db) { .where(and(eq(decisions.companyId, candidate.companyId), eq(decisions.id, sourceId), eq(decisions.status, "open"))); if (!row || row.originIssueId && !await visibleIssue(database, candidate.companyId, row.originIssueId)) return null; } else if (candidate.sourceKind === "issue_thread_interaction") { - const [row] = await database.select().from(issueThreadInteractions).where(and( - eq(issueThreadInteractions.companyId, candidate.companyId), eq(issueThreadInteractions.id, sourceId), - eq(issueThreadInteractions.status, "pending"), isNull(issueThreadInteractions.addresseeUserId))); - if (!row || !await visibleIssue(database, candidate.companyId, row.issueId, true)) return null; - const roster = row.addresseeAgentId ? await database.select().from(agents).where(eq(agents.companyId, candidate.companyId)) : []; - if (row.addresseeAgentId && evaluateAgentInvokability(roster.find(agent => agent.id === row.addresseeAgentId), roster).invokable) return null; - if (row.kind === "request_confirmation") { - const siblings = await database.select().from(issueThreadInteractions).where(and( - eq(issueThreadInteractions.companyId, candidate.companyId), eq(issueThreadInteractions.issueId, row.issueId), - eq(issueThreadInteractions.status, "pending"), eq(issueThreadInteractions.kind, "request_confirmation"), - isNull(issueThreadInteractions.addresseeUserId))); - const companyAgents = roster.length ? roster : await database.select().from(agents).where(eq(agents.companyId, candidate.companyId)); - const board = siblings.filter(sibling => !sibling.addresseeAgentId - || !evaluateAgentInvokability(companyAgents.find(agent => agent.id === sibling.addresseeAgentId), companyAgents).invokable); - if (!collapsePendingConfirmationsToNewest(board).some(sibling => sibling.id === sourceId)) return null; - } + const attention = interactionAttention ?? await interactionEligibility(database, [candidate]); + if (!attention.get(sourceId)) return null; } else if (candidate.sourceKind === "recovery_action") { const [row] = await database.select().from(issueRecoveryActions).where(and(eq(issueRecoveryActions.companyId, candidate.companyId), eq(issueRecoveryActions.id, sourceId))); if (!row || !await visibleIssue(database, candidate.companyId, row.sourceIssueId)) return null; } else if (candidate.sourceKind === "review") { - const row = await visibleIssue(database, candidate.companyId, sourceId); - if (!row) return null; - const state = parseIssueExecutionState(row.executionState); - const [approval] = await database.select({ id: approvals.id }).from(issueApprovals).innerJoin(approvals, eq(approvals.id, issueApprovals.approvalId)) - .where(and(eq(issueApprovals.companyId, candidate.companyId), eq(approvals.companyId, candidate.companyId), - eq(issueApprovals.issueId, sourceId), eq(approvals.status, "pending"))).limit(1); - if (!requiresReviewAttention({ hasHumanParticipant: state?.status === "pending" && state.currentParticipant?.type === "user", - hasUserAssignee: Boolean(row.assigneeUserId), hasPendingApproval: Boolean(approval), stalled: false })) { - const attention = await issueService(database).listReviewAttention(candidate.companyId, [row]); - if (!requiresReviewAttention({ hasHumanParticipant: false, hasUserAssignee: false, hasPendingApproval: false, - stalled: attention.get(sourceId)?.state === "stalled" })) return null; - } + const attention = reviewAttention ?? await reviewEligibility(database, [candidate]); + if (!attention.get(sourceId)) return null; } else if (candidate.sourceKind === "blocker_attention") { const row = await visibleIssue(database, candidate.companyId, sourceId); if (!row) return null; @@ -157,10 +196,12 @@ export function attentionRetentionService(db: Db) { const key = JSON.stringify(scope ?? {}); const rows = await candidates(db, scope, cursors.get(key)); const entries: Entry[] = []; + const reviews = await reviewEligibility(db, rows); + const interactions = await interactionEligibility(db, rows); for (const row of rows) { if (signal?.aborted) return { rows: null, complete: true }; for (const variant of row.sourceKind === "blocker_attention" ? ["owner", "terminal"] as const : [undefined]) { - const entry = await eligible(db, row, variant); + const entry = await eligible(db, row, variant, false, reviews, interactions); if (entry) entries.push(entry); } } diff --git a/server/src/services/attention-source-eligibility.ts b/server/src/services/attention-source-eligibility.ts index 6e3f484c03a..f129c522130 100644 --- a/server/src/services/attention-source-eligibility.ts +++ b/server/src/services/attention-source-eligibility.ts @@ -1,6 +1,10 @@ import { and, eq, gt, inArray, sql, type SQL } from "drizzle-orm"; import { heartbeatRuns, type Db } from "@paperclipai/db"; +export const HUMAN_RECOVERY_OWNER_TYPES = ["user", "board"] as const; +export const ATTENTION_BUDGET_WARNING_PERCENT = 85; +export const ATTENTION_AGENT_ERROR_STATUS = "error"; + export function collapsePendingConfirmationsToNewest = { const PENDING_INTERACTION_STATUSES = ["pending"] as const; const OPEN_RECOVERY_STATUSES = ["active", "escalated"] as const; -const HUMAN_RECOVERY_OWNER_TYPES = ["user", "board"] as const; const DETAIL_EXCERPT_LENGTH = 160; const DETAIL_IMAGE_LIMIT = 3; const OPEN_DECISION_DEFAULT_LIMIT = 500; @@ -1762,7 +1761,7 @@ export function attentionService(db: Db, serviceOptions: AttentionServiceOptions const budgetOverview = await budgetService(db).overview(companyId); for (const incident of budgetOverview.activeIncidents) { const observedPercent = budgetObservedPercent(incident.amountObserved, incident.amountLimit); - if (incident.thresholdType !== "hard" && observedPercent < 85) continue; + if (incident.thresholdType !== "hard" && observedPercent < ATTENTION_BUDGET_WARNING_PERCENT) continue; const dedupKey = `budget:${incident.policyId}:${toIso(incident.windowStart)}:${incident.thresholdType}`; add(createItem({ companyId, @@ -1825,7 +1824,7 @@ export function attentionService(db: Db, serviceOptions: AttentionServiceOptions updatedAt: agents.updatedAt, }) .from(agents) - .where(and(eq(agents.companyId, companyId), eq(agents.status, "error"))) + .where(and(eq(agents.companyId, companyId), eq(agents.status, ATTENTION_AGENT_ERROR_STATUS))) .orderBy(desc(agents.updatedAt), desc(agents.id)); for (const agent of erroredAgents) { diff --git a/server/src/services/budgets.ts b/server/src/services/budgets.ts index 93403926772..c9cedb7f7ee 100644 --- a/server/src/services/budgets.ts +++ b/server/src/services/budgets.ts @@ -505,8 +505,8 @@ export function budgetServiceInTransaction(db: Db, publications: ActivityPublica return incident ? { incident, created: true } : null; } - async function resolveOpenSoftIncidents(policyId: string) { - await notifyAttentionRetentionWork(db, { sourceKind: "budget_alert" }); + async function resolveOpenSoftIncidents(policyId: string, companyId: string) { + await notifyAttentionRetentionWork(db, { companyId, sourceKind: "budget_alert" }); await db .update(budgetIncidents) .set({ @@ -525,11 +525,12 @@ export function budgetServiceInTransaction(db: Db, publications: ActivityPublica async function resolveOpenIncidentsForPolicy( policyId: string, + companyId: string, approvalStatus: "approved" | "rejected" | null, decidedByUserId: string | null, thresholdType?: BudgetThresholdType, ) { - await notifyAttentionRetentionWork(db, { sourceKind: "budget_alert" }); + await notifyAttentionRetentionWork(db, { companyId, sourceKind: "budget_alert" }); const condition = and(eq(budgetIncidents.policyId, policyId), eq(budgetIncidents.status, "open"), thresholdType ? eq(budgetIncidents.thresholdType, thresholdType) : undefined); const openRows = await db @@ -618,9 +619,9 @@ export function budgetServiceInTransaction(db: Db, publications: ActivityPublica if (policy.isActive && policy.hardStopEnabled && policy.amount > 0 && observed.recoveringRunCount > 0) recoveringPolicy ??= policy; // A closed UTC window, inactive policy, or budget raise releases only // budget-owned pauses. Manual pauses remain operator decisions. - await resolveOpenIncidentsForPolicy(policy.id, "approved", "budget_service", "hard"); + await resolveOpenIncidentsForPolicy(policy.id, policy.companyId, "approved", "budget_service", "hard"); if (!policy.isActive || !policy.notifyEnabled || compareCents(observed.totalExact, Math.ceil(policy.amount * policy.warnPercent / 100)) < 0) { - await resolveOpenIncidentsForPolicy(policy.id, "approved", "budget_service", "soft"); + await resolveOpenIncidentsForPolicy(policy.id, policy.companyId, "approved", "budget_service", "soft"); } } // Agent status has its own admission gate and error code. Preserve a @@ -677,7 +678,7 @@ export function budgetServiceInTransaction(db: Db, publications: ActivityPublica } if (observedBlocks(policy, observed)) { - await resolveOpenSoftIncidents(policy.id); + await resolveOpenSoftIncidents(policy.id, policy.companyId); const hardIncident = await createIncidentIfNeeded(policy, "hard", observed); await pauseAndCancelScopeForBudget(policy); if (hardIncident?.created) { @@ -847,11 +848,11 @@ export function budgetServiceInTransaction(db: Db, publications: ActivityPublica const observed = await computeObservedSpend(db, row); if (!observedBlocks(row, observed)) { await resumeScopeFromBudget(row); - await resolveOpenIncidentsForPolicy(row.id, "approved", actorUserId ?? "budget_service", "hard"); + await resolveOpenIncidentsForPolicy(row.id, row.companyId, "approved", actorUserId ?? "budget_service", "hard"); if (row.notifyEnabled && compareCents(observed.totalExact, Math.ceil(row.amount * row.warnPercent / 100)) >= 0) { await createIncidentIfNeeded(row, "soft", observed); } else { - await resolveOpenIncidentsForPolicy(row.id, null, null, "soft"); + await resolveOpenIncidentsForPolicy(row.id, row.companyId, null, null, "soft"); } } else { const softThreshold = Math.ceil((row.amount * row.warnPercent) / 100); @@ -859,14 +860,14 @@ export function budgetServiceInTransaction(db: Db, publications: ActivityPublica await createIncidentIfNeeded(row, "soft", observed); } if (observedBlocks(row, observed)) { - await resolveOpenSoftIncidents(row.id); + await resolveOpenSoftIncidents(row.id, row.companyId); await createIncidentIfNeeded(row, "hard", observed); await pauseAndCancelScopeForBudget(row); } } } else { await resumeScopeFromBudget(row); - await resolveOpenIncidentsForPolicy(row.id, "approved", actorUserId ?? "budget_service"); + await resolveOpenIncidentsForPolicy(row.id, row.companyId, "approved", actorUserId ?? "budget_service"); } await recordActivity({ diff --git a/server/src/services/decision-expiry.ts b/server/src/services/decision-expiry.ts index ea5236a6ac0..796200128a7 100644 --- a/server/src/services/decision-expiry.ts +++ b/server/src/services/decision-expiry.ts @@ -23,7 +23,7 @@ export function decisionExpiryService(db: Db, deliverContinuation: (id: string) agentId: row.originAgentId, runId: row.originRunId, action: "decision.expired", entityType: "decision", entityId: row.id, details: { expiredReason: reason } }, publications); return row; - }); + }, { companyId: decision.companyId, sourceId: decision.id }); if (!updated) return 0; for (const publication of publications) publishActivity(publication); await deliverContinuation(updated.id); diff --git a/server/src/services/decision-work-notifications.ts b/server/src/services/decision-work-notifications.ts index 0abc8e95829..9e2ca840960 100644 --- a/server/src/services/decision-work-notifications.ts +++ b/server/src/services/decision-work-notifications.ts @@ -2,20 +2,22 @@ import { notifyAttentionRetentionWork } from "./retention-work-notifications.js" import type { Db } from "@paperclipai/db"; import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js"; -export async function notifyDecisionCreated(transaction: object) { - await notifyAttentionRetentionWork(transaction, { sourceKind: "decision" }); +type DecisionWorkScope = { companyId: string; sourceId?: string }; + +export async function notifyDecisionCreated(transaction: object, scope: DecisionWorkScope) { + await notifyAttentionRetentionWork(transaction, { ...scope, sourceKind: "decision" }); return notifyDeliveryWork(transaction, DELIVERY_QUEUES.decisionExpiry); } -export async function notifyDecisionChange(transaction: object) { - await notifyAttentionRetentionWork(transaction, { sourceKind: "decision" }); +export async function notifyDecisionChange(transaction: object, scope: DecisionWorkScope) { + await notifyAttentionRetentionWork(transaction, { ...scope, sourceKind: "decision" }); await notifyDeliveryWork(transaction, DELIVERY_QUEUES.decisionContinuation); await notifyDeliveryWork(transaction, DELIVERY_QUEUES.decisionExpiry); } -export function writeDecisionChange(db: Db, write: (transaction: Db) => Promise): Promise { +export function writeDecisionChange(db: Db, write: (transaction: Db) => Promise, scope: DecisionWorkScope): Promise { return db.transaction(async transaction => { - await notifyDecisionChange(transaction); + await notifyDecisionChange(transaction, scope); await notifyDeliveryWork(transaction, DELIVERY_QUEUES.decisionEffectRecovery); return write(transaction as unknown as Db); }); diff --git a/server/src/services/decisions.ts b/server/src/services/decisions.ts index 282327d433f..8a84743fcef 100644 --- a/server/src/services/decisions.ts +++ b/server/src/services/decisions.ts @@ -239,7 +239,7 @@ export function decisionService(db: Db, options: DecisionServiceOptions) { ...(input.additionalTargetSnapshots ?? {}), }; const id = randomUUID(); - await notifyDecisionCreated(dbOrTx); + await notifyDecisionCreated(dbOrTx, { companyId: input.companyId, sourceId: id }); await notifyDecisionTargetWork(dbOrTx, `decision:${id}`); const [created] = await dbOrTx.insert(decisions).values({ id, companyId: input.companyId, bundleId: input.bundleId ?? null, originAgentId: input.agentId, originIssueId: provenance.issueId, originRunId: input.runId, ruleKey: input.ruleKey ?? null, @@ -581,7 +581,8 @@ export function decisionService(db: Db, options: DecisionServiceOptions) { return attempt; } - async function settleInvalidEffectOutcome(id: string) { + async function settleInvalidEffectOutcome(decision: typeof decisions.$inferSelect) { + const id = decision.id; const publications: ActivityPublication[] = []; await writeDecisionChange(db, async tx => { const [current] = await tx.select().from(decisions).where(eq(decisions.id, id)).for("update"); @@ -601,14 +602,14 @@ export function decisionService(db: Db, options: DecisionServiceOptions) { action: "decision.recovery_failed", entityType: "decision", entityId: id, details: { reason, executionStatus, chosenOptionId: current.chosenOptionId, decidedByUserId: current.decidedByUserId }, }, publications); - }); + }, { companyId: decision.companyId, sourceId: id }); for (const publication of publications) publishActivity(publication); return outcome(id); } async function executeDecisionEffects(decision: typeof decisions.$inferSelect, userActor: AuthorizationActor) { const option = decision.options.find((item) => item.id === decision.chosenOptionId); - if (!option || !decision.decidedByUserId) return settleInvalidEffectOutcome(decision.id); + if (!option || !decision.decidedByUserId) return settleInvalidEffectOutcome(decision); const run = await db.select({ responsibleUserId: heartbeatRuns.responsibleUserId }).from(heartbeatRuns).where(eq(heartbeatRuns.id, decision.originRunId)).then((rows) => rows[0] ?? null); const metadata = decision.metadata as Record; if (metadata.kind === "attention_archive_proposal") { @@ -618,7 +619,7 @@ export function decisionService(db: Db, options: DecisionServiceOptions) { targetSnapshots: decision.targetSnapshots as Record, }), decision.signedSpec)) { await writeDecisionChange(db, async tx => tx.update(decisions).set({ executionStatus: "failed", updatedAt: new Date(), metadata: { ...metadata, archiveProposalError: "invalid_signature", ...(decision.continuationPolicy === "wake_origin_agent" ? { continuationPending: true } : {}) } }) - .where(and(eq(decisions.id, decision.id), eq(decisions.executionStatus, "running")))); + .where(and(eq(decisions.id, decision.id), eq(decisions.executionStatus, "running"))), { companyId: decision.companyId, sourceId: decision.id }); return outcome(decision.id); } if (option.id === "archive") { @@ -629,7 +630,7 @@ export function decisionService(db: Db, options: DecisionServiceOptions) { const manifestHash = manifest ? hashAttentionArchiveManifest(manifest) : null; if (!manifest || manifestHash !== metadata.manifestHash) { await writeDecisionChange(db, async tx => tx.update(decisions).set({ executionStatus: "failed", updatedAt: new Date(), metadata: { ...metadata, archiveProposalError: "manifest_mismatch", ...(decision.continuationPolicy === "wake_origin_agent" ? { continuationPending: true } : {}) } }) - .where(and(eq(decisions.id, decision.id), eq(decisions.executionStatus, "running")))); + .where(and(eq(decisions.id, decision.id), eq(decisions.executionStatus, "running"))), { companyId: decision.companyId, sourceId: decision.id }); return outcome(decision.id); } const originActor: AuthorizationActor = { @@ -651,7 +652,7 @@ export function decisionService(db: Db, options: DecisionServiceOptions) { } catch (error) { const message = error instanceof Error ? error.message : "archive_proposal_failed"; await writeDecisionChange(db, async tx => tx.update(decisions).set({ executionStatus: "failed", updatedAt: new Date(), metadata: { ...metadata, archiveProposalError: message, ...(decision.continuationPolicy === "wake_origin_agent" ? { continuationPending: true } : {}) } }) - .where(and(eq(decisions.id, decision.id), eq(decisions.executionStatus, "running")))); + .where(and(eq(decisions.id, decision.id), eq(decisions.executionStatus, "running"))), { companyId: decision.companyId, sourceId: decision.id }); return outcome(decision.id); } } @@ -659,7 +660,7 @@ export function decisionService(db: Db, options: DecisionServiceOptions) { executionStatus: "succeeded", updatedAt: new Date(), metadata: { ...metadata, ...(decision.continuationPolicy === "wake_origin_agent" ? { continuationPending: true } : {}) }, - }).where(and(eq(decisions.id, decision.id), eq(decisions.executionStatus, "running")))); + }).where(and(eq(decisions.id, decision.id), eq(decisions.executionStatus, "running"))), { companyId: decision.companyId, sourceId: decision.id }); return outcome(decision.id); } for (let index = 0; index < option.effects.length; index += 1) await executeEffect(decision, option.effects[index]!, index, userActor, decision.decidedByUserId, run?.responsibleUserId ?? null); @@ -667,7 +668,7 @@ export function decisionService(db: Db, options: DecisionServiceOptions) { const successful = rows.filter((row) => row.status === "executed").length; const status = rows.every((row) => row.status === "executed") ? "succeeded" : successful ? "partial" : "failed"; await writeDecisionChange(db, async tx => tx.update(decisions).set({ executionStatus: status, updatedAt: new Date(), metadata: { ...decision.metadata, - ...(decision.continuationPolicy === "wake_origin_agent" ? { continuationPending: true } : {}) } }).where(and(eq(decisions.id, decision.id), eq(decisions.executionStatus, "running")))); + ...(decision.continuationPolicy === "wake_origin_agent" ? { continuationPending: true } : {}) } }).where(and(eq(decisions.id, decision.id), eq(decisions.executionStatus, "running"))), { companyId: decision.companyId, sourceId: decision.id }); return outcome(decision.id); } @@ -706,7 +707,7 @@ export function decisionService(db: Db, options: DecisionServiceOptions) { decideIdempotencyKey: input.idempotencyKey ?? null, ...(current.continuationPolicy === "wake_origin_agent" ? { continuationPending: true } : {}), ...(input.dismissed ? { dismissed: true, dismissReason: input.dismissReason ?? null } : {}) } }) - .where(and(eq(decisions.id, current.id), eq(decisions.status, "open"))).returning()); + .where(and(eq(decisions.id, current.id), eq(decisions.status, "open"))).returning(), { companyId: current.companyId, sourceId: current.id }); if (!claimed) throw conflict("decision_already_resolved", { code: "decision_already_resolved" }); const run = await db.select({ responsibleUserId: heartbeatRuns.responsibleUserId }).from(heartbeatRuns).where(eq(heartbeatRuns.id, claimed.originRunId)).then((rows) => rows[0] ?? null); await logActivity(db, { companyId: claimed.companyId, actorType: "system", actorId: "decision-executor", agentId: claimed.originAgentId, runId: claimed.originRunId, @@ -724,7 +725,7 @@ export function decisionService(db: Db, options: DecisionServiceOptions) { return (await get(id))!; } const [updated] = await writeDecisionChange(db, async tx => tx.update(decisions).set({ status: "cancelled", updatedAt: new Date(), metadata: { ...current.metadata, - ...(current.continuationPolicy === "wake_origin_agent" ? { continuationPending: true } : {}) } }).where(and(eq(decisions.id, id), eq(decisions.status, "open"))).returning()); + ...(current.continuationPolicy === "wake_origin_agent" ? { continuationPending: true } : {}) } }).where(and(eq(decisions.id, id), eq(decisions.status, "open"))).returning(), { companyId: current.companyId, sourceId: id }); if (!updated) throw conflict("decision_already_resolved", { code: "decision_already_resolved" }); await logActivity(db, { companyId: updated.companyId, actorType: actor.actorType, actorId: actor.actorId, runId: actor.runId, action: "decision.cancelled", entityType: "decision", entityId: id }); await continuations.deliver(updated.id); @@ -738,7 +739,7 @@ export function decisionService(db: Db, options: DecisionServiceOptions) { if (empty) return decide({ id, optionId: empty.id, decidedByUserId: userId, userActor, dismissed: true, dismissReason: reason }); const [updated] = await writeDecisionChange(db, async tx => tx.update(decisions).set({ status: "decided", executionStatus: "succeeded", chosenOptionId: "dismissed", decidedByUserId: userId, decidedAt: new Date(), updatedAt: new Date(), metadata: { ...current.metadata, dismissed: true, dismissReason: reason ?? null, - ...(current.continuationPolicy === "wake_origin_agent" ? { continuationPending: true } : {}) } }).where(and(eq(decisions.id, id), eq(decisions.status, "open"))).returning()); + ...(current.continuationPolicy === "wake_origin_agent" ? { continuationPending: true } : {}) } }).where(and(eq(decisions.id, id), eq(decisions.status, "open"))).returning(), { companyId: current.companyId, sourceId: id }); if (!updated) throw conflict("decision_already_resolved", { code: "decision_already_resolved" }); await logActivity(db, { companyId: updated.companyId, actorType: "system", actorId: "decision-executor", agentId: updated.originAgentId, runId: updated.originRunId, responsibleUserIdOverride: userId, action: "decision.dismissed", entityType: "decision", entityId: updated.id, diff --git a/server/src/services/heartbeat-run-events.ts b/server/src/services/heartbeat-run-events.ts index 22219811668..ced43f5376f 100644 --- a/server/src/services/heartbeat-run-events.ts +++ b/server/src/services/heartbeat-run-events.ts @@ -10,6 +10,8 @@ import type { Db } from "@paperclipai/db"; import { heartbeatRunEvents, heartbeatRuns } from "@paperclipai/db"; import { nativeSha256 } from "./native-runtime/canonical.js"; +export const RETRY_EXHAUSTION_MESSAGE_PREFIX = "Bounded retry exhausted"; + export interface AppendHeartbeatRunEventInput { companyId: string; runId: string; @@ -75,7 +77,7 @@ export async function appendHeartbeatRunEvent( db: Db, input: AppendHeartbeatRunEventInput, ): Promise { - const projectsRetention = input.eventType === "lifecycle" && Boolean(input.message?.startsWith("Bounded retry exhausted")); + const projectsRetention = input.eventType === "lifecycle" && Boolean(input.message?.startsWith(RETRY_EXHAUSTION_MESSAGE_PREFIX)); const projectsReassessment = readNativeEventEvidenceOutcome(input.payload?.prpEvent) !== null; const projectsProgress = isSafeNativeChatProgressEventType(input.eventType); const projectsResponse = isNativeResponseEvidence(input.eventType, input.payload?.prpEvent); @@ -131,7 +133,7 @@ export async function appendHeartbeatRunEvent( eq(heartbeatRunEvents.runId, input.runId), eq(heartbeatRunEvents.agentId, input.agentId), eq(heartbeatRunEvents.eventType, "lifecycle"), - sql`${heartbeatRunEvents.message} like 'Bounded retry exhausted%'`, + sql`${heartbeatRunEvents.message} like ${`${RETRY_EXHAUSTION_MESSAGE_PREFIX}%`}`, sql`${heartbeatRunEvents.payload} @> ${JSON.stringify(input.retryExhaustion)}::jsonb`, )) .limit(1) diff --git a/server/src/services/issues.ts b/server/src/services/issues.ts index 4ee7e681907..7239075de99 100644 --- a/server/src/services/issues.ts +++ b/server/src/services/issues.ts @@ -7598,7 +7598,7 @@ export function issueService(db: Db) { companyId: string, blockedByIssueIds: string[], actor: { agentId?: string | null; userId?: string | null } = {}, - dbOrTx: any = db, + dbOrTx: any, ) { const deduped = [...new Set(blockedByIssueIds)]; if (deduped.some((candidate) => candidate === issueId)) { @@ -9676,27 +9676,32 @@ export function issueService(db: Db) { }); if (blockParentUntilDone) { - const existingBlockers = await db - .select({ blockerIssueId: issueRelations.issueId }) - .from(issueRelations) - .where( - and( - eq(issueRelations.companyId, parent.companyId), - eq(issueRelations.relatedIssueId, parent.id), - eq(issueRelations.type, "blocks"), - ), + await db.transaction(async tx => { + await tx.select({ id: issues.id }).from(issues).where(and( + eq(issues.companyId, parent.companyId), eq(issues.id, parent.id))).for("update"); + const existingBlockers = await tx + .select({ blockerIssueId: issueRelations.issueId }) + .from(issueRelations) + .where( + and( + eq(issueRelations.companyId, parent.companyId), + eq(issueRelations.relatedIssueId, parent.id), + eq(issueRelations.type, "blocks"), + ), + ); + await syncBlockedByIssueIds( + parent.id, + parent.companyId, + [ + ...new Set([ + ...existingBlockers.map((row) => row.blockerIssueId), + child.id, + ]), + ], + { agentId: actorAgentId ?? null, userId: actorUserId ?? null }, + tx, ); - await syncBlockedByIssueIds( - parent.id, - parent.companyId, - [ - ...new Set([ - ...existingBlockers.map((row) => row.blockerIssueId), - child.id, - ]), - ], - { agentId: actorAgentId ?? null, userId: actorUserId ?? null }, - ); + }); [child] = await withIssueRelationSummaries( parent.companyId, [child], From 65602133e7cf2b9e6a0eb55171816a85526825f0 Mon Sep 17 00:00:00 2001 From: Dotta Date: Fri, 30 Oct 2026 17:10:16 -0500 Subject: [PATCH 25/70] perf(heartbeat): let the dispatch worker own startup retry promotion Co-Authored-By: Paperclip --- server/src/__tests__/server-startup-feedback-export.test.ts | 2 +- server/src/index.ts | 4 +--- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/server/src/__tests__/server-startup-feedback-export.test.ts b/server/src/__tests__/server-startup-feedback-export.test.ts index ec7f09bc3ac..e00c8519df7 100644 --- a/server/src/__tests__/server-startup-feedback-export.test.ts +++ b/server/src/__tests__/server-startup-feedback-export.test.ts @@ -1341,7 +1341,7 @@ describe("startServer feedback export wiring", () => { expect(heartbeatServiceMock.sweepStaleIssueLocks).toHaveBeenCalledTimes(2); expect(heartbeatServiceMock.reapOrphanedRuns).toHaveBeenLastCalledWith({ staleThresholdMs: 300_000, skipScheduledRetryCancellation: true, skipCapturedInstructionCommit: true, skipInstructionDirectoryCleanup: true, skipStoppedInstructionCollection: true, skipCancelledWorkspacePreservation: true, skipObsoletePolicyReviews: true, skipAutomaticCompletionReviews: true, skipUnsafeWorkspaceExports: true, skipNativePendingEffects: true, skipNativeResponsePresentation: true, skipNativeReviewResponse: true, skipNativeStatusReassessment: true, skipRetainedNativeSessionCleanup: true, skipNativeWorkspaceFinalization: true, skipNativeResultCommit: true, skipNativeRunProjection: true, skipNativeRecoveryNotices: true, skipExternalConversationSettlement: true, skipExternalAgentSettlement: true, skipNativeEnvironmentSettlement: true }); expect(heartbeatServiceMock.reconcileCostAccounting).toHaveBeenCalledTimes(1); - expect(heartbeatServiceMock.promoteDueScheduledRetries).toHaveBeenCalledTimes(1); + expect(heartbeatServiceMock.promoteDueScheduledRetries).not.toHaveBeenCalled(); expect(retiredDetector).not.toHaveBeenCalled(); } finally { delete (runtime as Partial).reconcileProductivityReviews; diff --git a/server/src/index.ts b/server/src/index.ts index 879ac991182..cd8bb951946 100644 --- a/server/src/index.ts +++ b/server/src/index.ts @@ -1778,7 +1778,6 @@ async function startServerWithDatabaseTeardown( } } - const promotion = await timeStartupRecovery("scheduled_retries", () => heartbeat.promoteDueScheduledRetries()); await timeStartupRecovery("queued_runs", () => heartbeat.resumeQueuedRuns()); const recoveredGoalActions = await timeStartupRecovery("goal_actions", () => heartbeat.recoverPendingSessionGoalActions()); if ( @@ -1799,7 +1798,6 @@ async function startServerWithDatabaseTeardown( } const reconciled = await heartbeat.reconcileStrandedAssignedIssues(); if ( - promotion.promoted > 0 || reconciled.assignmentDispatched > 0 || reconciled.dispatchRequeued > 0 || reconciled.continuationRequeued > 0 || @@ -1808,7 +1806,7 @@ async function startServerWithDatabaseTeardown( reconciled.escalated > 0 ) { logger.warn( - { promotedScheduledRetries: promotion.promoted, promotedScheduledRetryRunIds: promotion.runIds, ...reconciled }, + { ...reconciled }, "startup heartbeat recovery changed assigned issue state", ); } From 30730dd61983ebcd1500170bef66df85c296871c Mon Sep 17 00:00:00 2001 From: Dotta Date: Fri, 30 Oct 2026 16:54:51 -0500 Subject: [PATCH 26/70] test(skills): verify Cloud CLI reads through shared object storage Use the real S3 provider with a controlled HTTP fixture and a stable test key. Preserve read-only/no-seeding behavior and verify corrupt, missing, and restored object outcomes under managed Cloud policy. Co-Authored-By: Paperclip --- .../__tests__/managed-skill-content.test.ts | 90 ++++++++++++++++--- 1 file changed, 77 insertions(+), 13 deletions(-) diff --git a/server/src/__tests__/managed-skill-content.test.ts b/server/src/__tests__/managed-skill-content.test.ts index cd4ddb8135f..59566d743e3 100644 --- a/server/src/__tests__/managed-skill-content.test.ts +++ b/server/src/__tests__/managed-skill-content.test.ts @@ -1,3 +1,5 @@ +import { createServer } from "node:http"; +import { createS3StorageProvider } from "../storage/s3-provider.js"; import { skillInstallationMaintenance } from "../services/skill-installation-maintenance.js"; import { removeRuntimeAssetTree } from "../services/native-runtime/runtime-asset-retention.js"; import { eq } from "drizzle-orm"; @@ -79,26 +81,88 @@ describe.skipIf(!support.supported)("managed skills use committed versions", () it("loads persisted Cloud configuration for read-only maintenance without seeding it", async () => { const companyId = await company(); - await companySkillService(db).createLocalSkill(companyId, { name: "Cloud CLI", markdown: "# Cloud CLI" }); + const objects = new Map(); + const server = createServer((request, response) => { + void (async () => { + const key = new URL(request.url!, "http://fixture").pathname; + if (request.method === "PUT") { + const chunks: Buffer[] = []; + for await (const chunk of request) chunks.push(Buffer.from(chunk)); + let body = Buffer.concat(chunks); + if (request.headers["content-encoding"]?.includes("aws-chunked")) { + const decoded: Buffer[] = []; + let offset = 0; + while (offset < body.length) { + const end = body.indexOf("\r\n", offset); + const size = Number.parseInt(body.subarray(offset, end).toString().split(";")[0]!, 16); + if (size === 0) break; + decoded.push(body.subarray(end + 2, end + 2 + size)); offset = end + 2 + size + 2; + } + body = Buffer.concat(decoded); + } + objects.set(key, body); response.writeHead(200, { etag: '"fixture"' }).end(); return; + } + const body = objects.get(key); + if (!body) { response.writeHead(404).end(); return; } + response.writeHead(200, { "content-length": body.length, "content-type": "application/octet-stream", etag: '"fixture"' }); + response.end(request.method === "HEAD" ? undefined : body); + })().catch(() => { response.writeHead(500).end(); }); + }); + await new Promise(resolve => { server.listen(0, "127.0.0.1", resolve); }); + const address = server.address(); if (!address || typeof address === "string") throw new Error("Expected fixture address"); + const endpoint = `http://127.0.0.1:${address.port}`; + const credentials = { AWS_ACCESS_KEY_ID: "fixture-access-key", AWS_SECRET_ACCESS_KEY: "fixture-secret-key", AWS_SESSION_TOKEN: "" }; + const previousCredentials = Object.fromEntries(Object.keys(credentials).map(key => [key, process.env[key]])); + Object.assign(process.env, credentials); + vi.mocked(storageModule.getStorageService).mockReturnValue(createStorageService(createS3StorageProvider({ + bucket: "managed-skill-fixture", region: "us-east-1", endpoint, forcePathStyle: true, + }))); const key = "cloud-runtime-config:v1"; - const run = (phase: string) => promisify(execFile)(process.execPath, [ - "--import", fileURLToPath(import.meta.resolve("tsx")), - fileURLToPath(new URL("../../scripts/migrate-skill-packages.ts", import.meta.url)), - `--phase=${phase}`, `--company=${companyId}`, "--limit=50", - ], { env: { ...process.env, DATABASE_URL: database.connectionString, - PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN: "disposable-cloud-cli-fixture", - PAPERCLIP_STORAGE_PROVIDER: "local_disk", PAPERCLIP_STORAGE_LOCAL_DIR: path.join(root, "objects") }, - timeout: 60_000, maxBuffer: 1024 * 1024 }); - await expect(run("eligibility")).rejects.toThrow("Initialize Cloud runtime configuration"); - expect(await db.select().from(instanceSettings).where(eq(instanceSettings.singletonKey, key))).toHaveLength(0); - await db.insert(instanceSettings).values({ singletonKey: key, general: { version: 1, revision: 0, config: null } }); try { + await companySkillService(db).createLocalSkill(companyId, { name: "Cloud CLI", markdown: "# Cloud CLI" }); + const run = (phase: string) => promisify(execFile)(process.execPath, [ + "--import", fileURLToPath(import.meta.resolve("tsx")), + fileURLToPath(new URL("../../scripts/migrate-skill-packages.ts", import.meta.url)), + `--phase=${phase}`, `--company=${companyId}`, "--limit=50", + ], { env: { ...process.env, ...credentials, DATABASE_URL: database.connectionString, + PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN: "disposable-cloud-cli-fixture", + PAPERCLIP_SECRETS_MASTER_KEY: "a".repeat(64), PAPERCLIP_STORAGE_PROVIDER: "s3", + PAPERCLIP_STORAGE_S3_BUCKET: "managed-skill-fixture", PAPERCLIP_STORAGE_S3_REGION: "us-east-1", + PAPERCLIP_STORAGE_S3_ENDPOINT: endpoint, PAPERCLIP_STORAGE_S3_FORCE_PATH_STYLE: "true", PAPERCLIP_STORAGE_S3_PREFIX: "" }, + timeout: 60_000, maxBuffer: 1024 * 1024 }); + await expect(run("eligibility")).rejects.toThrow("Initialize Cloud runtime configuration"); + expect(await db.select().from(instanceSettings).where(eq(instanceSettings.singletonKey, key))).toHaveLength(0); + await db.insert(instanceSettings).values({ singletonKey: key, general: { version: 1, revision: 0, config: null } }); const eligibility = JSON.parse((await run("eligibility")).stdout.trim().split("\n").at(-1)!); expect(eligibility.results).toEqual([expect.objectContaining({ status: "version_backed" })]); + const before = [...objects.entries()].map(([key, body]) => [key, Buffer.from(body)] as const); const verification = JSON.parse((await run("verification")).stdout.trim().split("\n").at(-1)!); - expect(verification.results).toEqual([expect.objectContaining({ status: "local_storage" })]); + expect(verification.results).toEqual([expect.objectContaining({ status: "remote_storage" })]); + expect([...objects.entries()]).toEqual(before); + const archiveKey = [...objects.keys()].find(key => key.endsWith("/files.tar.gz"))!; + const archive = objects.get(archiveKey)!; + const corrupted = Buffer.from(archive); corrupted[0] = corrupted[0]! ^ 1; + objects.set(archiveKey, corrupted); + const corrupt = JSON.parse((await run("verification")).stdout.trim().split("\n").at(-1)!); + expect(corrupt.results).toEqual([expect.objectContaining({ status: "corrupt_package" })]); + objects.set(archiveKey, archive); + const manifestKey = [...objects.keys()].find(key => key.endsWith("/manifest.json"))!; + const manifest = objects.get(manifestKey)!; objects.delete(manifestKey); + const unavailable = JSON.parse((await run("verification")).stdout.trim().split("\n").at(-1)!); + expect(unavailable.results).toEqual([expect.objectContaining({ status: "unavailable_or_corrupt_package" })]); + objects.set(manifestKey, manifest); + const recovered = JSON.parse((await run("verification")).stdout.trim().split("\n").at(-1)!); + expect(recovered.results).toEqual([expect.objectContaining({ status: "remote_storage" })]); + expect([...objects.entries()].sort()).toEqual([...before].sort()); + expect(await db.select().from(instanceSettings).where(eq(instanceSettings.singletonKey, key))) + .toEqual([expect.objectContaining({ general: { version: 1, revision: 0, config: null } })]); } finally { await db.delete(instanceSettings).where(eq(instanceSettings.singletonKey, key)); + vi.mocked(storageModule.getStorageService).mockReturnValue(createStorageService(createLocalDiskStorageProvider(path.join(root, "objects")))); + for (const [key, value] of Object.entries(previousCredentials)) { + if (value === undefined) delete process.env[key]; else process.env[key] = value; + } + server.closeAllConnections(); await new Promise(resolve => { server.close(() => resolve()); }); } }, 180_000); From b45b6a225d8a9983ace61cca426f0c0c949e2ab7 Mon Sep 17 00:00:00 2001 From: Dotta Date: Fri, 30 Oct 2026 17:18:38 -0500 Subject: [PATCH 27/70] fix(external-objects): bound refresh lifecycle and activation scope Reuse the provider-owned request cancellation contract from merge-confirmation work. Keep host signals out of plugin RPC payloads, preserve refresh debt on shutdown, and target identities or known objects for source writes. Co-Authored-By: Paperclip --- .../2026-10-30-external-objects-sleep.md | 26 +++++- .../external-object-refresh-work.test.ts | 79 +++++++++++++++++-- .../server-startup-feedback-export.test.ts | 22 +++++- .../external-object-refresh-notifications.ts | 6 +- server/src/services/external-objects.ts | 60 ++++++++++---- .../src/services/github-connection-events.ts | 3 +- .../github-external-object-provider.ts | 18 +++-- 7 files changed, 179 insertions(+), 35 deletions(-) diff --git a/doc/plans/2026-10-30-external-objects-sleep.md b/doc/plans/2026-10-30-external-objects-sleep.md index ab566999917..61c7d8dded1 100644 --- a/doc/plans/2026-10-30-external-objects-sleep.md +++ b/doc/plans/2026-10-30-external-objects-sleep.md @@ -24,7 +24,9 @@ probe before their terminal verdict is definitive. A null deadline retains its existing never-refresh meaning. Registration does not change stored timestamps. The worker reconstructs eligible deadlines in pages of at most 250 rows. A -company notification reads that company only. A due callback reads only its +company notification reads that company only. Ordinary ingestion uses object IDs +or stable provider/object/external identity hints, including concurrent upserts, +so one mention write does not reconstruct all company history. A due callback reads only its object. Pages select scheduling columns and retain compact deadlines in memory. Dispatch uses fair batches of at most 50 objects. The deadline is the later of `next_refresh_at` and the current refresh lease expiry. Lease renewals and atomic @@ -46,7 +48,7 @@ and fences automatic result persistence. It does not modify that draft PR. | Mutation | Owner and activation | | --- | --- | -| Issue title/description, comment and document references | Existing sync methods register company intent before mention replacement and object upsert. Routes, runtime tools, adapters and plugin paths reuse these methods. | +| Issue title/description, comment and document references | Existing sync methods register object intent before mention replacement and stable identity intent before object upsert. Routes, runtime tools, adapters and plugin paths reuse these methods. | | Comment/document disappearance | Sync derives the retained mention owners and registers only those companies before deleting obsolete mentions. | | Last mention removal and reattachment | Mention replacement invalidates company deadlines. Retained objects stay available for reattachment. | | Issue deletion | Issue service registers its company before cascading mention deletion. | @@ -90,3 +92,23 @@ The conservative external-object host-sleep blocker remains. An in-process deadline cannot wake stopped compute. This change proves worker quiet and reactivation in the tested process; it does not establish Cloud alarm coverage, host/database suspension or deployed savings. + + +## Host cancellation and explicit legacy entry points + +The resolver contract accepts an optional host `AbortSignal`. The worker passes +its coordinator signal through refresh to the resolver. This reuses the narrow +provider-owned capability introduced with merge-confirmation work: GitHub combines +host cancellation with a ten-second request budget for real fetch, response-body +reads and visibility probes. Plugin RPC payloads remain unchanged. The host +observes cancellation while retaining handlers for any eventual plugin/custom +result; it releases only its own claim and cannot publish that late result. +Startup does not await provider work. Shutdown closes the active GitHub HTTP +request, and its saved due row can resume immediately after restart. + +The two explicit `refreshDueObjects*` helpers remain manual diagnostic/test entry +points. They have no production scheduler caller. Their regression fixtures also +preserve the prior selection/lease semantics. The scheduled check retains its +post-refresh scoped read to revalidate current company, reference and resolver +eligibility and recover an authoritative lease after a competing writer; all such +reads target one object rather than rebuilding a company. diff --git a/server/src/__tests__/external-object-refresh-work.test.ts b/server/src/__tests__/external-object-refresh-work.test.ts index f429c31396b..754725c7d80 100644 --- a/server/src/__tests__/external-object-refresh-work.test.ts +++ b/server/src/__tests__/external-object-refresh-work.test.ts @@ -1,8 +1,9 @@ +import { createServer } from "node:http"; import { randomUUID } from "node:crypto"; import { setTimeout as delay } from "node:timers/promises"; import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest"; import { eq, sql } from "drizzle-orm"; -import { activityLog, approvals, applyPendingMigrations, budgetIncidents, budgetPolicies, companies, costEvents, createDb, externalObjects, issues, issueComments, plugins } from "@paperclipai/db"; +import { activityLog, approvals, applyPendingMigrations, databaseWorkPending, budgetIncidents, budgetPolicies, companies, costEvents, createDb, externalObjects, externalObjectMentions, issues, issueComments, plugins } from "@paperclipai/db"; import { externalObjectService, type ExternalObjectResolveResult } from "../services/external-objects.js"; import type { PaperclipPluginManifestV1 } from "@paperclipai/shared"; import type { PluginWorkerManager } from "../services/plugin-worker-manager.js"; @@ -12,6 +13,7 @@ import { pluginRegistryService } from "../services/plugin-registry.js"; import { companyService } from "../services/companies.js"; import { instanceSettingsService } from "../services/instance-settings.js"; import { notifyExternalObjectRefresh } from "../services/external-object-refresh-notifications.js"; +import { DELIVERY_QUEUES } from "../services/delivery-work-notifications.js"; import { createDeliveryWorkCoordinator } from "../services/delivery-work-coordinator.js"; import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js"; import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; @@ -295,8 +297,12 @@ const active: ExternalObjectResolveResult = { ok: true, snapshot: { key: "notes", format: "markdown", body: `https://example.com/${a.companyId}` }); id = result.document.id; await svc.syncDocument(id); } - await settle(async () => (await row(a.companyId))?.lastResolvedAt !== null && (await row(b.companyId))?.lastResolvedAt !== null); - await settle(() => coordinator!.nextWakeAt() !== null); + await settle(async () => { + const left = await row(a.companyId), right = await row(b.companyId); + return Boolean(left?.lastResolvedAt && right?.lastResolvedAt); + }); + const expectedDeadline = Math.min((await row(a.companyId)).nextRefreshAt!.getTime(), (await row(b.companyId)).nextRefreshAt!.getTime()); + await settle(() => coordinator!.nextWakeAt() === expectedDeadline && !databaseWorkPending(db, DELIVERY_QUEUES.externalObjectRefresh)); const reads: Array<{ query: string; params: unknown[] }> = []; const debug = db.$client.options.debug; db.$client.options.debug = (_connection, query, params) => { @@ -344,7 +350,7 @@ const active: ExternalObjectResolveResult = { ok: true, snapshot: { finish(terminal); await settle(async () => (await row(f.companyId)).isTerminal); await quiet(); }); - it("uses durable provider retry deadlines and waits for in-flight shutdown without overlap", async () => { + it("uses durable provider retry deadlines and cancels the observed result on shutdown without overlap", async () => { clock(); const svc = service(); await start(svc); provider.mockResolvedValueOnce({ ok: false, liveness: "rate_limited", errorCode: "slow_down", retryAfterSeconds: 5 }); const f = await fixture(); await svc.syncIssue(f.issueId); @@ -360,13 +366,74 @@ const active: ExternalObjectResolveResult = { ok: true, snapshot: { await svc.syncIssue(f.issueId); await vi.advanceTimersByTimeAsync(0); expect(provider).toHaveBeenCalledTimes(2); let stopped = false; const stop = coordinator!.stop().then(() => { stopped = true; }); - await delay(10); expect(stopped).toBe(false); - finish(terminal); await stop; coordinator = undefined; + await stop; expect(stopped).toBe(true); coordinator = undefined; + const cancelled = await row(f.companyId); expect(cancelled.refreshToken).toBeNull(); expect(cancelled.isTerminal).toBe(false); + finish(terminal); await delay(10); + expect((await row(f.companyId)).isTerminal).toBe(false); const selects = vi.spyOn(db, "select"); await vi.advanceTimersByTimeAsync(600_000); expect(selects).not.toHaveBeenCalled(); expect(provider).toHaveBeenCalledTimes(2); }); + it("cancels an actual stalled GitHub HTTP body, retains due debt and resumes after restart", async () => { + clock(); let stall = true, started = false, closed = false; + const server = createServer((request, response) => { + if (request.url?.includes("/pulls/")) { + if (stall) { + started = true; response.on("close", () => { closed = true; }); + response.writeHead(200, { "content-type": "application/json" }); response.write('{"state":'); + } else response.end(JSON.stringify({ state: "closed", merged: true, title: "Done" })); + } else response.end(JSON.stringify({ private: false })); + }); + await new Promise(resolve => server.listen(0, "127.0.0.1", resolve)); + const address = server.address(); if (!address || typeof address === "string") throw new Error("Missing HTTP test port"); + const svc = externalObjectService(db, { github: { + fetch: (url, init) => fetch(`http://127.0.0.1:${address.port}${new URL(url).pathname}`, init), + credentialProvider: () => ({ token: "simulated-company-token", source: "company_secret" }), + } }); + try { + await start(svc); const f = await fixture(); + await db.update(issues).set({ description: "https://github.com/acme/app/pull/42" }).where(eq(issues.id, f.issueId)); + await svc.syncIssue(f.issueId); await settle(() => started); + await coordinator!.stop(); coordinator = undefined; + await settle(() => closed); + const cancelled = await row(f.companyId); + expect(cancelled.refreshToken).toBeNull(); expect(cancelled.lastResolvedAt).toBeNull(); + expect(cancelled.nextRefreshAt!.getTime()).toBeLessThanOrEqual(Date.now()); + stall = false; await start(svc); + await settle(async () => Boolean((await row(f.companyId))?.isTerminal)); await quiet(); + } finally { await coordinator?.stop(); coordinator = undefined; server.closeAllConnections(); await new Promise(resolve => server.close(() => resolve())); } + }); + + it("loads only the changed identity and objects after ingestion into ten thousand future objects", async () => { + clock(); const svc = service(); const f = await fixture(); + const future = new Date(Date.now() + 86_400_000); + const ids = Array.from({ length: 10_000 }, () => randomUUID()); + for (let offset = 0; offset < ids.length; offset += 500) { + const batch = ids.slice(offset, offset + 500); + await db.insert(externalObjects).values(batch.map(id => ({ id, companyId: f.companyId, + providerKey: "url", objectType: "link", externalId: id, nextRefreshAt: future }))); + await db.insert(externalObjectMentions).values(batch.map(objectId => ({ companyId: f.companyId, + sourceIssueId: f.issueId, sourceKind: "description" as const, objectId }))); + } + await start(svc); await settle(() => coordinator!.nextWakeAt() === future.getTime()); + const [incoming] = await db.insert(issues).values({ companyId: f.companyId, title: "Incoming", + description: "https://example.com/new-object" }).returning(); + const reads: Array<{ query: string; params: unknown[] }> = []; + const debug = db.$client.options.debug; + db.$client.options.debug = (_connection, query, params) => { + if (/^select/i.test(query) && query.includes('from "external_objects"') && query.includes('inner join "companies"')) + reads.push({ query, params: [...params] }); + }; + try { + await svc.syncIssue(incoming!.id); + await settle(() => provider.mock.calls.length === 1); + await settle(() => coordinator!.nextWakeAt() === future.getTime() && !databaseWorkPending(db, DELIVERY_QUEUES.externalObjectRefresh)); + expect(reads.length).toBeGreaterThan(0); expect(reads.length).toBeLessThanOrEqual(6); + for (const read of reads) expect(read.query).toMatch(/"external_objects"\."(?:id|external_id)" =/); + } finally { db.$client.options.debug = debug; } + }); + it("reconstructs paginated multi-company work and never invokes inactive or never-refresh objects", async () => { clock(); const svc = service(); const a = await fixture(); const b = await fixture(); const inactive = await fixture("paused"); for (const companyId of [b.companyId, inactive.companyId]) { diff --git a/server/src/__tests__/server-startup-feedback-export.test.ts b/server/src/__tests__/server-startup-feedback-export.test.ts index bbb020c865d..5002dc44a50 100644 --- a/server/src/__tests__/server-startup-feedback-export.test.ts +++ b/server/src/__tests__/server-startup-feedback-export.test.ts @@ -207,6 +207,8 @@ const { }; const executionWorkspaceServiceFactoryMock = vi.fn(() => executionWorkspaceServiceMock); const externalObjectsServiceMock = { + registerRefreshWork: vi.fn((coordinator: { register: typeof deliveryWorkRegisterMock }, _canRun: () => boolean) => + coordinator.register("external-object-refresh", { run: async () => undefined, hasPending: async () => false })), refreshDueObjectsForActiveCompanies: vi.fn(async () => ({ companies: 0, checked: 0, refreshed: 0 })), }; const externalObjectsServiceFactoryMock = vi.fn(() => externalObjectsServiceMock); @@ -1399,7 +1401,10 @@ describe("startServer feedback export wiring", () => { await Promise.resolve(); expect(heartbeatServiceMock.tickAgentTimers).toHaveBeenCalledTimes(1); - expect(externalObjectsServiceMock.refreshDueObjectsForActiveCompanies).toHaveBeenCalledTimes(1); + expect(externalObjectsServiceMock.registerRefreshWork).toHaveBeenCalledTimes(1); + expect(externalObjectsServiceMock.registerRefreshWork).toHaveBeenCalledWith(expect.any(Object), expect.any(Function)); + expect(deliveryWorkRegisterMock.mock.calls.some(([queue]) => queue === DELIVERY_QUEUES.externalObjectRefresh)).toBe(true); + expect(externalObjectsServiceMock.refreshDueObjectsForActiveCompanies).not.toHaveBeenCalled(); expect(issueThreadInteractionServiceMock.sweepMergedPullRequestConfirmations).toHaveBeenCalledTimes(1); expect(executionWorkspaceServiceMock.sweepTerminalWorkspaces).toHaveBeenCalledTimes(1); expect(routineServiceMock.tickScheduledTriggers).toHaveBeenCalledTimes(1); @@ -1422,6 +1427,16 @@ describe("startServer feedback export wiring", () => { } }); + it.each([true, false])("does not wait for external-object provider work before readiness (scheduler=%s)", async enabled => { + loadConfigMock.mockReturnValue(buildTestConfig({ heartbeatSchedulerEnabled: enabled })); + externalObjectsServiceMock.registerRefreshWork.mockReturnValueOnce({ + ready: new Promise(() => {}), wake: vi.fn(), stop: vi.fn(async () => {}), + }); + await startServer(); + expect(externalObjectsServiceMock.registerRefreshWork).toHaveBeenCalledOnce(); + expect(externalObjectsServiceMock.refreshDueObjectsForActiveCompanies).not.toHaveBeenCalled(); + }); + it("keeps external object refresh active when heartbeat scheduling is disabled", async () => { loadConfigMock.mockReturnValue(buildTestConfig({ heartbeatSchedulerEnabled: false, @@ -1449,7 +1464,10 @@ describe("startServer feedback export wiring", () => { await Promise.resolve(); await Promise.resolve(); - expect(externalObjectsServiceMock.refreshDueObjectsForActiveCompanies).toHaveBeenCalledTimes(1); + expect(externalObjectsServiceMock.registerRefreshWork).toHaveBeenCalledTimes(1); + expect(externalObjectsServiceMock.registerRefreshWork).toHaveBeenCalledWith(expect.any(Object), expect.any(Function)); + expect(deliveryWorkRegisterMock.mock.calls.some(([queue]) => queue === DELIVERY_QUEUES.externalObjectRefresh)).toBe(true); + expect(externalObjectsServiceMock.refreshDueObjectsForActiveCompanies).not.toHaveBeenCalled(); expect(routineServiceMock.tickScheduledTriggers).not.toHaveBeenCalled(); expect(heartbeatServiceMock.reconcileCostAccounting).toHaveBeenCalledTimes(1); expect(environmentCustomImagesServiceMock.cleanupExpiredSetupSessions).not.toHaveBeenCalled(); diff --git a/server/src/services/external-object-refresh-notifications.ts b/server/src/services/external-object-refresh-notifications.ts index 95833421a8a..271d172e463 100644 --- a/server/src/services/external-object-refresh-notifications.ts +++ b/server/src/services/external-object-refresh-notifications.ts @@ -1,7 +1,9 @@ import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js"; -export async function notifyExternalObjectRefresh(tx: object, companyId?: string, objectId?: string) { +export type ExternalObjectRefreshIdentity = { providerKey: string; objectType: string; externalId: string; ignoreExternalIdCase?: boolean }; + +export async function notifyExternalObjectRefresh(tx: object, companyId?: string, object?: string | ExternalObjectRefreshIdentity) { await notifyDeliveryWork(tx, DELIVERY_QUEUES.externalObjectRefresh, - companyId ? JSON.stringify({ companyId, ...(objectId ? { objectId } : {}) }) : "*"); + companyId ? JSON.stringify({ companyId, ...(typeof object === "string" ? { objectId: object } : object ? { identity: object } : {}) }) : "*"); } diff --git a/server/src/services/external-objects.ts b/server/src/services/external-objects.ts index d524b473e1a..0208cdab7b6 100644 --- a/server/src/services/external-objects.ts +++ b/server/src/services/external-objects.ts @@ -18,7 +18,7 @@ import { notFound } from "../errors.js"; import { logger } from "../middleware/logger.js"; import { logActivity, type LogActivityInput } from "./activity-log.js"; import { createGitHubExternalObjectProvider, type GitHubExternalObjectProviderOptions } from "./github-external-object-provider.js"; -import { notifyExternalObjectRefresh } from "./external-object-refresh-notifications.js"; +import { notifyExternalObjectRefresh, type ExternalObjectRefreshIdentity } from "./external-object-refresh-notifications.js"; import { DELIVERY_QUEUES } from "./delivery-work-notifications.js"; import type { createDeliveryWorkCoordinator } from "./delivery-work-coordinator.js"; import { registerScopedDeadlineWork } from "./scoped-deadline-work.js"; @@ -88,6 +88,7 @@ export interface ExternalObjectResolver { resolve(input: { companyId: string; object: ExternalObjectRecord; + signal?: AbortSignal; }): Promise; } @@ -99,8 +100,8 @@ const DEFAULT_RETRY_AFTER_SECONDS = 300; const DEFAULT_REFRESH_LEASE_SECONDS = 300; const REFRESH_LEASE_RENEW_INTERVAL_MS = 60_000; -type RefreshScope = { companyId: string; objectId?: string }; -type RefreshDeadline = { id: string; companyId: string; dueAt: number }; +type RefreshScope = { companyId: string; objectId?: string; identity?: ExternalObjectRefreshIdentity }; +type RefreshDeadline = { id: string; companyId: string; dueAt: number; providerKey: string; objectType: string; externalId: string }; function refreshableObjectWhere() { @@ -471,7 +472,7 @@ export function externalObjectService( nextRefreshAt: now, updatedAt: now, }; - await notifyExternalObjectRefresh(dbOrTx, companyId); + await notifyExternalObjectRefresh(dbOrTx, companyId, { providerKey: detection.providerKey, objectType: detection.objectType, externalId: detection.externalId }); const inserted = await dbOrTx .insert(externalObjects) .values(values) @@ -501,7 +502,9 @@ export function externalObjectService( dbOrTx: any = db, ) { const urls = extractExternalObjectCanonicalUrls(input.text ?? ""); - await notifyExternalObjectRefresh(dbOrTx, input.companyId); + const previous: Array<{ objectId: string | null }> = await dbOrTx.selectDistinct({ objectId: externalObjectMentions.objectId }) + .from(externalObjectMentions).where(sourceWhere(input)); + for (const row of previous) if (row.objectId) await notifyExternalObjectRefresh(dbOrTx, input.companyId, row.objectId); await dbOrTx.delete(externalObjectMentions).where(sourceWhere(input)); if (urls.length === 0) return; @@ -520,6 +523,7 @@ export function externalObjectService( if (seen.has(sourceKey)) continue; seen.add(sourceKey); const object = await upsertObjectFromDetection(input.companyId, detection, dbOrTx); + await notifyExternalObjectRefresh(dbOrTx, input.companyId, object.id); values.push({ companyId: input.companyId, sourceIssueId: input.sourceIssueId, @@ -573,9 +577,9 @@ export function externalObjectService( async function deleteMissingSourceMentions(kind: "comment" | "document", recordId: string, tx: any) { const source = and(eq(externalObjectMentions.sourceKind, kind), eq(externalObjectMentions.sourceRecordId, recordId)); - const owners: Array<{ companyId: string }> = await tx.selectDistinct({ companyId: externalObjectMentions.companyId }) + const owners: Array<{ companyId: string; objectId: string | null }> = await tx.selectDistinct({ companyId: externalObjectMentions.companyId, objectId: externalObjectMentions.objectId }) .from(externalObjectMentions).where(source); - for (const owner of owners) await notifyExternalObjectRefresh(tx, owner.companyId); + for (const owner of owners) if (owner.objectId) await notifyExternalObjectRefresh(tx, owner.companyId, owner.objectId); await tx.delete(externalObjectMentions).where(source); } @@ -836,6 +840,7 @@ export function externalObjectService( actor?: Pick; force?: boolean; scheduled?: boolean; + signal?: AbortSignal; now?: Date; }; @@ -875,7 +880,8 @@ export function externalObjectService( now: Date, refreshToken: string, ) { - const pluginResult = await resolveViaPluginProvider(db, opts.pluginWorkerManager, object); + input.signal?.throwIfAborted(); + const pluginResult = await observeRefreshCancellation(resolveViaPluginProvider(db, opts.pluginWorkerManager, object), input.signal); const resolver = pluginResult ? null : resolverRegistry.find(object); if (!pluginResult && !resolver) { const [updated] = await persistRefresh(object, refreshToken, { @@ -889,7 +895,9 @@ export function externalObjectService( return { object: toObjectPayload(updated ?? object, now), refreshed: false, reason: "no_resolver" as const }; } - const result = pluginResult ?? await resolver!.resolve({ companyId: object.companyId, object }); + const result = pluginResult ?? await observeRefreshCancellation( + resolver!.resolve({ companyId: object.companyId, object, signal: input.signal }), input.signal); + input.signal?.throwIfAborted(); if (!result.ok) { // A failed refresh means the object's status is currently unknown, not // terminal. Clear a previous terminal verdict so the object keeps being @@ -1027,6 +1035,16 @@ export function externalObjectService( }; } + function observeRefreshCancellation(work: Promise, signal?: AbortSignal): Promise { + if (!signal) return work; + return new Promise((resolve, reject) => { + const abort = () => reject(signal.reason); + work.then(value => { signal.removeEventListener("abort", abort); resolve(value); }, + error => { signal.removeEventListener("abort", abort); reject(error); }); + if (signal.aborted) abort(); else signal.addEventListener("abort", abort, { once: true }); + }); + } + const objectRefreshesInFlight = new Map>>>(); async function refreshObject( @@ -1046,7 +1064,7 @@ export function externalObjectService( const refreshKey = `${object.companyId}:${object.id}`; const existingRefresh = objectRefreshesInFlight.get(refreshKey); - if (existingRefresh) return existingRefresh; + if (existingRefresh) return observeRefreshCancellation(existingRefresh, input.signal); const claimed = await claimObjectRefresh(object, input, now); if (!claimed) { @@ -1063,7 +1081,10 @@ export function externalObjectService( } const stopRenewingRefreshLease = startRefreshLeaseRenewal(object, claimed.refreshToken); - const refresh = resolveObjectRefresh(object, input, now, claimed.refreshToken).finally(() => { + const refresh = resolveObjectRefresh(object, input, now, claimed.refreshToken).catch(async error => { + if (input.signal?.aborted) await persistRefresh(object, claimed.refreshToken!, { refreshStartedAt: null, refreshToken: null }); + throw error; + }).finally(() => { stopRenewingRefreshLease(); objectRefreshesInFlight.delete(refreshKey); }); @@ -1149,12 +1170,17 @@ export function externalObjectService( const key = scopeKey(scope); const cursor = cursors.get(key); const rows = await db.select({ id: externalObjects.id, companyId: externalObjects.companyId, - nextRefreshAt: externalObjects.nextRefreshAt, refreshStartedAt: externalObjects.refreshStartedAt }).from(externalObjects) + nextRefreshAt: externalObjects.nextRefreshAt, refreshStartedAt: externalObjects.refreshStartedAt, + providerKey: externalObjects.providerKey, objectType: externalObjects.objectType, externalId: externalObjects.externalId }).from(externalObjects) .innerJoin(companies, eq(companies.id, externalObjects.companyId)) .where(and( eq(companies.status, "active"), automaticRefreshWhere(), scope ? eq(externalObjects.companyId, scope.companyId) : undefined, scope?.objectId ? eq(externalObjects.id, scope.objectId) : undefined, + scope?.identity ? and(eq(externalObjects.providerKey, scope.identity.providerKey), + eq(externalObjects.objectType, scope.identity.objectType), scope.identity.ignoreExternalIdCase + ? sql`lower(${externalObjects.externalId}) = lower(${scope.identity.externalId})` + : eq(externalObjects.externalId, scope.identity.externalId)) : undefined, cursor ? gt(externalObjects.id, cursor) : undefined, )).orderBy(asc(externalObjects.id)).limit(250); const entries = rows.map(deadline); @@ -1162,8 +1188,8 @@ export function externalObjectService( else cursors.delete(key); return { rows: entries, complete: rows.length < 250 }; } - function deadline(object: Pick): RefreshDeadline { - return { id: object.id, companyId: object.companyId, + function deadline(object: Pick): RefreshDeadline { + return { id: object.id, companyId: object.companyId, providerKey: object.providerKey, objectType: object.objectType, externalId: object.externalId, dueAt: Math.max(object.nextRefreshAt!.getTime(), object.refreshStartedAt ? object.refreshStartedAt.getTime() + DEFAULT_REFRESH_LEASE_SECONDS * 1000 : 0) }; } @@ -1173,7 +1199,10 @@ export function externalObjectService( minimumPassIntervalMs: 0, decodeScope: hint => hint === "*" ? undefined : JSON.parse(hint) as RefreshScope, matches: (entry, scope) => !scope || (entry.companyId === scope.companyId - && (!scope.objectId || entry.id === scope.objectId)), + && (!scope.objectId || entry.id === scope.objectId) + && (!scope.identity || (entry.providerKey === scope.identity.providerKey && entry.objectType === scope.identity.objectType + && (scope.identity.ignoreExternalIdCase ? entry.externalId.toLowerCase() === scope.identity.externalId.toLowerCase() + : entry.externalId === scope.identity.externalId)))), load, check: async (entry, signal) => { if (signal?.aborted || !canRun()) return entry; @@ -1183,6 +1212,7 @@ export function externalObjectService( await refreshObject(entry.id, { companyId: entry.companyId, scheduled: true, + signal, actor: { actorType: "system", actorId: "external-object-resolver", agentId: null, runId: null }, }); const updated = await load({ companyId: entry.companyId, objectId: entry.id }); diff --git a/server/src/services/github-connection-events.ts b/server/src/services/github-connection-events.ts index 76755ce30c3..643bc557b93 100644 --- a/server/src/services/github-connection-events.ts +++ b/server/src/services/github-connection-events.ts @@ -246,7 +246,8 @@ export function githubConnectionEventService( if (!snapshot) return; const appliedAt = now(); await db.transaction(async tx => { - await notifyExternalObjectRefresh(tx, companyId); + await notifyExternalObjectRefresh(tx, companyId, { providerKey: "github", objectType: "pull_request", + externalId: snapshot.externalId, ignoreExternalIdCase: true }); await tx.update(externalObjects).set({ statusKey: snapshot.statusKey, statusLabel: snapshot.statusLabel, diff --git a/server/src/services/github-external-object-provider.ts b/server/src/services/github-external-object-provider.ts index 6dedac97efb..83b47354035 100644 --- a/server/src/services/github-external-object-provider.ts +++ b/server/src/services/github-external-object-provider.ts @@ -15,6 +15,7 @@ type FetchLike = (url: string, init?: RequestInit) => Promise; export interface GitHubExternalObjectProviderOptions { fetch?: FetchLike; + requestTimeoutMs?: number; tokenProvider?: (companyId: string) => Promise | string | null; /** * Resolve the credential and its source. Prefer this over `tokenProvider` when the @@ -243,12 +244,12 @@ type RepoProbeOutcome = { async function probeRepoVisibility( identity: GitHubObjectIdentity, - opts: { fetch: FetchLike; headers?: Record | undefined }, + opts: { fetch: FetchLike; headers?: Record | undefined; signal?: AbortSignal }, ): Promise { try { const probe = await opts.fetch( `${gitHubApiBase(identity.host)}/repos/${encodeURIComponent(identity.owner)}/${encodeURIComponent(identity.repo)}`, - { headers: opts.headers }, + { headers: opts.headers, signal: opts.signal }, ); if (probe.ok) { const body = await safeJson(probe); @@ -427,12 +428,13 @@ export function createGitHubExternalObjectProvider( companyId: string, identity: GitHubObjectIdentity, headers: Record, + signal: AbortSignal, ): Promise { const credentialFingerprint = createHash("sha256").update(headers.authorization ?? "anonymous").digest("hex"); const key = `${companyId}\u0000${credentialFingerprint}\u0000${identity.owner.toLowerCase()}\u0000${identity.repo.toLowerCase()}`; const cached = probeCache.get(key); if (cached && cached.expiresAt > Date.now()) return cached.outcome; - const outcome = await probeRepoVisibility(identity, { fetch: fetchImpl, headers }); + const outcome = await probeRepoVisibility(identity, { fetch: fetchImpl, headers, signal }); if (outcome.visibility === "visible" || outcome.visibility === "hidden") { probeCache.set(key, { outcome, expiresAt: Date.now() + GITHUB_OBJECT_TTL_SECONDS * 1000 }); } @@ -464,7 +466,9 @@ export function createGitHubExternalObjectProvider( return { providerKey: "github", objectType, - async resolve({ companyId, object }) { + async resolve({ companyId, object, signal: ownerSignal }) { + const deadline = AbortSignal.timeout(opts.requestTimeoutMs ?? 10_000); + const signal = ownerSignal ? AbortSignal.any([ownerSignal, deadline]) : deadline; const identity = parseGitHubObject(object); if (!identity || identity.objectType !== objectType) { return { @@ -509,7 +513,7 @@ export function createGitHubExternalObjectProvider( // "public" verdict must not authorize a later read of its private contents. // The cache is still used below to classify a 404, where a stale verdict can // only make the object look deleted — it can never expose private data. - const probe = await probeRepoVisibility(identity, { fetch: fetchImpl, headers }); + const probe = await probeRepoVisibility(identity, { fetch: fetchImpl, headers, signal }); if (probe.visibility === "hidden") { return { ok: false, @@ -554,7 +558,7 @@ export function createGitHubExternalObjectProvider( let response: Response; try { - response = await fetchImpl(url, { headers }); + response = await fetchImpl(url, { headers, signal }); } catch { return { ok: false, @@ -573,7 +577,7 @@ export function createGitHubExternalObjectProvider( // Probe the repository metadata to classify: if the repo itself is visible // the object truly is gone; if it is not, surface credentials-needed // instead of permanently freezing a reference as "Not found". - const probe = await cachedProbeRepoVisibility(companyId, identity, headers); + const probe = await cachedProbeRepoVisibility(companyId, identity, headers, signal); if (probe.visibility === "visible") { return { ok: true, snapshot: notFoundSnapshot(identity, etag) }; } From dbd400ce17d85b88c4b51ff9860e779dc169555b Mon Sep 17 00:00:00 2001 From: Dotta Date: Fri, 30 Oct 2026 17:24:55 -0500 Subject: [PATCH 28/70] fix: back off expired deadlines during domain drain Keep a future in-memory resume wake when domain dispatch is paused independently of coordinator admission. Preserve the saved obligation without rescheduling its expired deadline into an immediate skipped-attempt loop. Co-Authored-By: Paperclip --- server/src/__tests__/deadline-work.test.ts | 47 ++++++++++++++++++++++ server/src/services/deadline-work.ts | 2 +- 2 files changed, 48 insertions(+), 1 deletion(-) diff --git a/server/src/__tests__/deadline-work.test.ts b/server/src/__tests__/deadline-work.test.ts index 7657a7e9d99..c00daeb2f00 100644 --- a/server/src/__tests__/deadline-work.test.ts +++ b/server/src/__tests__/deadline-work.test.ts @@ -122,3 +122,50 @@ it("does not release an ambiguous database intent while dispatch is drained", as draining = false; await vi.advanceTimersByTimeAsync(30_000); expect(run).toHaveBeenCalledTimes(2); expect(coordinator.nextWakeAt()).toBeNull(); }); + +it("backs off an expired deadline when only the domain is drained and resumes promptly", async () => { + vi.useFakeTimers(); + let paused = false; + const owner = installDatabaseWorkSignals({ execute: vi.fn(async () => []) }); + const run = vi.fn(async () => {}); + const domainCanRun = vi.fn(() => !paused); + const deadline = Date.now() + 1_000; + const next = vi.fn(async () => run.mock.calls.length === 1 ? deadline : null); + coordinator = createDeliveryWorkCoordinator({ owner, canRun: () => true, onError: vi.fn() }); + const worker = registerDeadlineWork(coordinator, DELIVERY_QUEUES.agentTimers, { + run, next, canRun: domainCanRun, retryMs: 30_000, + }); + await worker.ready; + paused = true; + const before = idleWorkSnapshot(); + await vi.advanceTimersByTimeAsync(1_000); + expect(coordinator.nextWakeAt()).toBe(Date.now() + 30_000); + const attempts = domainCanRun.mock.calls.length; + await vi.advanceTimersByTimeAsync(90_000); + expect(domainCanRun.mock.calls.length - attempts).toBeLessThanOrEqual(12); + // Four attempts (expiry plus three retries), each begins and ends tracked work. + expect(idleWorkSnapshot().generation - before.generation).toBe(8); + expect(idleWorkSnapshot().active).toBe(before.active); + expect(run).toHaveBeenCalledOnce(); expect(next).toHaveBeenCalledOnce(); expect(owner.execute).not.toHaveBeenCalled(); + paused = false; + await vi.advanceTimersByTimeAsync(30_000); + expect(run).toHaveBeenCalledTimes(2); expect(coordinator.nextWakeAt()).toBeNull(); +}); + +it("retains a prompt paused wake when a new request preempts a far-future deadline", async () => { + vi.useFakeTimers(); + let paused = false; + const owner = installDatabaseWorkSignals({ execute: vi.fn(async () => []) }); + const run = vi.fn(async () => {}); + const deadline = Date.now() + 86_400_000; + coordinator = createDeliveryWorkCoordinator({ owner, canRun: () => true, onError: vi.fn() }); + const worker = registerDeadlineWork(coordinator, DELIVERY_QUEUES.agentTimers, { + run, next: async () => run.mock.calls.length === 1 ? deadline : null, canRun: () => !paused, retryMs: 30_000, + }); + await worker.ready; paused = true; worker.wake(); + await vi.advanceTimersByTimeAsync(0); + expect(coordinator.nextWakeAt()).toBe(Date.now() + 30_000); + expect(owner.execute).not.toHaveBeenCalled(); + paused = false; await vi.advanceTimersByTimeAsync(30_000); + expect(run).toHaveBeenCalledTimes(2); expect(coordinator.nextWakeAt()).toBeNull(); +}); diff --git a/server/src/services/deadline-work.ts b/server/src/services/deadline-work.ts index 15ba76abdae..208b208e051 100644 --- a/server/src/services/deadline-work.ts +++ b/server/src/services/deadline-work.ts @@ -52,7 +52,7 @@ export function registerDeadlineWork( // the domain to probe its database while dispatch is closed. hasPending: async () => task.canRun() && requested !== reconciled, nextRunAt: () => !task.canRun() && requested !== reconciled - ? Math.min(next ?? Infinity, Date.now() + task.retryMs) + ? Date.now() + task.retryMs : next, }); return { ...worker, wake: () => { requested++; worker.wake(); } }; From 4af412eaea9e780e2d867bd930e77865a1c4672b Mon Sep 17 00:00:00 2001 From: Dotta Date: Fri, 30 Oct 2026 17:25:18 -0500 Subject: [PATCH 29/70] refactor: drive status-card evaluation from committed work Move generation sequencing into a feature-owned deadline worker and retain persisted evaluation, eligibility and relative-search obligations. Add transactional producer hints and initial PostgreSQL acceptance coverage; follow-up completes the producer audit and remaining lifecycle gates. Co-Authored-By: Paperclip --- doc/plans/2026-10-30-status-cards-sleep.md | 33 ++++ .../status-card-evaluation-work.test.ts | 180 ++++++++++++++++++ .../status-card-update-engine.test.ts | 23 +++ server/src/__tests__/status-cards.test.ts | 9 +- server/src/index.ts | 35 +--- .../active-run-watchdog/adapters/postgres.ts | 2 + server/src/services/access.ts | 2 + server/src/services/agent-deletion.ts | 2 + server/src/services/companies.ts | 3 + server/src/services/company-deletion.ts | 2 + server/src/services/company-search.ts | 10 +- server/src/services/company-skills.ts | 2 + .../services/delivery-work-notifications.ts | 1 + .../execution-control-notifications.ts | 3 + server/src/services/execution-workspaces.ts | 4 + server/src/services/heartbeat/retries.ts | 2 + server/src/services/instance-settings.ts | 2 + server/src/services/issue-comment-writes.ts | 2 + server/src/services/issue-prefix.ts | 2 + .../issue-review-state-notifications.ts | 2 + server/src/services/issue-title.ts | 2 + server/src/services/issue-tree-control.ts | 2 + server/src/services/issues.ts | 6 + .../native-finalization-reconciler.ts | 2 + .../native-review-participant.ts | 2 + .../native-workspace-export-recovery.ts | 2 + .../native-workspace-export-retry.ts | 2 + .../native-workspace-reconstruction.ts | 2 + .../paperclip-runner-tool-authority.ts | 2 + server/src/services/plugin-workspaces.ts | 2 + .../src/services/provider-fresh-workspace.ts | 2 + server/src/services/routines.ts | 2 + .../services/slack-conversation-lifecycle.ts | 2 + .../src/services/slack-conversation-state.ts | 2 + server/src/services/status-card-evaluation.ts | 90 +++++++++ .../src/services/status-card-finalization.ts | 2 + .../src/services/status-card-notifications.ts | 10 + .../src/services/status-card-update-engine.ts | 40 +++- server/src/services/status-cards.ts | 159 ++++++++++------ 39 files changed, 561 insertions(+), 93 deletions(-) create mode 100644 doc/plans/2026-10-30-status-cards-sleep.md create mode 100644 server/src/__tests__/status-card-evaluation-work.test.ts create mode 100644 server/src/services/status-card-evaluation.ts create mode 100644 server/src/services/status-card-notifications.ts diff --git a/doc/plans/2026-10-30-status-cards-sleep.md b/doc/plans/2026-10-30-status-cards-sleep.md new file mode 100644 index 00000000000..dcd6c5ca226 --- /dev/null +++ b/doc/plans/2026-10-30-status-cards-sleep.md @@ -0,0 +1,33 @@ +# Status-card evaluation deadlines + +Baseline: `76fde891ccc29283b3f5c8dcdce089c5b3d937d1`. + +Status-card ownership covers evaluation policy, card claims, generation, ordinary assignment wakeups, and failed-assignment cancellation. `index.ts` registers a typed capability. Shared delivery/deadline infrastructure owns transaction settlement, coalescing, admission/drain, retries and shutdown. Existing cards, updates and issues remain durable truth; no timer table or sleep-safety exemption is introduced. + +Manual cards have no automatic obligation. Interval cards retain their requested evaluation cadence. Reactive cards evaluate committed source changes, retain the actual debounce/rate/budget/active-hours deadline while relevant changes remain, and disarm once unchanged. Saved relative-search windows retain their actual membership expiry; the duration interpretation stays with company search. Startup reconstructs existing work without advancing business timestamps. + +Scheduled reactive source events can preempt a future evaluation. A feature-owned transaction advisory lock serializes scheduler evaluation across connections while existing card/issue services perform their own committed writes. The lock holder has no card row lock, avoiding self-deadlock. Existing next-evaluation CAS, generating-issue ownership and issue idempotency keys remain required, including for manual refreshes and older server instances that do not use the advisory lock. The persisted five-minute claim remains the failure retry after a scheduled refresh cannot finish. Internal timers do not establish a Cloud wake alarm. + +## Producer audit (in progress) + +| Source | Owning committed boundary | State | +| --- | --- | --- | +| Card create/config/archive/delete, compile, query/summary writeback, evaluation | status-card mutation transactions | wired; ordinary lifecycle exercised | +| Experimental feature flag | instance settings update transaction | wired; disable exercised | +| Company pause/archive/reactivate/delete | company owning transactions | wired; lifecycle acceptance remains | +| Ordinary issue create/update/remove | issue service transactions | wired; create/change exercised | +| Comments create/delete/tombstone | common insert and issue mutation transactions | wired; comment acceptance remains | +| Review state / chat-run milestones | issue-review-state notification boundary | wired; exact caller coverage audit remains | +| Execution-control writes/run settlement | execution-control transaction helpers | wired; exact caller coverage audit remains | +| Agent deletion/membership reassignment, issue tree transitions, title/prefix | owner transaction | wired; representative acceptance remains | +| Workspace selection/binding, plugin/provider workspace, native recovery/reclaim, active watchdog, retry | owner transactions where uncovered by shared notifications | partially wired; remaining per-write audit required | +| Raw workspace cleanup/routes, conversation activation/settlement, queue paths | direct writes and adjacent owner notifications | exact coverage incomplete; not acceptance-ready | +| Imports/independent writers/replicas | supported service writes and startup reconstruction | full ingress contract audit remains | + +Process-local notifications cover writes using the installed database owner. They do not notify an unrelated SQL writer or another server process; those supported paths need explicit ingress or reconstruction coverage. No claim of cross-replica delivery, host suspension, or fleet savings is made by the current tests. + +## Acceptance evidence and remaining gates + +The original status-card PostgreSQL suite passed25/25. New real PostgreSQL tests exercise ordinary card compilation/writeback, source-change debounce, persisted assignment at a controlled provider boundary, result persistence, quiet SQL after completion, earlier/cancelled deadlines, nested outer commit/rollback and drain/resume. The provider boundary is simulated; no model runs. Additional race, restart, failure, active-hour/rate/budget persistence, multi-company and producer-class coverage remain before PR delivery. + +Shared deadline regressions reproduced and corrected a settled-write admission hold during drain. The correction releases only settled holds, retains an in-memory resume wake, and leaves ambiguous transaction intent fenced by the database coordinator. Unit coverage includes first write during drain, failure entering drain, preserved future deadlines, warm standby and ambiguous commit settlement. diff --git a/server/src/__tests__/status-card-evaluation-work.test.ts b/server/src/__tests__/status-card-evaluation-work.test.ts new file mode 100644 index 00000000000..427ad3385eb --- /dev/null +++ b/server/src/__tests__/status-card-evaluation-work.test.ts @@ -0,0 +1,180 @@ +import { randomUUID } from "node:crypto"; +import { eq, sql } from "drizzle-orm"; +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; +import { activityLog, agentWakeupRequests, agents, companies, companyMemberships, createDb, documents, + heartbeatRuns, instanceSettings, issueComments, issues, statusCards, statusCardUpdates, documentRevisions } from "@paperclipai/db"; +import { companySearchQuerySchema, statusCardRefreshPolicySchema } from "@paperclipai/shared"; +import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js"; +import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; +import { createDeliveryWorkCoordinator } from "../services/delivery-work-coordinator.js"; +import { registerStatusCardEvaluationWork, statusCardEvaluationService } from "../services/status-card-evaluation.js"; +import { statusCardService } from "../services/status-cards.js"; +import { issueService } from "../services/issues.js"; +import { instanceSettingsService } from "../services/instance-settings.js"; +import { idleWorkSnapshot } from "../services/task-admission.js"; + +const external = resolveExternalTestDatabaseUrl("PAPERCLIP_STATUS_CARD_EVALUATION_TEST_DATABASE_URL"); +const support = external ? { supported: true } : await getEmbeddedPostgresTestSupport(); +(support.supported ? describe : describe.skip)("status-card evaluation work", () => { + let temporary: Awaited> | undefined; + let db: ReturnType; + let coordinator: ReturnType | undefined; + let draining = false; + let companyId: string, agentId: string; + const errors: unknown[] = []; + const actor = { agentId: null, userId: "board-user" }; + const policy = (mode: "manual" | "reactive" | "interval" = "reactive") => statusCardRefreshPolicySchema.parse({ + mode, debounceSeconds: 60, intervalMinutes: 15, + }); + // Controlled provider boundary: assignment persists a queued wake, and tests + // explicitly run the authorized summarizer writeback. No model/provider runs. + const wakeup = vi.fn(async (id: string, options: { payload?: Record | null; reason?: string | null }) => { + return db.insert(agentWakeupRequests).values({ companyId, agentId: id, source: "assignment", + reason: options.reason, payload: options.payload, status: "queued" }).returning(); + }); + + beforeAll(async () => { + if (!external) temporary = await startEmbeddedPostgresTestDatabase("status-card-evaluation-"); + db = createDb(external ?? temporary!.connectionString, { maxConnections: 6, idleTimeoutSeconds: 0, maxLifetimeSeconds: 0 }); + }, 30_000); + beforeEach(async () => { + companyId = randomUUID(); agentId = randomUUID(); draining = false; errors.length = 0; wakeup.mockClear(); + await db.insert(companies).values({ id: companyId, name: "Status", issuePrefix: `S${companyId.slice(0, 6)}` }); + await db.insert(agents).values({ id: agentId, companyId, name: "Summarizer", role: "general", status: "idle", adapterType: "process", adapterConfig: {} }); + await instanceSettingsService(db).updateExperimental({ enableStatusCards: true }); + await Promise.all(Array.from({ length: 6 }, () => db.execute(sql`select pg_sleep(0.01)`))); + vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"], shouldAdvanceTime: true }); + }); + afterEach(async () => { + await coordinator?.stop(); coordinator = undefined; + vi.useRealTimers(); vi.restoreAllMocks(); + await db.delete(agentWakeupRequests); await db.delete(statusCardUpdates); await db.delete(statusCards); + await db.delete(documentRevisions); await db.delete(documents); await db.delete(activityLog); + await db.delete(issueComments); await db.delete(issues); await db.delete(heartbeatRuns); + await db.delete(instanceSettings); await db.delete(agents); await db.delete(companyMemberships); await db.delete(companies); + }); + afterAll(async () => { await db?.$client.end({ timeout: 1 }); await temporary?.cleanup(); }); + async function start() { + coordinator = createDeliveryWorkCoordinator({ owner: db, canRun: () => !draining, canReconcile: () => true, + onError: error => errors.push(error) }); + await registerStatusCardEvaluationWork(coordinator, statusCardEvaluationService(db, { wakeup }), () => !draining).ready; + } + async function settled() { + await vi.waitFor(async () => { + await vi.advanceTimersByTimeAsync(30_000); + expect(idleWorkSnapshot().active).toBe(0); + }, { timeout: 5000 }); + } + async function quiet() { + await settled(); + expect(coordinator!.nextWakeAt()).toBeNull(); + const reads = vi.spyOn(db, "select"), writes = vi.spyOn(db, "update"), executes = vi.spyOn(db, "execute"); + const calls = wakeup.mock.calls.length; + await vi.advanceTimersByTimeAsync(10 * 60_000); + expect(reads).not.toHaveBeenCalled(); expect(writes).not.toHaveBeenCalled(); expect(executes).not.toHaveBeenCalled(); + expect(wakeup).toHaveBeenCalledTimes(calls); + reads.mockRestore(); writes.mockRestore(); executes.mockRestore(); + } + async function writeback(cardId: string, generationIssueId: string, compile = false) { + const [run] = await db.insert(heartbeatRuns).values({ companyId, agentId, status: "running" }).returning(); + await db.update(issues).set({ checkoutRunId: run!.id }).where(eq(issues.id, generationIssueId)); + const writer = { agentId, runId: run!.id }; + const cards = statusCardService(db); + if (compile) await cards.writeQuery(cardId, { queries: [companySearchQuerySchema.parse({ scope: "issues", status: ["blocked"] })], + title: "Blockers", changeSummary: "Compiled", generationIssueId }, writer); + await cards.writeSummary(cardId, { markdown: "Current blockers", changeSummary: "Updated", generationIssueId }, writer); + await issueService(db).update(generationIssueId, { status: "done" }); + } + async function card(mode: "manual" | "reactive" | "interval" = "reactive") { + const cards = statusCardService(db); + const created = await cards.create(companyId, { interestPrompt: "Blockers", titlePinned: false, agentId, refreshPolicy: policy(mode) }, actor); + const compile = await cards.requestCompile(created.id, actor); + await writeback(created.id, compile.generatingIssue.id, true); + return cards.getById(created.id).then(row => row!); + } + + it("settles empty, disabled, configured manual and unchanged reactive history without recurring SQL", async () => { + await start(); await quiet(); + await card("manual"); await card("reactive"); await quiet(); + await instanceSettingsService(db).updateExperimental({ enableStatusCards: false }); await quiet(); + expect(errors).toEqual([]); + }); + + it("ordinary issue change schedules debounce, assigns one durable wake, writes a result and becomes quiet", async () => { + await start(); const created = await card(); await quiet(); + const changed = await issueService(db).create(companyId, { title: "New blocker", status: "blocked", priority: "medium", createdByUserId: "board-user" }); + await vi.waitFor(async () => { + expect((await statusCardService(db).getById(created.id))?.pendingChangeCount).toBeGreaterThan(0); + }, { timeout: 5000 }); + const pending = await statusCardService(db).getById(created.id); + expect(pending?.pendingChangeCount).toBeGreaterThan(0); + expect(pending?.nextEvalAt).toBeInstanceOf(Date); + expect(wakeup).not.toHaveBeenCalled(); + await vi.advanceTimersByTimeAsync(60_000); await settled(); + const generating = await statusCardService(db).getById(created.id); + expect(generating?.generatingIssueId).toBeTruthy(); + const durable = await db.select().from(agentWakeupRequests); + expect(durable).toHaveLength(1); expect(durable[0]?.payload?.issueId).toBe(generating?.generatingIssueId); + expect(changed.companyId).toBe(companyId); + await writeback(created.id, generating!.generatingIssueId!); await quiet(); + expect(await db.select().from(statusCardUpdates).then(rows => rows.filter(row => row.generationIssueId === generating?.generatingIssueId))) + .toMatchObject([{ status: "ok", finishedAt: expect.any(Date) }]); + expect(errors).toEqual([]); + }); + + it("cancels archived or disabled future work and replaces an earlier interval deadline", async () => { + await start(); let created = await card("interval"); await settled(); + const old = coordinator!.nextWakeAt(); + created = await statusCardService(db).update(created, { refreshPolicy: { ...policy("interval"), intervalMinutes: 1 } }, actor); + await settled(); expect(coordinator!.nextWakeAt()).toBeLessThan(old!); + await statusCardService(db).update(created, { archived: true }, actor); await quiet(); + expect(wakeup).not.toHaveBeenCalled(); expect(errors).toEqual([]); + }); + + it("does not react to rolled-back or nested uncommitted card configuration", async () => { + await start(); const created = await card("manual"); await quiet(); + await expect(db.transaction(async tx => { + await statusCardService(tx as unknown as typeof db).update(created, { refreshPolicy: policy("interval") }, actor); + await vi.advanceTimersByTimeAsync(30_000); expect(wakeup).not.toHaveBeenCalled(); + throw new Error("rollback"); + })).rejects.toThrow("rollback"); + await quiet(); expect((await statusCardService(db).getById(created.id))?.nextEvalAt).toBeNull(); + await db.transaction(async tx => { + await tx.transaction(async nested => { + await statusCardService(nested as unknown as typeof db).update(created, { refreshPolicy: policy("interval") }, actor); + }); + expect(coordinator!.nextWakeAt()).not.toBeNull(); + expect((await statusCardService(db).getById(created.id))?.nextEvalAt).toBeNull(); + }); + await settled(); expect(coordinator!.nextWakeAt()).not.toBeNull(); expect(errors).toEqual([]); + }); + + it("serializes competing connections when one source event preempts a future evaluation", async () => { + const created = await card(); + await db.update(statusCards).set({ nextEvalAt: new Date(Date.now() + 3_600_000) }).where(eq(statusCards.id, created.id)); + await issueService(db).create(companyId, { title: "Concurrent blocker", status: "blocked", priority: "medium", createdByUserId: "board-user" }); + const first = statusCardEvaluationService(db, { wakeup }); + const second = statusCardEvaluationService(db, { wakeup }); + first.changed(new Set([`changes:${companyId}`])); second.changed(new Set([`changes:${companyId}`])); + await Promise.all([first.run(new AbortController().signal), second.run(new AbortController().signal)]); + const pending = await statusCardService(db).getById(created.id); + expect(pending?.pendingChangeCount).toBeGreaterThan(0); + expect(pending?.nextEvalAt?.getTime()).toBeLessThan(Date.now() + 3_600_000); + expect(wakeup).not.toHaveBeenCalled(); + await vi.advanceTimersByTimeAsync(60_000); + await Promise.all([first.run(new AbortController().signal), second.run(new AbortController().signal)]); + const updated = await statusCardService(db).getById(created.id); + expect(updated?.generatingIssueId).toBeTruthy(); expect(wakeup).toHaveBeenCalledOnce(); + expect(await db.select().from(statusCardUpdates).then(rows => rows.filter(row => row.generationIssueId === updated?.generatingIssueId))) + .toHaveLength(1); + }); + + it("preserves the first write during drain until resume", async () => { + await start(); const created = await card("manual"); await quiet(); + draining = true; + await statusCardService(db).update(created, { refreshPolicy: policy("interval") }, actor); + await settled(); expect(wakeup).not.toHaveBeenCalled(); + draining = false; await vi.advanceTimersByTimeAsync(30_000); await settled(); + expect(coordinator!.nextWakeAt()).not.toBeNull(); expect(errors).toEqual([]); + }); +}); diff --git a/server/src/__tests__/status-card-update-engine.test.ts b/server/src/__tests__/status-card-update-engine.test.ts index 09a660e8741..b8309415770 100644 --- a/server/src/__tests__/status-card-update-engine.test.ts +++ b/server/src/__tests__/status-card-update-engine.test.ts @@ -8,6 +8,7 @@ import { filterStatusCardChanges, isWithinStatusCardActiveHours, nextStatusCardEvaluationAt, + nextStatusCardPolicyEligibility, statusCardChangesHash, } from "../services/status-card-update-engine.js"; @@ -78,6 +79,28 @@ describe("status card update engine", () => { expect(evaluateStatusCardPolicy({ policy: hours, now: new Date("2026-07-23T18:00:00.000Z"), lastChangeAt: null, updatesLastHour: 0, tokensToday: 0, manual: false }).action).toBe("pause_hours"); }); + it("disarms unchanged reactive evaluation while retaining exact pending eligibility", () => { + const policy = statusCardRefreshPolicySchema.parse({ mode: "reactive", debounceSeconds: 90, maxUpdatesPerHour: 2, + activeHours: { start: "09:00", end: "17:00", timezone: "UTC" } }); + const now = new Date("2026-07-23T16:59:30Z"); + expect(nextStatusCardEvaluationAt(policy, now)).toBeNull(); + expect(nextStatusCardPolicyEligibility({ policy, now, lastChangeAt: now, completedUpdateTimes: [], tokensToday: 0 })) + .toEqual(new Date("2026-07-24T09:00:00Z")); + const open = new Date("2026-07-23T14:00:00Z"); + expect(nextStatusCardPolicyEligibility({ policy, now: open, lastChangeAt: new Date("2026-07-23T13:00:00Z"), + completedUpdateTimes: [new Date("2026-07-23T13:30:00Z"), new Date("2026-07-23T13:45:00Z")], tokensToday: 0 })) + .toEqual(new Date("2026-07-23T14:30:00.001Z")); + expect(nextStatusCardPolicyEligibility({ policy, now: open, lastChangeAt: open, completedUpdateTimes: [], tokensToday: 100_000 })) + .toEqual(new Date("2026-07-24T09:00:00Z")); + }); + + it("retains empty-window pending changes without a recurring eligibility scan", () => { + const policy = statusCardRefreshPolicySchema.parse({ mode: "reactive", debounceSeconds: 1, + activeHours: { start: "09:00", end: "09:00", timezone: "UTC" } }); + const now = new Date("2026-07-23T14:00:00Z"); + expect(nextStatusCardPolicyEligibility({ policy, now, lastChangeAt: now, completedUpdateTimes: [], tokensToday: 0 })).toBeNull(); + }); + it("selects full rebuilds for bounded drift rules and incremental otherwise", () => { const base = { hasDocument: true, changeCount: 2, queryVersion: 3, lastUpdateQueryVersion: 3, incrementalCount: 2, configurationChanged: false }; expect(chooseStatusCardUpdateKind(base)).toBe("incremental"); diff --git a/server/src/__tests__/status-cards.test.ts b/server/src/__tests__/status-cards.test.ts index e1436666316..0c24b05dd70 100644 --- a/server/src/__tests__/status-cards.test.ts +++ b/server/src/__tests__/status-cards.test.ts @@ -1,3 +1,4 @@ +import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js"; import { randomUUID } from "node:crypto"; import express from "express"; import request from "supertest"; @@ -37,7 +38,8 @@ import { startEmbeddedPostgresTestDatabase, } from "./helpers/embedded-postgres.js"; -const embeddedPostgresSupport = await getEmbeddedPostgresTestSupport(); +const externalDatabaseUrl = resolveExternalTestDatabaseUrl("PAPERCLIP_STATUS_CARDS_TEST_DATABASE_URL"); +const embeddedPostgresSupport = externalDatabaseUrl ? { supported: true } : await getEmbeddedPostgresTestSupport(); const describeEmbeddedPostgres = embeddedPostgresSupport.supported ? describe : describe.skip; type Db = ReturnType; @@ -79,8 +81,8 @@ describeEmbeddedPostgres("status card routes", () => { let tempDb: Awaited> | null = null; beforeAll(async () => { - tempDb = await startEmbeddedPostgresTestDatabase("paperclip-status-cards-"); - db = createDb(tempDb.connectionString); + if (!externalDatabaseUrl) tempDb = await startEmbeddedPostgresTestDatabase("paperclip-status-cards-"); + db = createDb(externalDatabaseUrl ?? tempDb!.connectionString, { maxConnections: 6 }); }, 20_000); afterEach(async () => { @@ -100,6 +102,7 @@ describeEmbeddedPostgres("status card routes", () => { }); afterAll(async () => { + await db?.$client.end({ timeout: 1 }); await tempDb?.cleanup(); }); diff --git a/server/src/index.ts b/server/src/index.ts index ae229f03518..6fb78f1612d 100644 --- a/server/src/index.ts +++ b/server/src/index.ts @@ -145,13 +145,12 @@ import { reconcileCodexLocalManagedHomesOnStartup, reconcilePersistedRuntimeServicesOnStartup, routineService, - statusCardService, toolAccessService, workspaceOperationService, } from "./services/index.js"; import { questionResponseDeliveryService } from "./services/question-response-delivery.js"; import { deliverNativeQuestionResponse } from "./services/native-runtime/native-question-bridge.js"; -import { queueIssueAssignmentWakeup } from "./services/issue-assignment-wakeup.js"; +import { registerStatusCardEvaluationWork, statusCardEvaluationService } from "./services/status-card-evaluation.js"; import { createSecretProposalsService } from "./services/secret-proposals.js"; import { environmentRuntimeService } from "./services/environment-runtime.js"; import { historicalCleanupVerificationService } from "./services/historical-cleanup-verification.js"; @@ -1569,7 +1568,7 @@ async function startServerWithDatabaseTeardown( if (result.settled > 0) logger.info(result, "reconciled routine execution issue status"); }).catch(() => logger.warn("routine execution status reconciliation deferred"))); }; - const statusCards = statusCardService(db as any); + const statusCardEvaluation = statusCardEvaluationService(db as any, heartbeat); const issues = issueService(db as any); const mergedPullRequestConfirmations = issueThreadInteractionService(db as any, { wakeup: heartbeat.wakeup, @@ -1980,6 +1979,8 @@ async function startServerWithDatabaseTeardown( () => !isIdleTaskDrainActive() && !isWarmStandby() && !heartbeatSchedulerStopped).ready); await step("join_request_retention", () => registerJoinRequestRetentionWork(deliveryWork, joinRequestRetentionService(db as any), () => !isIdleTaskDrainActive() && !isWarmStandby() && !heartbeatSchedulerStopped).ready); + await step("status_card_evaluation", () => registerStatusCardEvaluationWork(deliveryWork, statusCardEvaluation, + () => !isIdleTaskDrainActive() && !isWarmStandby() && !heartbeatSchedulerStopped).ready); await step("retention_sweep", runRetentionSweep); }; @@ -2010,33 +2011,7 @@ async function startServerWithDatabaseTeardown( scheduleGitHubConnectionContinuitySweep(); if (isIdleTaskDrainActive() || heartbeatSchedulerStopped) return; - trackHeartbeatSchedulerWork((async () => { - const experimental = await instanceSettingsService(db).getExperimental(); - if (experimental.enableStatusCards !== true) return; - const result = await statusCards.tickDueStatusCards(new Date()); - await Promise.all(result.enqueued.map(async ({ cardId, generatingIssue }) => { - try { - await queueIssueAssignmentWakeup({ - heartbeat, - issue: generatingIssue, - reason: "status_card_update_assigned", - mutation: "status_card.scheduler_update_requested", - contextSource: "status_card_scheduler", - requestedByActorType: "system", - taskKey: `status-card:${cardId}`, - rethrowOnError: true, - }); - } catch (err) { - await issues.update(generatingIssue.id, { status: "cancelled" }); - throw err; - } - })); - if (result.evaluated > 0 || result.enqueued.length > 0) { - logger.info({ evaluated: result.evaluated, enqueued: result.enqueued.length }, "status-card scheduler tick complete"); - } - })().catch((err) => { - logger.error({ err }, "status-card scheduler tick failed"); - })); + if (isIdleTaskDrainActive() || heartbeatSchedulerStopped) return; if (!(await heartbeat.resolveSchedulingSuppression()).suppressed) { diff --git a/server/src/modules/active-run-watchdog/adapters/postgres.ts b/server/src/modules/active-run-watchdog/adapters/postgres.ts index 9531c449260..002415c03a6 100644 --- a/server/src/modules/active-run-watchdog/adapters/postgres.ts +++ b/server/src/modules/active-run-watchdog/adapters/postgres.ts @@ -1,3 +1,4 @@ +import { notifyStatusCardIssueChange } from "../../../services/status-card-notifications.js"; import { DELIVERY_QUEUES, notifyDeliveryWork } from "../../../services/delivery-work-notifications.js"; import { notifyExternalAgentSettlement } from "../../external-agents/index.js"; import { notifyNativeStatusReassessment } from "../../../services/native-status-reassessment-notifications.js"; @@ -385,6 +386,7 @@ export function createPostgresWatchdogAdapter(db: Db): WatchdogRunReader & Watch )); } + await notifyStatusCardIssueChange(tx, companyId); await tx .update(issues) .set({ diff --git a/server/src/services/access.ts b/server/src/services/access.ts index aa0f9dcc828..477c0434c50 100644 --- a/server/src/services/access.ts +++ b/server/src/services/access.ts @@ -1,3 +1,4 @@ +import { notifyStatusCardIssueChange } from "./status-card-notifications.js"; import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js"; import { notifyNativeStatusReassessment } from "./native-status-reassessment-notifications.js"; import { notifyUnsafeWorkspaceExportRecovery } from "./unsafe-workspace-export-notifications.js"; @@ -745,6 +746,7 @@ export function accessService(db: Db) { await notifyNativeStatusReassessment(tx, { companyId }); await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors); await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers); + await notifyStatusCardIssueChange(tx, companyId); const resetInProgress = await tx .update(issues) .set({ diff --git a/server/src/services/agent-deletion.ts b/server/src/services/agent-deletion.ts index 3650c40f1a6..5907d222e1d 100644 --- a/server/src/services/agent-deletion.ts +++ b/server/src/services/agent-deletion.ts @@ -1,3 +1,4 @@ +import { notifyStatusCardIssueChange } from "./status-card-notifications.js"; import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js"; import { notifyExternalConversationSettlement } from "./external-conversation-settlement-notifications.js"; import { notifyUnsafeWorkspaceExportRecovery } from "./unsafe-workspace-export-notifications.js"; @@ -38,6 +39,7 @@ export async function deleteAgentDependencies(tx: Db, companyId: string, id: str await issueThreadInteractionService(tx).cancelPendingForDeletedAddressee(companyId, id); await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors); await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers); + await notifyStatusCardIssueChange(tx, companyId); await tx.update(agents).set({ reportsTo: null }).where(eq(agents.reportsTo, id)); await tx .update(issues) diff --git a/server/src/services/companies.ts b/server/src/services/companies.ts index 0f2ed547a79..8efd6a269fb 100644 --- a/server/src/services/companies.ts +++ b/server/src/services/companies.ts @@ -1,3 +1,4 @@ +import { notifyStatusCardIssueChange } from "./status-card-notifications.js"; import { notifyChatInboundWakeNotices } from "./chat-inbound-wake-notice-notifications.js"; import { notifyCompanyRetentionWork } from "./retention-work-notifications.js"; import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js"; @@ -336,6 +337,7 @@ export function companyService(db: Db, _budgetHooks: BudgetServiceHooks = {}) { await notifyDeliveryWork(tx, DELIVERY_QUEUES.skillSourceSync); await notifyDeliveryWork(tx, DELIVERY_QUEUES.routineSchedules); await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors); + await notifyStatusCardIssueChange(tx, id); await notifyCompanyRetentionWork(tx, id); await notifyChatInboundWakeNotices(tx, { companyId: id }); await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentLifecycle, id); @@ -492,6 +494,7 @@ export function companyService(db: Db, _budgetHooks: BudgetServiceHooks = {}) { await notifyDeliveryWork(tx, DELIVERY_QUEUES.skillSourceSync); await notifyDeliveryWork(tx, DELIVERY_QUEUES.routineSchedules); await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors); + await notifyStatusCardIssueChange(tx, id); await notifyCompanyRetentionWork(tx, id); await notifyChatInboundWakeNotices(tx, { companyId: id }); await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentLifecycle, id); diff --git a/server/src/services/company-deletion.ts b/server/src/services/company-deletion.ts index 7ab88aadfa9..0379638968b 100644 --- a/server/src/services/company-deletion.ts +++ b/server/src/services/company-deletion.ts @@ -1,3 +1,4 @@ +import { notifyStatusCardEvaluation } from "./status-card-notifications.js"; import { notifyUnsafeWorkspaceExportRecovery } from "./unsafe-workspace-export-notifications.js"; import { notifyCancelledWorkspacePreservation } from "./cancelled-workspace-preservation-notifications.js"; import { notifyInstructionCollection } from "./instruction-collection-notifications.js"; @@ -77,6 +78,7 @@ export async function deleteCompany(db: Db, id: string) { await notifyDeliveryWork(tx, DELIVERY_QUEUES.skillPackages); await notifyDeliveryWork(tx, DELIVERY_QUEUES.routineSchedules); await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors); + await notifyStatusCardEvaluation(tx, id); await notifyDeliveryWork(tx, DELIVERY_QUEUES.secretProposalExpiry, id); await notifyToolReviewChange(tx, id); await agentObservabilityService(tx as unknown as Db).removeForCompanyDeletion(id); diff --git a/server/src/services/company-search.ts b/server/src/services/company-search.ts index 220967e362d..bcf2a550d3f 100644 --- a/server/src/services/company-search.ts +++ b/server/src/services/company-search.ts @@ -207,14 +207,20 @@ function priorityRank(priority: string | null | undefined) { return index >= 0 ? index : ISSUE_PRIORITIES.length; } -function updatedWithinStart(value: string | undefined, now = new Date()): Date | null { +/** Duration used by saved search membership and its deadline consumers. */ +export function companySearchUpdatedWithinMs(value: string | undefined): number | null { if (!value) return null; const match = /^(\d+)(h|d|w|m)$/.exec(value); if (!match) return null; const amount = Number.parseInt(match[1]!, 10); const unit = match[2]; const hours = unit === "h" ? amount : unit === "d" ? amount * 24 : unit === "w" ? amount * 24 * 7 : amount * 24 * 30; - return new Date(now.getTime() - hours * 60 * 60 * 1000); + return hours * 60 * 60 * 1000; +} + +function updatedWithinStart(value: string | undefined, now = new Date()): Date | null { + const duration = companySearchUpdatedWithinMs(value); + return duration === null ? null : new Date(now.getTime() - duration); } function issueOnlyFiltersActive(query: CompanySearchQuery) { diff --git a/server/src/services/company-skills.ts b/server/src/services/company-skills.ts index 7ba950039e9..faf54c410a9 100644 --- a/server/src/services/company-skills.ts +++ b/server/src/services/company-skills.ts @@ -1,3 +1,4 @@ +import { notifyStatusCardIssueChange } from "./status-card-notifications.js"; import { notifySkillVersionRetention } from "./skill-version-retention-notifications.js"; import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js"; import { verifyFileManifest } from "./file-manifest-cache.js"; @@ -7450,6 +7451,7 @@ export function companySkillService(db: Db, options: { actor?: AuthorizationActo for (const row of rows) { await db.transaction(async (tx) => { await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers); + await notifyStatusCardIssueChange(tx, companyId); await tx .update(issues) .set({ hiddenAt: now, updatedAt: now }) diff --git a/server/src/services/delivery-work-notifications.ts b/server/src/services/delivery-work-notifications.ts index 9195f1fcb12..fdc4aaadc20 100644 --- a/server/src/services/delivery-work-notifications.ts +++ b/server/src/services/delivery-work-notifications.ts @@ -2,6 +2,7 @@ import { signalDatabaseWork, subscribeDatabaseWork } from "@paperclipai/db"; /** Topics for existing durable queues; register intent before their writes. */ export const DELIVERY_QUEUES = { + statusCardEvaluation: "status-card-evaluation", runEvidenceMigration: "run-evidence-migration", providerTraceRetention: "provider-trace-retention", nativeRunnerEvidenceRetention: "native-runner-evidence-retention", diff --git a/server/src/services/execution-control-notifications.ts b/server/src/services/execution-control-notifications.ts index 22c8913bbcd..c7b26a8bd72 100644 --- a/server/src/services/execution-control-notifications.ts +++ b/server/src/services/execution-control-notifications.ts @@ -1,7 +1,9 @@ +import { notifyStatusCardIssueChange } from "./status-card-notifications.js"; import type { Db } from "@paperclipai/db"; import { DELIVERY_QUEUES, notifyDeliveryWork, type DeliveryQueue } from "./delivery-work-notifications.js"; export async function notifyExecutionRunWork(transaction: object) { + await notifyStatusCardIssueChange(transaction); for (const queue of [DELIVERY_QUEUES.executionStatus, DELIVERY_QUEUES.executionFinalization, DELIVERY_QUEUES.executionReplacement, DELIVERY_QUEUES.executionDisposition, DELIVERY_QUEUES.issueMonitors]) { await notifyDeliveryWork(transaction, queue); @@ -14,6 +16,7 @@ export function writeExecutionControl( write: (tx: Db) => PromiseLike, ): Promise { return db.transaction(async transaction => { + await notifyStatusCardIssueChange(transaction); await notifyDeliveryWork(transaction, DELIVERY_QUEUES.issueMonitors); for (const topic of typeof queue === "string" ? [queue] : queue) await notifyDeliveryWork(transaction, topic); return write(transaction as unknown as Db); diff --git a/server/src/services/execution-workspaces.ts b/server/src/services/execution-workspaces.ts index 4b716916638..caaae186ad8 100644 --- a/server/src/services/execution-workspaces.ts +++ b/server/src/services/execution-workspaces.ts @@ -1,3 +1,4 @@ +import { notifyStatusCardIssueChange } from "./status-card-notifications.js"; import { effectiveProjectWorkspacePolicy } from "./project-workspace-provider.js"; import { inspectPluginWorkspace, removePluginWorkspace, restorePluginWorkspace, type WorkspaceWorkers } from "./plugin-workspaces.js"; import { acquireExecutionWorkspaceLifecycleLock, withExecutionWorkspaceLifecycleLock } from "./execution-workspace-lifecycle-lock.js"; @@ -1590,6 +1591,7 @@ export function executionWorkspaceService(db: Db, opts: ExecutionWorkspaceServic await validateSelection(input, tx); const intent: TaskWorkspaceIntent = { version: 1, request: { key: input.requestKey, expectedBindingRevision: input.expectedBindingRevision }, selection: input.selection, source: input.source ?? "explicit" }; const nextRevision = task.workspaceBindingRevision + 1; + await notifyStatusCardIssueChange(tx, task.companyId); await tx.update(issues).set({ workspaceBindingRevision: nextRevision, workspacePendingSelection: { version: 1, requestKey: input.requestKey, expectedBindingRevision: nextRevision, intent }, updatedAt: new Date() }).where(eq(issues.id, task.id)); @@ -1633,6 +1635,7 @@ export function executionWorkspaceService(db: Db, opts: ExecutionWorkspaceServic } } } + await notifyStatusCardIssueChange(tx, task.companyId); await tx.update(issues).set({ ...patch, assigneeAdapterOverrides, workspaceSelection: pending.intent, workspacePendingSelection: null, workspaceBindingRevision: task.workspaceBindingRevision + 1, updatedAt: new Date() }).where(eq(issues.id, task.id)); return true; @@ -1651,6 +1654,7 @@ export function executionWorkspaceService(db: Db, opts: ExecutionWorkspaceServic const pendingAfterFirstBinding = !task.executionWorkspaceId && pending?.expectedBindingRevision === task.workspaceBindingRevision ? { ...pending, expectedBindingRevision: nextRevision } : pending; + await notifyStatusCardIssueChange(tx, companyId); const [bound] = await tx.update(issues).set({ ...patch, executionWorkspaceId: workspaceId, workspaceBindingRevision: nextRevision, workspacePendingSelection: pendingAfterFirstBinding, updatedAt: new Date() }) .where(eq(issues.id, issueId)).returning(); diff --git a/server/src/services/heartbeat/retries.ts b/server/src/services/heartbeat/retries.ts index 946f5e309f8..de03aebeca0 100644 --- a/server/src/services/heartbeat/retries.ts +++ b/server/src/services/heartbeat/retries.ts @@ -1,3 +1,4 @@ +import { notifyStatusCardIssueChange } from "../status-card-notifications.js"; import { deriveTaskKeyWithHeartbeatFallback } from "./run-state.js"; import { infrastructureRetryDelayMs } from "../system-recovery-state.js"; import { @@ -1178,6 +1179,7 @@ export function createHeartbeatRetries(db: Db, dependencies: HeartbeatRetryDepen } if (issueId) { + await notifyStatusCardIssueChange(tx, run.companyId); await tx .update(issues) .set({ diff --git a/server/src/services/instance-settings.ts b/server/src/services/instance-settings.ts index 0208653a86f..35eddc10660 100644 --- a/server/src/services/instance-settings.ts +++ b/server/src/services/instance-settings.ts @@ -1,3 +1,4 @@ +import { notifyStatusCardEvaluation } from "./status-card-notifications.js"; import { notifySkillVersionRetentionReconstruction } from "./skill-version-retention-notifications.js"; import { notifyNativeEnvironmentSettlement } from "./native-environment-settlement-notifications.js"; import { notifyExternalAgentSettlement } from "../modules/external-agents/index.js"; @@ -625,6 +626,7 @@ export function instanceSettingsService(db: Db, options: InstanceSettingsService } const now = new Date(); const [updated] = await db.transaction(async tx => { + if (patch.enableStatusCards !== undefined) await notifyStatusCardEvaluation(tx); await notifyDeliveryWork(tx, DELIVERY_QUEUES.mcpEvents, "*"); await notifyDeliveryWork(tx, DELIVERY_QUEUES.dotEvents, "*"); await notifyDeliveryWork(tx, DELIVERY_QUEUES.grokDispatch); diff --git a/server/src/services/issue-comment-writes.ts b/server/src/services/issue-comment-writes.ts index e24e78e9855..4d4cf374e5b 100644 --- a/server/src/services/issue-comment-writes.ts +++ b/server/src/services/issue-comment-writes.ts @@ -1,3 +1,4 @@ +import { notifyStatusCardIssueChange } from "./status-card-notifications.js"; import type { PgInsertValue } from "drizzle-orm/pg-core"; import { issueComments, type Db } from "@paperclipai/db"; import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js"; @@ -6,6 +7,7 @@ type WriteDb = Db | Parameters[0]>[0]; export function insertIssueComment(db: WriteDb, input: PgInsertValue & { companyId: string }) { return db.transaction(async tx => { + await notifyStatusCardIssueChange(tx, input.companyId); await notifyDeliveryWork(tx, DELIVERY_QUEUES.dotEvents, `company:${input.companyId}`); return tx.insert(issueComments).values(input).returning(); }); diff --git a/server/src/services/issue-prefix.ts b/server/src/services/issue-prefix.ts index 5e31f9bd7ff..649a4aaae9b 100644 --- a/server/src/services/issue-prefix.ts +++ b/server/src/services/issue-prefix.ts @@ -1,3 +1,4 @@ +import { notifyStatusCardIssueChange } from "./status-card-notifications.js"; import { and, eq, sql } from "drizzle-orm"; import type { PgColumn } from "drizzle-orm/pg-core"; import type { Db } from "@paperclipai/db"; @@ -127,6 +128,7 @@ export async function rekeyCompanyIssueIdentifiers( // overload, which returns NULL for a non-pattern argument. const tailStart = sql`${fromPrefix.length + 1}::int`; + await notifyStatusCardIssueChange(tx, companyId); const rekeyedIssues = await tx .update(issues) .set({ identifier: sql`${toPrefix} || substring(${issues.identifier} from ${tailStart})` }) diff --git a/server/src/services/issue-review-state-notifications.ts b/server/src/services/issue-review-state-notifications.ts index 679b7e37475..d66ca96eb0e 100644 --- a/server/src/services/issue-review-state-notifications.ts +++ b/server/src/services/issue-review-state-notifications.ts @@ -1,9 +1,11 @@ +import { notifyStatusCardIssueChange } from "./status-card-notifications.js"; import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js"; import { notifyExternalConversationSettlement } from "./external-conversation-settlement-notifications.js"; import { notifyNativeReviewResponse } from "./native-review-response-notifications.js"; import { notifyNativeStatusReassessment } from "./native-status-reassessment-notifications.js"; export async function notifyIssueReviewState(tx: object, scope: { companyId: string; issueId?: string; runId?: string; agentId?: string }) { + await notifyStatusCardIssueChange(tx, scope.companyId); await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors); await notifyExternalConversationSettlement(tx, scope); await notifyNativeReviewResponse(tx, scope); diff --git a/server/src/services/issue-title.ts b/server/src/services/issue-title.ts index af6716e3aa4..51e360269d7 100644 --- a/server/src/services/issue-title.ts +++ b/server/src/services/issue-title.ts @@ -1,3 +1,4 @@ +import { notifyStatusCardIssueChange } from "./status-card-notifications.js"; import { and, eq } from "drizzle-orm"; import { heartbeatRuns, issues, type Db } from "@paperclipai/db"; import { setIssueTitleSchema, type SetIssueTitle } from "@paperclipai/shared"; @@ -76,6 +77,7 @@ export async function setIssueTitle( || (issue.title === title && !issue.titleNeedsGeneration)) { return { result: await saveReceipt({ id: issue.id, title: issue.title, titleNeedsGeneration: issue.titleNeedsGeneration, changed: false }), publication: null }; } + await notifyStatusCardIssueChange(tx, companyId); await tx.update(issues).set({ title, titleNeedsGeneration: false, updatedAt: new Date() }).where(predicate); await issueReferenceService(tx).syncIssue(issueId, tx); const { publication } = await persistActivity(tx, { diff --git a/server/src/services/issue-tree-control.ts b/server/src/services/issue-tree-control.ts index 0d7633f846a..a8009559827 100644 --- a/server/src/services/issue-tree-control.ts +++ b/server/src/services/issue-tree-control.ts @@ -1,3 +1,4 @@ +import { notifyStatusCardIssueChange } from "./status-card-notifications.js"; import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js"; import { notifyNativeStatusReassessment } from "./native-status-reassessment-notifications.js"; import { notifyUnsafeWorkspaceExportRecovery } from "./unsafe-workspace-export-notifications.js"; @@ -1011,6 +1012,7 @@ export function issueTreeControlService(db: Db) { await notifyChatResponseAvailability(tx, { companyId, issueId }); if (status === "blocked" || status === "in_review") await notifyUnsafeWorkspaceExportRecovery(tx, { companyId, issueId }); } + await notifyStatusCardIssueChange(tx, companyId); const rows = await tx .update(issues) .set({ diff --git a/server/src/services/issues.ts b/server/src/services/issues.ts index 42c65bb5d02..8787ac867a8 100644 --- a/server/src/services/issues.ts +++ b/server/src/services/issues.ts @@ -1,3 +1,4 @@ +import { notifyStatusCardIssueChange } from "./status-card-notifications.js"; import { notifyExternalConversationSettlement } from "./external-conversation-settlement-notifications.js"; import { notifyNativeRecoveryNotices } from "./native-recovery-notice-notifications.js"; import { notifyNativeRunProjection } from "./native-run-projection-notifications.js"; @@ -10606,6 +10607,7 @@ export function issueService(db: Db) { } if (values.monitorNextCheckAt) await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors); await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers); + await notifyStatusCardIssueChange(tx, companyId); const [issue] = await tx.insert(issues).values(values).returning(); await attachOwnedDraftImages(tx, issue, issue.description, { userId: issueData.createdByUserId, agentId: issueData.createdByAgentId }); if (issue.visibility === "private" && !inheritedPrivateSubtree && issueData.createdByAgentId) { @@ -11486,6 +11488,7 @@ export function issueService(db: Db) { (blockedByIssueIds !== undefined || issueData.unblockDescriptor !== undefined)))) { patch.statusVersion = sql`${issues.statusVersion} + 1` as unknown as number; } + await notifyStatusCardIssueChange(tx, receiptExisting.companyId); if ((patch.status !== undefined && patch.status !== receiptExisting.status) || patch.completedAt !== undefined || patch.cancelledAt !== undefined || issueData.parentId !== undefined || issueData.executionWorkspaceId !== undefined) { @@ -11954,6 +11957,7 @@ export function issueService(db: Db) { if (owner) { await notifyExternalConversationSettlement(tx, { companyId: owner.companyId }); await notifyApprovalWork(tx, owner.companyId); + await notifyStatusCardIssueChange(tx, owner.companyId); } await notifyDecisionTargetIssueRemoval(tx, id); await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors); @@ -12726,6 +12730,7 @@ export function issueService(db: Db) { .from(issueComments).where(eq(issueComments.id, commentId)); if (!source) return null; await notifyExternalConversationSettlement(tx, source); + await notifyStatusCardIssueChange(tx, source.companyId); const [comment] = await tx .delete(issueComments) .where(eq(issueComments.id, commentId)) @@ -12765,6 +12770,7 @@ export function issueService(db: Db) { .for("update"); if (!source) return null; await notifyExternalConversationSettlement(tx, source); + await notifyStatusCardIssueChange(tx, source.companyId); await notifyChatInboundWakeNotices(tx, source); const now = new Date(); const [comment] = await tx diff --git a/server/src/services/native-runtime/native-finalization-reconciler.ts b/server/src/services/native-runtime/native-finalization-reconciler.ts index af828edfb98..c5cf832982e 100644 --- a/server/src/services/native-runtime/native-finalization-reconciler.ts +++ b/server/src/services/native-runtime/native-finalization-reconciler.ts @@ -1,3 +1,4 @@ +import { notifyStatusCardIssueChange } from "../status-card-notifications.js"; import { notifyExternalAgentSettlement } from "../../modules/external-agents/index.js"; import { pendingNativeRunProjectionCondition } from "./native-run-projection.js"; import { nativeStatusReassessmentSourceIsSuperseded, reassessNativeStatusFromSource } from "./native-status-reassessment.js"; @@ -334,6 +335,7 @@ export async function claimNativeSessionResumptions(input: { nativePhaseUpdatedAt: now, updatedAt: now, }).where(eq(heartbeatRuns.id, row.run.id)); + await notifyStatusCardIssueChange(tx, run.companyId); await tx.update(issues).set({ executionRunId: run.id, executionAgentNameKey: agent.name.trim().toLowerCase() || null, diff --git a/server/src/services/native-runtime/native-review-participant.ts b/server/src/services/native-runtime/native-review-participant.ts index 573574dd19d..6391d506d5d 100644 --- a/server/src/services/native-runtime/native-review-participant.ts +++ b/server/src/services/native-runtime/native-review-participant.ts @@ -1,3 +1,4 @@ +import { notifyStatusCardIssueChange } from "../status-card-notifications.js"; import { agents, heartbeatRuns, @@ -240,6 +241,7 @@ export async function claimNativeReviewExecutionLock(db: Db, input: { if (!run || !["queued", "running"].includes(run.status) || !admitted || !expected || admitted.nativeReviewInteractionId !== expected.nativeReviewInteractionId || admitted.nativeReviewDecisionId !== expected.nativeReviewDecisionId) return false; + await notifyStatusCardIssueChange(tx, input.companyId); const claimed = await tx.update(issues).set({ executionRunId: input.runId, executionAgentNameKey: input.agentNameKey, diff --git a/server/src/services/native-runtime/native-workspace-export-recovery.ts b/server/src/services/native-runtime/native-workspace-export-recovery.ts index 41e51f2ddc7..0c3aac34799 100644 --- a/server/src/services/native-runtime/native-workspace-export-recovery.ts +++ b/server/src/services/native-runtime/native-workspace-export-recovery.ts @@ -1,3 +1,4 @@ +import { notifyStatusCardIssueChange } from "../status-card-notifications.js"; import { notifyNativeFinalizationWork } from "../native-finalization-notifications.js"; import { alias } from "drizzle-orm/pg-core"; import { notifyUnsafeWorkspaceExportRecovery, type UnsafeWorkspaceExportScope } from "../unsafe-workspace-export-notifications.js"; @@ -107,6 +108,7 @@ async function recoverCandidate(db: Db, candidate: { runId: string; companyId: s nativePhase: "result_accepted", nativePhaseUpdatedAt: now, resultJson: { ...run.resultJson, finalizationPhase: "result_accepted", failureCode: null, originalFailureCode: null, nextAttemptAt: null }, updatedAt: now, }).where(eq(heartbeatRuns.id, run.id)).returning(); + await notifyStatusCardIssueChange(tx, run.companyId); await tx.update(issues).set({ executionRunId: run.id, updatedAt: now }).where(eq(issues.id, issue.id)); await appendHeartbeatRunEvent(tx as unknown as Db, { companyId: run.companyId, agentId: run.agentId, runId: run.id, eventType: "workspace_export_omitted", stream: "system", level: "info", diff --git a/server/src/services/native-runtime/native-workspace-export-retry.ts b/server/src/services/native-runtime/native-workspace-export-retry.ts index 47de8c73cf4..5a79c9e782e 100644 --- a/server/src/services/native-runtime/native-workspace-export-retry.ts +++ b/server/src/services/native-runtime/native-workspace-export-retry.ts @@ -1,3 +1,4 @@ +import { notifyStatusCardIssueChange } from "../status-card-notifications.js"; import { notifyNativeFinalizationWork } from "../native-finalization-notifications.js"; import { notifyEnvironmentCleanup } from "../environment-cleanup-notifications.js"; import { randomUUID } from "node:crypto"; @@ -141,6 +142,7 @@ export async function retryNativeWorkspaceExport(input: { await tx.update(heartbeatRuns).set({ status: "running", finishedAt: null, error: null, errorCode: null, nativePhase: "result_accepted", nativePhaseUpdatedAt: now, resultJson: { ...current.run.resultJson, workspaceExportRetry: retry, finalizationPhase: "result_accepted", failureCode: null, originalFailureCode: null, nextAttemptAt: null }, updatedAt: now }).where(eq(heartbeatRuns.id, current.run.id)); + await notifyStatusCardIssueChange(tx, current.run.companyId); await tx.update(issues).set({ executionRunId: current.run.id, updatedAt: now }).where(eq(issues.id, current.issue.id)); // Reclaim the same lease only for control-plane copyback. The normal // finalization settlement will stop/retain it again on success or failure. diff --git a/server/src/services/native-runtime/native-workspace-reconstruction.ts b/server/src/services/native-runtime/native-workspace-reconstruction.ts index 5273b30216a..33e37704389 100644 --- a/server/src/services/native-runtime/native-workspace-reconstruction.ts +++ b/server/src/services/native-runtime/native-workspace-reconstruction.ts @@ -1,3 +1,4 @@ +import { notifyStatusCardIssueChange } from "../status-card-notifications.js"; import { DELIVERY_QUEUES, notifyDeliveryWork } from "../delivery-work-notifications.js"; import { notifyUnsafeWorkspaceExportRecovery } from "../unsafe-workspace-export-notifications.js"; import { and, eq, gt, inArray, ne, notInArray, or, sql } from "drizzle-orm"; @@ -187,6 +188,7 @@ export async function reconstructNativeWorkspace(input: { }).where(eq(issueRecoveryActions.id, source.action.id)); await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors); await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers); + await notifyStatusCardIssueChange(tx, input.companyId); await tx.update(issues).set({ status: "todo", unblockDescriptor: null, blockedTransitionAt: null, blockedOwnerNotifiedAt: null, updatedAt: now }).where(eq(issues.id, issue.id)); const publication = await persistActivity(db, { companyId: input.companyId, actorType: "user", actorId: input.actorId, diff --git a/server/src/services/native-runtime/paperclip-runner-tool-authority.ts b/server/src/services/native-runtime/paperclip-runner-tool-authority.ts index aa8970db19f..cf07e70eb7a 100644 --- a/server/src/services/native-runtime/paperclip-runner-tool-authority.ts +++ b/server/src/services/native-runtime/paperclip-runner-tool-authority.ts @@ -1,3 +1,4 @@ +import { notifyStatusCardIssueChange } from "../status-card-notifications.js"; import { requiredQualityReviewSchema } from "@paperclipai/shared"; import { priorRecoveryApiReceipt } from "../recovery-effect-receipts.js"; import { DELIVERY_QUEUES, notifyDeliveryWork } from "../delivery-work-notifications.js"; @@ -2059,6 +2060,7 @@ export class PaperclipRunnerToolAuthority { } const patch = prepareIssueMonitorUpdate(target, input.monitor, actor); await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors); + await notifyStatusCardIssueChange(tx, this.binding.companyId); const [updated] = await tx.update(issues).set({ ...patch, updatedAt: new Date() }) .where(and(eq(issues.id, target.id), eq(issues.companyId, this.binding.companyId))).returning(); if (!updated) throw notFound("Task not found"); diff --git a/server/src/services/plugin-workspaces.ts b/server/src/services/plugin-workspaces.ts index e2383dcfc74..ca04b081e48 100644 --- a/server/src/services/plugin-workspaces.ts +++ b/server/src/services/plugin-workspaces.ts @@ -1,3 +1,4 @@ +import { notifyStatusCardIssueChange } from "./status-card-notifications.js"; import { effectiveProjectWorkspacePolicy, isWorkspaceProvider } from "./project-workspace-provider.js"; import { releaseProviderRecoveryTransfers, stageProviderFreshWorkspace } from "./provider-fresh-workspace.js"; import { randomUUID, createHash } from "node:crypto"; @@ -418,6 +419,7 @@ export async function preparePluginWorkspace( }, }) .returning(); + await notifyStatusCardIssueChange(tx, task.companyId); await tx .update(issues) .set({ diff --git a/server/src/services/provider-fresh-workspace.ts b/server/src/services/provider-fresh-workspace.ts index fcc184e15f4..98a478a375d 100644 --- a/server/src/services/provider-fresh-workspace.ts +++ b/server/src/services/provider-fresh-workspace.ts @@ -1,3 +1,4 @@ +import { notifyStatusCardIssueChange } from "./status-card-notifications.js"; import { assertTaskWorkspaceExecutionAccess } from "./task-workspace-access.js"; import { isDeepStrictEqual } from "node:util"; import { createHash, randomUUID } from "node:crypto"; @@ -152,6 +153,7 @@ export async function reserveFreshPluginTaskWorkspace(input: { providerType: "adapter_managed", cwd: `/workspaces/${input.fresh.attemptId}`, metadata: { runnerTaskFreshWorkspace: input.fresh, runnerTaskWorkspaceOwner: input.owner }, }); + await notifyStatusCardIssueChange(tx, input.companyId); await tx.update(issues).set({ executionWorkspaceId: input.fresh.attemptId, executionWorkspacePreference: "reuse_existing", workspaceBindingRevision: task.workspaceBindingRevision + 1, workspacePendingSelection: task.workspacePendingSelection?.expectedBindingRevision === task.workspaceBindingRevision diff --git a/server/src/services/routines.ts b/server/src/services/routines.ts index 23787b45249..a831a3e489a 100644 --- a/server/src/services/routines.ts +++ b/server/src/services/routines.ts @@ -1,3 +1,4 @@ +import { notifyStatusCardIssueChange } from "./status-card-notifications.js"; import { notifyApprovalWork } from "./approval-work-notifications.js"; import { notifyDecisionTargetIssueRemoval } from "./decision-targets.js"; import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js"; @@ -1949,6 +1950,7 @@ export function routineService( if (createdIssue) { await notifyApprovalWork(txDb, createdIssue.companyId); await notifyDecisionTargetIssueRemoval(txDb, createdIssue.id); + await notifyStatusCardIssueChange(txDb, createdIssue.companyId); await txDb.delete(issues).where(eq(issues.id, createdIssue.id)); } const failureReason = error instanceof Error ? error.message : String(error); diff --git a/server/src/services/slack-conversation-lifecycle.ts b/server/src/services/slack-conversation-lifecycle.ts index 0fd213dc084..9e784ff005d 100644 --- a/server/src/services/slack-conversation-lifecycle.ts +++ b/server/src/services/slack-conversation-lifecycle.ts @@ -1,3 +1,4 @@ +import { notifyStatusCardIssueChange } from "./status-card-notifications.js"; import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js"; import { notifyUnsafeWorkspaceExportRecovery } from "./unsafe-workspace-export-notifications.js"; import { and, asc, desc, eq, gt, inArray, isNull, sql } from "drizzle-orm"; @@ -133,6 +134,7 @@ export async function settleExternalConversation(db: Db, companyId: string, issu await notifyUnsafeWorkspaceExportRecovery(tx, { companyId, issueId }); await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors); await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers); + await notifyStatusCardIssueChange(tx, companyId); await tx.update(issues).set({ status: "in_review", updatedAt: new Date() }) .where(and(eq(issues.id, issueId), eq(issues.companyId, companyId))); activity = (await persistActivity(tx as unknown as Db, { diff --git a/server/src/services/slack-conversation-state.ts b/server/src/services/slack-conversation-state.ts index dedaa8d6676..08f6c9c038b 100644 --- a/server/src/services/slack-conversation-state.ts +++ b/server/src/services/slack-conversation-state.ts @@ -1,3 +1,4 @@ +import { notifyStatusCardIssueChange } from "./status-card-notifications.js"; import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js"; import { notifyExternalConversationSettlement } from "./external-conversation-settlement-notifications.js"; import { chatConversations, issues, type Db } from "@paperclipai/db"; @@ -49,6 +50,7 @@ export async function resumeExternalConversation(tx: Db, companyId: string, issu await notifyExternalConversationSettlement(tx, { companyId, issueId }); // Only the verified idle convention may leave In Review automatically. // A real decision or delivery problem keeps the existing review semantics. + await notifyStatusCardIssueChange(tx, companyId); await tx.update(issues).set({ status: "todo", updatedAt: new Date() }) .where(and(eq(issues.companyId, companyId), eq(issues.id, issueId), sql`${externalConversationStateSql()} = 'waiting'`)); diff --git a/server/src/services/status-card-evaluation.ts b/server/src/services/status-card-evaluation.ts new file mode 100644 index 00000000000..143df32344a --- /dev/null +++ b/server/src/services/status-card-evaluation.ts @@ -0,0 +1,90 @@ +import { and, asc, eq, isNotNull, isNull } from "drizzle-orm"; +import { companies, statusCards, type Db } from "@paperclipai/db"; +import { logger } from "../middleware/logger.js"; +import { registerDeadlineWork } from "./deadline-work.js"; +import type { createDeliveryWorkCoordinator } from "./delivery-work-coordinator.js"; +import { DELIVERY_QUEUES } from "./delivery-work-notifications.js"; +import { instanceSettingsService } from "./instance-settings.js"; +import { issueService } from "./issues.js"; +import { queueIssueAssignmentWakeup, type IssueAssignmentWakeupDeps } from "./issue-assignment-wakeup.js"; +import { statusCardService } from "./status-cards.js"; + +/** Feature-owned generation and assignment. Existing issue services retain + * execution authority, durable wake dedupe and stalled-generation cleanup. */ +export function statusCardEvaluationService(db: Db, heartbeat: IssueAssignmentWakeupDeps) { + const cards = statusCardService(db); + const issues = issueService(db); + const settings = instanceSettingsService(db); + let reconstructReactive = true; + let allChanges = false; + const changes = new Set(); + let enabled = false; + + return { + changed(hints?: ReadonlySet) { + for (const hint of hints ?? ["*"]) { + if (hint === "*" || hint === "changes:*") allChanges = true; + else if (hint.startsWith("changes:")) changes.add(hint.slice("changes:".length)); + } + }, + async run(signal: AbortSignal) { + enabled = (await settings.getExperimental()).enableStatusCards === true; + if (!enabled || signal.aborted) return; + const reconstruct = reconstructReactive; + const evaluateAllChanges = allChanges; + const companiesChanged = new Set(changes); + changes.clear(); + reconstructReactive = false; + allChanges = false; + try { + const result = await cards.tickDueStatusCards(new Date(), evaluateAllChanges ? "changed" : reconstruct && companiesChanged.size === 0 ? "all" : companiesChanged); + const assignments = await Promise.allSettled(result.enqueued.map(async ({ cardId, generatingIssue }) => { + // Once generated, dispatch or cancel even if shutdown begins: leaving + // an unassigned claimed card is not a completed sweep. + try { + await queueIssueAssignmentWakeup({ + heartbeat, issue: generatingIssue, + reason: "status_card_update_assigned", + mutation: "status_card.scheduler_update_requested", + contextSource: "status_card_scheduler", + requestedByActorType: "system", taskKey: `status-card:${cardId}`, + rethrowOnError: true, + }); + } catch (error) { + await issues.update(generatingIssue.id, { status: "cancelled" }); + throw error; + } + })); + const failures = assignments.filter(result => result.status === "rejected"); + if (failures.length) throw new AggregateError(failures.map(result => result.reason), "Status-card assignment failed"); + if (result.evaluated || result.enqueued.length) { + logger.info({ evaluated: result.evaluated, enqueued: result.enqueued.length }, "status-card evaluation complete"); + } + } catch (error) { + reconstructReactive ||= reconstruct; + allChanges ||= evaluateAllChanges; + for (const companyId of companiesChanged) changes.add(companyId); + throw error; + } + }, + async next() { + if (!enabled) return null; + const [next] = await db.select({ at: statusCards.nextEvalAt }).from(statusCards) + .innerJoin(companies, eq(companies.id, statusCards.companyId)) + .where(and(eq(companies.status, "active"), isNull(statusCards.archivedAt), + isNull(statusCards.generatingIssueId), isNotNull(statusCards.nextEvalAt))) + .orderBy(asc(statusCards.nextEvalAt)).limit(1); + return next?.at?.getTime() ?? null; + }, + }; +} + +export function registerStatusCardEvaluationWork( + coordinator: ReturnType, + service: ReturnType, + canRun: () => boolean, +) { + return registerDeadlineWork(coordinator, DELIVERY_QUEUES.statusCardEvaluation, { + run: service.run, next: service.next, onSettled: service.changed, canRun, retryMs: 30_000, + }); +} diff --git a/server/src/services/status-card-finalization.ts b/server/src/services/status-card-finalization.ts index 7c7d260992e..5d7ee62b523 100644 --- a/server/src/services/status-card-finalization.ts +++ b/server/src/services/status-card-finalization.ts @@ -1,3 +1,4 @@ +import { notifyStatusCardEvaluation } from "./status-card-notifications.js"; import { and, eq, isNull } from "drizzle-orm"; import type { Db } from "@paperclipai/db"; import { statusCards, statusCardUpdates } from "@paperclipai/db"; @@ -36,6 +37,7 @@ export async function finalizeStatusCardsForStalledGeneration( ) { if (!STALLED_GENERATION_STATUSES.has(issue.status)) return []; + await notifyStatusCardEvaluation(dbOrTx, issue.companyId); const now = new Date(); const failureReason = failureReasonForIssue(issue); const cards = await dbOrTx diff --git a/server/src/services/status-card-notifications.ts b/server/src/services/status-card-notifications.ts new file mode 100644 index 00000000000..6fc12985847 --- /dev/null +++ b/server/src/services/status-card-notifications.ts @@ -0,0 +1,10 @@ +import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js"; + +/** Card/settings changes reconstruct deadlines; issue changes also evaluate reactive scope. */ +export function notifyStatusCardEvaluation(transaction: object, companyId?: string) { + return notifyDeliveryWork(transaction, DELIVERY_QUEUES.statusCardEvaluation, companyId ? `cards:${companyId}` : "*"); +} + +export function notifyStatusCardIssueChange(transaction: object, companyId?: string) { + return notifyDeliveryWork(transaction, DELIVERY_QUEUES.statusCardEvaluation, companyId ? `changes:${companyId}` : "changes:*"); +} diff --git a/server/src/services/status-card-update-engine.ts b/server/src/services/status-card-update-engine.ts index ff14d35c640..1f734227808 100644 --- a/server/src/services/status-card-update-engine.ts +++ b/server/src/services/status-card-update-engine.ts @@ -128,10 +128,8 @@ export function isWithinStatusCardActiveHours(policy: StatusCardRefreshPolicy, n } export function nextStatusCardEvaluationAt(policy: StatusCardRefreshPolicy, now: Date) { - if (policy.mode === "manual") return null; - const seconds = policy.mode === "interval" - ? (policy.intervalMinutes ?? 15) * 60 - : Math.min(policy.debounceSeconds ?? 60, 60); + if (policy.mode !== "interval") return null; + const seconds = (policy.intervalMinutes ?? 15) * 60; return new Date(now.getTime() + seconds * 1000); } @@ -172,3 +170,37 @@ export function evaluateStatusCardPolicy(input: { } return { action: "run" as const }; } + +/** A retry exists only while filtered changes remain. Use the owning policy's + * actual eligibility boundary instead of repeatedly rediscovering a closed gate. */ +export function nextStatusCardPolicyEligibility(input: { + policy: StatusCardRefreshPolicy; + now: Date; + lastChangeAt: Date; + completedUpdateTimes: Date[]; + tokensToday: number; +}): Date | null { + if (input.policy.mode === "manual") return null; + let at = input.now.getTime(); + if (input.tokensToday >= (input.policy.dailyTokenCap ?? 100_000)) { + const day = new Date(input.now); + day.setUTCHours(24, 0, 0, 0); + at = Math.max(at, day.getTime()); + } + if (input.policy.mode === "reactive") { + at = Math.max(at, input.lastChangeAt.getTime() + (input.policy.debounceSeconds ?? 60) * 1000); + const cap = input.policy.maxUpdatesPerHour ?? 6; + const recent = input.completedUpdateTimes.map(time => time.getTime()) + .filter(time => time > at - 3_600_000).sort((a, b) => a - b); + if (recent.length >= cap) at = Math.max(at, recent[recent.length - cap]! + 3_600_000 + 1); + } + if (isWithinStatusCardActiveHours(input.policy, new Date(at))) return new Date(at); + // Active-hour precision is one local minute. Walking UTC minutes also honors + // skipped/repeated local times at daylight-saving transitions. + const end = at + 48 * 3_600_000; + for (at = Math.floor(at / 60_000) * 60_000 + 60_000; at <= end; at += 60_000) { + if (isWithinStatusCardActiveHours(input.policy, new Date(at))) return new Date(at); + } + // Equal start/end describes an empty window; retain changes, awaiting config. + return null; +} diff --git a/server/src/services/status-cards.ts b/server/src/services/status-cards.ts index 4a7439b6aa5..84bbf7b2cd4 100644 --- a/server/src/services/status-cards.ts +++ b/server/src/services/status-cards.ts @@ -1,7 +1,9 @@ +import { notifyStatusCardEvaluation } from "./status-card-notifications.js"; import { createHash } from "node:crypto"; import { and, desc, eq, gte, inArray, isNotNull, isNull, lte, ne, or, sql, type SQL } from "drizzle-orm"; import { agents, + companies, costEvents, documentRevisions, documents, @@ -23,7 +25,7 @@ import { conflict, forbidden, notFound, unprocessable } from "../errors.js"; import { logger } from "../middleware/logger.js"; import { readBuiltInAgentMarker } from "../lib/built-in-agent-metadata.js"; import { builtInAgentService } from "./built-in-agents.js"; -import { companySearchService } from "./company-search.js"; +import { companySearchService, companySearchUpdatedWithinMs } from "./company-search.js"; import { issueService } from "./issues.js"; import { SUMMARIZER_BUILT_IN_KEY } from "./summary-slots.js"; import { @@ -34,6 +36,7 @@ import { extractIssueMentions, filterStatusCardChanges, nextStatusCardEvaluationAt, + nextStatusCardPolicyEligibility, STATUS_CARD_MAX_MENTIONED_ISSUES, statusCardChangesHash, statusCardFingerprintHash, @@ -44,6 +47,7 @@ import { type StatusCardActor = { agentId: string | null; userId: string | null }; type StatusCardWriter = { agentId: string | null; runId: string | null }; type StatusCardRow = typeof statusCards.$inferSelect; +type WatchedIssue = CompanySearchIssueSummary & { scopeExpiresAt?: Date | null }; type StatusCardReadOptions = { issueReadCondition?: SQL }; const TERMINAL_ISSUE_STATUSES = new Set(["done", "cancelled"]); @@ -151,9 +155,15 @@ export function statusCardService( db: Db, deps: { issuesSvc?: ReturnType } = {}, ) { - const builtIns = builtInAgentService(db); - const issuesSvc = deps.issuesSvc ?? issueService(db); + const issuesSvc = () => deps.issuesSvc ?? issueService(db); const searchSvc = companySearchService(db); + type Transaction = Parameters[0]>[0]; + function mutate(companyId: string | undefined, write: (tx: Transaction) => Promise) { + return db.transaction(async tx => { + await notifyStatusCardEvaluation(tx, companyId); + return write(tx); + }); + } async function readWatchedIssueVisibility(card: StatusCardRow, options: StatusCardReadOptions = {}) { if (card.queries.length === 0 && (card.mentionedIssueIds?.length ?? 0) === 0) { @@ -240,10 +250,10 @@ export function statusCardService( }; const agentId = actor.agentId; if (!agentId) { - return db.insert(statusCards).values(values).returning().then((rows) => rows[0]!); + return mutate(companyId, tx => tx.insert(statusCards).values(values).returning().then((rows) => rows[0]!)); } - return db.transaction(async (tx) => { + return mutate(companyId, async (tx) => { const author = await tx .select({ id: agents.id }) .from(agents) @@ -292,7 +302,7 @@ export function statusCardService( ...(input.refreshPolicy !== undefined ? { refreshPolicy: input.refreshPolicy, - nextEvalAt: card.archivedAt ? null : nextStatusCardEvaluationAt(input.refreshPolicy, now), + nextEvalAt: card.archivedAt ? null : input.refreshPolicy.mode === "reactive" && card.queries.length > 0 ? now : nextStatusCardEvaluationAt(input.refreshPolicy, now), } : {}), ...(archiveChanged && input.archived @@ -309,21 +319,21 @@ export function statusCardService( } : {}), }; - const next = await db.update(statusCards).set({ + const next = await mutate(card.companyId, tx => tx.update(statusCards).set({ ...values, ...(archiveChanged && input.archived ? { generatingIssueId: null, pendingChangeHash: null } : {}), - }).where(eq(statusCards.id, card.id)).returning().then((rows) => rows[0]!); + }).where(eq(statusCards.id, card.id)).returning().then((rows) => rows[0]!)); if (archiveChanged && input.archived && card.generatingIssueId) { const generationIssue = await db.select().from(issues).where(eq(issues.id, card.generatingIssueId)).then((rows) => rows[0] ?? null); if (generationIssue && !TERMINAL_ISSUE_STATUSES.has(generationIssue.status)) { - await issuesSvc.update(generationIssue.id, { status: "cancelled" }); + await issuesSvc().update(generationIssue.id, { status: "cancelled" }); } } return next; } async function remove(id: string) { - return db.delete(statusCards).where(eq(statusCards.id, id)).returning().then((rows) => rows[0] ?? null); + return mutate(undefined, tx => tx.delete(statusCards).where(eq(statusCards.id, id)).returning().then((rows) => rows[0] ?? null)); } async function listUpdates(cardId: string) { @@ -360,7 +370,7 @@ export function statusCardService( .then((rows) => rows[0] ?? null); if (override) return override.id; } - const builtIn = await builtIns.get(card.companyId, SUMMARIZER_BUILT_IN_KEY); + const builtIn = await builtInAgentService(db).get(card.companyId, SUMMARIZER_BUILT_IN_KEY); if (builtIn.status !== "ready" || !builtIn.agentId) { throw unprocessable("Summarizer built-in agent is not configured", { code: "summarizer_not_configured", @@ -391,7 +401,7 @@ export function statusCardService( let deduplicated = false; const createdAt = new Date(); - const created = await issuesSvc.create(card.companyId, { + const created = await issuesSvc().create(card.companyId, { title: `Compile status card: ${card.title ?? card.interestPrompt.slice(0, 80)}`, description: compileDescription(card, null, hash), status: "todo", @@ -409,16 +419,16 @@ export function statusCardService( // covers idempotency-key hits that resolve to a terminal task (done/cancelled) // as well as a `blocked` one that a manual re-kick is reviving. const reopened = deduplicated && (TERMINAL_ISSUE_STATUSES.has(created.status) || created.status === "blocked") - ? await issuesSvc.update(created.id, { status: "todo", assigneeAgentId: summarizerAgentId }) + ? await issuesSvc().update(created.id, { status: "todo", assigneeAgentId: summarizerAgentId }) : created; - const generationIssue = await issuesSvc.update(reopened!.id, { + const generationIssue = await issuesSvc().update(reopened!.id, { description: compileDescription(card, reopened!.id, hash), }); - const [nextCard] = await db + const [nextCard] = await mutate(card.companyId, tx => tx .update(statusCards) .set({ generatingIssueId: generationIssue!.id, state: "compiling", failureReason: null, updatedAt: createdAt }) .where(eq(statusCards.id, card.id)) - .returning(); + .returning()); return { card: nextCard!, generatingIssue: generationIssue!, @@ -464,7 +474,7 @@ export function statusCardService( if (card.archivedAt) throw unprocessable("Archived status cards cannot accept generation writes"); await assertSummarizerWriter(card, input.generationIssueId, actor); const now = new Date(); - return db.transaction(async (tx) => { + return mutate(card.companyId, async (tx) => { const current = await tx.select().from(statusCards).where(eq(statusCards.id, card.id)).then((rows) => rows[0] ?? null); if (!current || current.archivedAt || current.generatingIssueId !== input.generationIssueId) { throw conflict("Status-card compilation was superseded by a newer task"); @@ -581,14 +591,21 @@ export function statusCardService( } async function executeQueries(card: StatusCardRow, options: StatusCardReadOptions = {}) { - const issueMap = new Map(); + const issueMap = new Map(); for (const storedQuery of card.queries) { const query = companySearchQuerySchema.parse(storedQuery); const response = await searchSvc.search(card.companyId, query, { issueReadCondition: options.issueReadCondition, }); for (const result of response.results) { - if (result.type === "issue" && result.issue) issueMap.set(result.issue.id, result.issue); + if (result.type !== "issue" || !result.issue) continue; + const duration = companySearchUpdatedWithinMs(query.updatedWithin); + const expiry = duration === null ? null : new Date(new Date(result.issue.updatedAt).getTime() + duration + 1); + const previous = issueMap.get(result.issue.id); + issueMap.set(result.issue.id, { ...result.issue, scopeExpiresAt: previous + ? previous.scopeExpiresAt == null || expiry === null ? null + : new Date(Math.max(previous.scopeExpiresAt.getTime(), expiry.getTime())) + : expiry }); } } // Issues mentioned in the latest summary join the watched set alongside the @@ -598,7 +615,11 @@ export function statusCardService( (card.mentionedIssueIds ?? []).filter((issueId) => !issueMap.has(issueId)), options, ); - for (const issue of mentioned) issueMap.set(issue.id, issue); + for (const issue of mentioned) issueMap.set(issue.id, { ...issue, scopeExpiresAt: null }); + for (const id of card.mentionedIssueIds ?? []) { + const issue = issueMap.get(id); + if (issue) issue.scopeExpiresAt = null; + } const snapshot = [...issueMap.values()]; if (snapshot.length === 0) return snapshot; const latestHumanComments = await db @@ -622,6 +643,13 @@ export function statusCardService( return snapshot.map((issue) => ({ ...issue, latestHumanCommentAt: commentByIssueId.get(issue.id) ?? null })); } + function nextEvaluationAt(card: StatusCardRow, snapshot: WatchedIssue[], now: Date) { + const interval = nextStatusCardEvaluationAt(card.refreshPolicy, now); + if (card.refreshPolicy.mode !== "reactive" || !card.refreshPolicy.triggers.membershipChanges) return interval; + const expiries = snapshot.flatMap(issue => issue.scopeExpiresAt && issue.scopeExpiresAt > now ? [issue.scopeExpiresAt.getTime()] : []); + return expiries.length ? new Date(Math.min(...expiries)) : null; + } + async function requestRefresh(cardId: string, input: { full?: boolean; trigger?: "manual" | "interval" | "reactive" | "restore"; @@ -648,15 +676,15 @@ export function statusCardService( const changes = filterStatusCardChanges(allChanges, card.refreshPolicy); const trigger = input.trigger ?? "manual"; const forceRun = trigger === "manual" || trigger === "restore"; - const nextEvalAt = nextStatusCardEvaluationAt(card.refreshPolicy, now); + const nextEvalAt = nextEvaluationAt(card, snapshot, now); if (!forceRun && changes.length === 0) { - const [next] = await db.update(statusCards).set({ + const [next] = await mutate(card.companyId, tx => tx.update(statusCards).set({ pendingChangeCount: 0, pendingChangeHash: null, lastChangeAt: null, state: "active", nextEvalAt, - }).where(eq(statusCards.id, card.id)).returning(); + }).where(eq(statusCards.id, card.id)).returning()); return { card: next!, generatingIssue: null, alreadyGenerating: false, enqueued: false }; } @@ -678,13 +706,17 @@ export function statusCardService( manual: forceRun, }); if (decision.action !== "run") { - const [next] = await db.update(statusCards).set({ + const [next] = await mutate(card.companyId, tx => tx.update(statusCards).set({ pendingChangeCount: changes.length, pendingChangeHash, lastChangeAt, state: decision.action === "pause_budget" ? "paused_budget" : decision.action === "pause_hours" ? "paused_hours" : "active", - nextEvalAt: decision.action === "wait" && "dueAt" in decision ? decision.dueAt : nextEvalAt, - }).where(eq(statusCards.id, card.id)).returning(); + nextEvalAt: nextStatusCardPolicyEligibility({ + policy: card.refreshPolicy, now, lastChangeAt, + completedUpdateTimes: recent.filter(row => row.kind !== "compile" && row.finishedAt).map(row => row.startedAt), + tokensToday: Number(daily[0]?.tokens ?? 0), + }), + }).where(eq(statusCards.id, card.id)).returning()); return { card: next!, generatingIssue: null, alreadyGenerating: false, enqueued: false }; } @@ -707,7 +739,7 @@ export function statusCardService( : null; const fingerprintHash = statusCardFingerprintHash(fingerprint); let deduplicated = false; - const created = await issuesSvc.create(card.companyId, { + const created = await issuesSvc().create(card.companyId, { title: `${kind === "full" ? "Rebuild" : "Update"} status card: ${card.title ?? card.interestPrompt.slice(0, 80)}`, description: updateDescription({ card, generationIssueId: null, fingerprint, changes, kind, trigger, previousSummary, snapshot }), status: "todo", @@ -720,15 +752,15 @@ export function statusCardService( onDeduplicated: (reason) => { deduplicated = reason === "idempotency_key"; }, }); const reopened = deduplicated && TERMINAL_ISSUE_STATUSES.has(created.status) - ? await issuesSvc.update(created.id, { status: "todo", assigneeAgentId: summarizerAgentId }) + ? await issuesSvc().update(created.id, { status: "todo", assigneeAgentId: summarizerAgentId }) : created; - const generationIssue = await issuesSvc.update(reopened!.id, { + const generationIssue = await issuesSvc().update(reopened!.id, { description: updateDescription({ card, generationIssueId: reopened!.id, fingerprint, changes, kind, trigger, previousSummary, snapshot }), }); const priorGenerationPredicate = card.generatingIssueId ? eq(statusCards.generatingIssueId, card.generatingIssueId) : isNull(statusCards.generatingIssueId); - const [next] = await db.update(statusCards).set({ + const [next] = await mutate(card.companyId, tx => tx.update(statusCards).set({ generatingIssueId: generationIssue!.id, pendingChangeCount: changes.length, pendingChangeHash, @@ -736,17 +768,17 @@ export function statusCardService( state: "active", nextEvalAt, failureReason: null, - }).where(and(eq(statusCards.id, card.id), isNull(statusCards.archivedAt), or(isNull(statusCards.generatingIssueId), priorGenerationPredicate))).returning(); + }).where(and(eq(statusCards.id, card.id), isNull(statusCards.archivedAt), or(isNull(statusCards.generatingIssueId), priorGenerationPredicate))).returning()); if (!next) { const winner = await getById(card.id); if (!winner?.generatingIssueId) { if (!TERMINAL_ISSUE_STATUSES.has(generationIssue!.status)) { - await issuesSvc.update(generationIssue!.id, { status: "cancelled" }); + await issuesSvc().update(generationIssue!.id, { status: "cancelled" }); } throw conflict("Status-card refresh claim was lost"); } if (generationIssue!.id !== winner.generatingIssueId && !TERMINAL_ISSUE_STATUSES.has(generationIssue!.status)) { - await issuesSvc.update(generationIssue!.id, { status: "cancelled" }); + await issuesSvc().update(generationIssue!.id, { status: "cancelled" }); } const winnerIssue = await db.select().from(issues).where(eq(issues.id, winner.generatingIssueId)).then((rows) => rows[0] ?? null); return { card: winner, generatingIssue: winnerIssue, alreadyGenerating: true, enqueued: false, kind, changes }; @@ -765,26 +797,44 @@ export function statusCardService( return { card: next, generatingIssue: generationIssue!, alreadyGenerating: deduplicated, enqueued: true, kind, changes }; } - async function tickDueStatusCards(now = new Date()) { - const due = await db.select().from(statusCards).where(and(isNull(statusCards.archivedAt), isNull(statusCards.generatingIssueId), isNotNull(statusCards.nextEvalAt), lte(statusCards.nextEvalAt, now))); + async function tickDueStatusCards(now = new Date(), reactiveCompanies?: ReadonlySet | "all" | "changed") { + const reactiveScope = reactiveCompanies === "all" ? isNull(statusCards.nextEvalAt) : reactiveCompanies === "changed" ? sql`true` : reactiveCompanies?.size + ? inArray(statusCards.companyId, [...reactiveCompanies]) : sql`false`; + const due = await db.select({ card: statusCards }).from(statusCards) + .innerJoin(companies, eq(companies.id, statusCards.companyId)) + .where(and(eq(companies.status, "active"), isNull(statusCards.archivedAt), isNull(statusCards.generatingIssueId), + or(and(isNotNull(statusCards.nextEvalAt), lte(statusCards.nextEvalAt, now)), + and(reactiveScope, ne(statusCards.state, "error"), ne(statusCards.state, "compiling"), sql`${statusCards.refreshPolicy}->>'mode' = 'reactive'`, sql`jsonb_array_length(${statusCards.queries}) > 0`)))) + .then(rows => rows.map(row => row.card)); const enqueued: Array<{ cardId: string; generatingIssue: typeof issues.$inferSelect }> = []; let evaluated = 0; for (const candidate of due) { - const claimUntil = new Date(now.getTime() + 5 * 60 * 1000); - const [claimed] = await db.update(statusCards).set({ nextEvalAt: claimUntil }) - .where(and(eq(statusCards.id, candidate.id), isNull(statusCards.generatingIssueId), lte(statusCards.nextEvalAt, now))) - .returning(); - if (!claimed) continue; - evaluated += 1; - try { - const result = await requestRefresh(claimed.id, { trigger: claimed.refreshPolicy.mode === "reactive" ? "reactive" : "interval", now }); - if (result.enqueued && result.generatingIssue) enqueued.push({ cardId: claimed.id, generatingIssue: result.generatingIssue }); - } catch (err) { - logger.warn( - { err, cardId: claimed.id, companyId: claimed.companyId }, - "status card scheduled refresh failed", + // nextEvalAt is both a retry lease and an evaluation deadline. A source + // event may preempt a genuine future deadline, so serialize scheduler + // evaluation across connections without holding a row lock while the + // existing issue/card services commit through their own transactions. + await db.transaction(async lockTx => { + const [lock] = await lockTx.execute<{ acquired: boolean }>( + sql`select pg_try_advisory_xact_lock(hashtextextended(${candidate.id}, 0)) as acquired`, ); - } + if (!lock?.acquired) return; + const claimUntil = new Date(now.getTime() + 5 * 60 * 1000); + const [claimed] = await mutate(candidate.companyId, tx => tx.update(statusCards).set({ nextEvalAt: claimUntil }) + .where(and(eq(statusCards.id, candidate.id), isNull(statusCards.archivedAt), isNull(statusCards.generatingIssueId), + candidate.nextEvalAt ? eq(statusCards.nextEvalAt, candidate.nextEvalAt) : isNull(statusCards.nextEvalAt))) + .returning()); + if (!claimed) return; + evaluated += 1; + try { + const result = await requestRefresh(claimed.id, { trigger: claimed.refreshPolicy.mode === "reactive" ? "reactive" : "interval", now }); + if (result.enqueued && result.generatingIssue) enqueued.push({ cardId: claimed.id, generatingIssue: result.generatingIssue }); + } catch (err) { + logger.warn( + { err, cardId: claimed.id, companyId: claimed.companyId }, + "status card scheduled refresh failed", + ); + } + }); } return { evaluated, enqueued }; } @@ -796,7 +846,7 @@ export function statusCardService( await assertSummarizerWriter(card, input.generationIssueId, actor); if (card.queries.length === 0) throw conflict("Compile the status-card query before writing its summary"); const now = new Date(); - return db.transaction(async (tx) => { + return mutate(card.companyId, async (tx) => { const current = await tx.select().from(statusCards).where(eq(statusCards.id, card.id)).then((rows) => rows[0] ?? null); if (!current || current.archivedAt || current.generatingIssueId !== input.generationIssueId) { throw conflict("Status-card generation was superseded by a newer task"); @@ -816,9 +866,8 @@ export function statusCardService( const mentionedIssueIds = await resolveMentionedIssueIds(current.companyId, input.markdown); // Current watched membership: compiled-query matches plus the issues this // summary mentions. - const watchedNow = buildStatusCardFingerprint( - await executeQueries({ ...current, mentionedIssueIds }), - ); + const watchedSnapshot = await executeQueries({ ...current, mentionedIssueIds }); + const watchedNow = buildStatusCardFingerprint(watchedSnapshot); let snapshot: StatusCardFingerprint; if (payloadFingerprint) { // Keep the generation-time fingerprint as the change baseline: issues @@ -889,7 +938,9 @@ export function statusCardService( pendingChangeCount: 0, pendingChangeHash: null, lastChangeAt: null, - nextEvalAt: nextStatusCardEvaluationAt(current.refreshPolicy, now), + nextEvalAt: current.refreshPolicy.mode === "reactive" && + filterStatusCardChanges(diffStatusCardFingerprint(snapshot, watchedNow), current.refreshPolicy).length > 0 + ? now : nextEvaluationAt(current, watchedSnapshot, now), updatedAt: now, }).where(and(eq(statusCards.id, current.id), eq(statusCards.generatingIssueId, input.generationIssueId))).returning(); if (!next) throw conflict("Status-card generation was superseded by a newer task"); From 46f2c0bb9a999b414497ba901b64a7c750c7fbc0 Mon Sep 17 00:00:00 2001 From: Dotta Date: Fri, 30 Oct 2026 17:25:24 -0500 Subject: [PATCH 30/70] fix(server): bound and scope routine projection recovery Co-Authored-By: Paperclip --- doc/plans/2026-10-30-routine-status-sleep.md | 13 +++-- .../__tests__/execution-control-work.test.ts | 4 +- ...tbeat-workspace-branch-containment.test.ts | 20 +++---- .../routine-execution-status-work.test.ts | 52 +++++++++++++++++++ .../routine-execution-status-worker.test.ts | 37 ++++++++++--- .../routine-status-reconciliation.test.ts | 17 +++++- .../server-startup-feedback-export.test.ts | 27 ++++++++++ .../execution-control-notifications.ts | 9 ++-- .../execution-control-reconciliation.ts | 2 +- .../services/execution-recovery-resolution.ts | 6 ++- .../services/explicit-native-continuation.ts | 2 +- server/src/services/heartbeat.ts | 2 +- server/src/services/heartbeat/queue.ts | 2 + .../src/services/heartbeat/run-lifecycle.ts | 4 +- .../services/heartbeat/runtime-execution.ts | 4 +- server/src/services/issue-recovery-actions.ts | 10 ++-- .../issue-review-state-notifications.ts | 2 +- server/src/services/issues.ts | 2 +- .../src/services/legacy-execution-recovery.ts | 2 +- .../native-failed-workspace-recovery.ts | 2 +- .../native-runtime/native-restart-recovery.ts | 2 +- .../native-runtime/native-run-finalizer.ts | 4 +- .../native-runtime/native-run-projection.ts | 2 +- .../native-runtime/native-safe-replacement.ts | 4 +- .../native-runtime/native-session-executor.ts | 6 +-- .../native-runtime/native-superseded-retry.ts | 2 +- .../native-workspace-reconstruction.test.ts | 8 ++- .../native-workspace-reconstruction.ts | 2 + .../routine-execution-status-notifications.ts | 7 ++- .../services/routine-execution-status-work.ts | 17 ++++-- server/src/services/routine-run-status.ts | 29 +++++------ server/src/services/routines.ts | 5 +- server/src/services/system-recovery.ts | 2 +- server/src/services/task-retry.ts | 2 +- 34 files changed, 227 insertions(+), 84 deletions(-) diff --git a/doc/plans/2026-10-30-routine-status-sleep.md b/doc/plans/2026-10-30-routine-status-sleep.md index ea6f9364999..271d0b544a7 100644 --- a/doc/plans/2026-10-30-routine-status-sleep.md +++ b/doc/plans/2026-10-30-routine-status-sleep.md @@ -7,14 +7,16 @@ scanned every five minutes, even with no routines or only completed history. ## Ownership and trigger `routine-run-status.ts` owns the projection, custody checks and repair transaction. -The worker loads actual mismatches at startup. It accepts issue hints from issue +The worker loads actual mismatches in keyset pages of 50 at startup. It accepts issue hints from issue creation, status/origin changes, checkout/release, routine issue association, execution control writes and review state writes. It uses the existing transaction-aware work notification protocol. An outer transaction owns the hint; a savepoint release does not wake the repair worker before commit. -A standard execution must name the same routine, invocation, company and linked -task. Plugin-managed executions may use their declared custom origin. Repair +An execution must retain reciprocal task/invocation links in the same company +and belong to a routine in that company. This is the existing routine status +owner contract. Origin labels can change when a plugin declaration changes or +is removed; retained reciprocal links preserve historical execution custody. Repair locks the routine and task and rechecks the linked invocation before writing. It preserves skipped/coalesced history and unrelated dispatch failures. A task that resumes clears an execution-task transient failure through the same owner @@ -33,8 +35,9 @@ history leave no recurring worker SQL. The existing Cloud and heartbeat schedule enablement gates remain in place. Ordinary routine scheduling is unchanged. Supported app writers register `notifyRoutineExecutionStatus(tx, issueId)` before -their mutation. Common execution writers without an issue hint request debt -reconstruction. Backups and independent SQL import/repair tools must restore +their mutation. Common execution writers without an issue hint request debt reconstruction only +for their recorded company. Projection debt settles even when that company is +paused; company activity does not create or cancel this obligation. Backups and independent SQL import/repair tools must restore before startup or call that same transaction contract; process-local hints do not observe independent SQL or another process. No permanent listener or periodic configured-routine scan is added. diff --git a/server/src/__tests__/execution-control-work.test.ts b/server/src/__tests__/execution-control-work.test.ts index 0046d8574aa..14cd9456eb5 100644 --- a/server/src/__tests__/execution-control-work.test.ts +++ b/server/src/__tests__/execution-control-work.test.ts @@ -383,11 +383,11 @@ describe.skipIf(!support.supported)("execution-control durable work", () => { const due = new Date(Date.now() + 120_000), earlier = new Date(Date.now() + 60_000); for (const at of [due, earlier]) { await writeExecutionControl(db, DELIVERY_QUEUES.executionFinalization, tx => tx.update(heartbeatRuns) - .set({ executionControlDeadlineAt: at }).where(eq(heartbeatRuns.id, run.id))); + .set({ executionControlDeadlineAt: at }).where(eq(heartbeatRuns.id, run.id)), { companyId }); await settle(); expect(c.nextWakeAt()).toBe(at.getTime()); } await writeExecutionControl(db, DELIVERY_QUEUES.executionFinalization, tx => tx.update(heartbeatRuns) - .set({ executionControlDeadlineAt: null }).where(eq(heartbeatRuns.id, run.id))); + .set({ executionControlDeadlineAt: null }).where(eq(heartbeatRuns.id, run.id)), { companyId }); await quiet(c); }); it("delivers a reconciled continuation when marking and resolving commit together", async () => { diff --git a/server/src/__tests__/heartbeat-workspace-branch-containment.test.ts b/server/src/__tests__/heartbeat-workspace-branch-containment.test.ts index 52d434a3078..484cf55e8ff 100644 --- a/server/src/__tests__/heartbeat-workspace-branch-containment.test.ts +++ b/server/src/__tests__/heartbeat-workspace-branch-containment.test.ts @@ -1,3 +1,5 @@ +import { subscribeDatabaseWork } from "@paperclipai/db"; +import { DELIVERY_QUEUES } from "../services/delivery-work-notifications.js"; import { EMBEDDED_POSTGRES_TEST_TIMEOUT_MS } from "@paperclipai/db"; import { execFile } from "node:child_process"; import { createHash, randomUUID } from "node:crypto"; @@ -888,7 +890,7 @@ describeEmbeddedPostgres("heartbeat workspace branch containment", () => { const tempRoots: string[] = []; beforeAll(async () => { - if (externalTestDatabaseUrl) db = createDb(externalTestDatabaseUrl); + if (externalTestDatabaseUrl) db = createDb(externalTestDatabaseUrl, { maxConnections: 6 }); else { tempDb = await startEmbeddedPostgresTestDatabase("paperclip-branch-containment-"); db = createDb(tempDb.connectionString); @@ -1173,15 +1175,15 @@ describeEmbeddedPostgres("heartbeat workspace branch containment", () => { }, }); + const routineEvents: string[] = []; + const unsubscribeRoutine = subscribeDatabaseWork(db, DELIVERY_QUEUES.routineExecutionStatus, event => routineEvents.push(event)); const heartbeat = heartbeatService(db); - const run = await heartbeat.wakeup(agentId, { - source: "assignment", - triggerDetail: "system", - reason: "issue_assigned", - payload: { issueId }, - contextSnapshot: { issueId, wakeReason: "issue_assigned" }, - }); - + let run; + try { + run = await heartbeat.wakeup(agentId, { source: "assignment", triggerDetail: "system", reason: "issue_assigned", + payload: { issueId }, contextSnapshot: { issueId, wakeReason: "issue_assigned" } }); + } finally { unsubscribeRoutine(); } + expect(routineEvents).toContain("intent"); expect(routineEvents).toContain("settled"); expect(run).toBeNull(); expect(adapterExecute).not.toHaveBeenCalled(); diff --git a/server/src/__tests__/routine-execution-status-work.test.ts b/server/src/__tests__/routine-execution-status-work.test.ts index 7415b63bfd5..53d7e5c0225 100644 --- a/server/src/__tests__/routine-execution-status-work.test.ts +++ b/server/src/__tests__/routine-execution-status-work.test.ts @@ -1,3 +1,5 @@ +import { writeExecutionControl } from "../services/execution-control-notifications.js"; +import { DELIVERY_QUEUES } from "../services/delivery-work-notifications.js"; import { randomUUID } from "node:crypto"; import { eq, sql } from "drizzle-orm"; import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; @@ -122,6 +124,56 @@ const support = external ? { supported: true } : await getEmbeddedPostgresTestSu await settled(); await quiet(); expect(await saved(run.id)).toEqual(before); expect((await saved(otherRun.id)).status).toBe("issue_created"); expect(errors).toEqual([]); }); + it.each(["changed", "removed"])("keeps historical custom-origin custody after its plugin declaration is %s", async mode => { + const f = await fixture(), pluginKey = `routine-history-${randomUUID()}`, originId = `operation:${randomUUID()}`; + const [plugin] = await db.insert(plugins).values({ pluginKey, packageName: pluginKey, version: "0.1.0", + manifestJson: { id: pluginKey, apiVersion: 1, version: "0.1.0", displayName: "Routine test", description: "Routine test", + author: "Paperclip", categories: ["automation"], capabilities: ["routines.managed"], entrypoints: { worker: "./worker.js" } } }).returning(); + const [resource] = await db.insert(pluginManagedResources).values({ companyId: f.companyId, pluginId: plugin.id, pluginKey, + resourceKind: "routine", resourceKey: "history", resourceId: f.routine.id, defaultsJson: { issueTemplate: { originId } } }).returning(); + const run = await dispatch(f); + expect((await db.select().from(issues).where(eq(issues.id, run.linkedIssueId!)))[0].originId).toBe(originId); + if (mode === "changed") await db.update(pluginManagedResources).set({ defaultsJson: { issueTemplate: { originId: "new-origin" } } }).where(eq(pluginManagedResources.id, resource.id)); + else await db.delete(pluginManagedResources).where(eq(pluginManagedResources.id, resource.id)); + await db.update(issues).set({ status: "done" }).where(eq(issues.id, run.linkedIssueId!)); + await start(); await projected(run.id, "completed"); await quiet(); expect(errors).toEqual([]); + }); + it("reconstructs more than one page and does not lose a committed issue hint behind the startup cursor", async () => { + const f = await fixture(), ids = Array.from({ length: 53 }, () => randomUUID()).sort(); + const issueIds = ids.map(() => randomUUID()); + await db.insert(issues).values(ids.map((id, index) => ({ id: issueIds[index], companyId: f.companyId, title: "Legacy execution", + status: index === 0 ? "todo" : "done", originKind: "routine_execution", originRunId: id, originId: f.routine.id }))); + await db.insert(routineRuns).values(ids.map((id, index) => ({ id, companyId: f.companyId, routineId: f.routine.id, + source: "schedule", linkedIssueId: issueIds[index], status: "issue_created" }))); + const service = routineExecutionStatusService(db), load = service.load; + let entered!: () => void, release!: () => void; + const enteredGate = new Promise(resolve => { entered = resolve; }); + const releaseGate = new Promise(resolve => { release = resolve; }); + const loaded = vi.spyOn(service, "load").mockImplementationOnce(async (scope, afterId) => { + const page = await load(scope, afterId); expect(page).toHaveLength(50); entered(); await releaseGate; return page; + }); + const starting = start(service); await enteredGate; + await issueService(db).update(issueIds[0]!, { status: "done" }); release(); await starting; + await vi.advanceTimersByTimeAsync(3000); + await vi.waitFor(async () => expect((await db.select().from(routineRuns)).every(row => row.status === "completed")).toBe(true)); + expect(loaded.mock.calls.some(([scope, afterId]) => scope === undefined && afterId !== undefined)).toBe(true); + expect(loaded.mock.calls.some(([scope]) => scope?.issueId === issueIds[0])).toBe(true); + await quiet(); expect(errors).toEqual([]); + }); + it("scopes shared execution writes to their company and settles paused-company projection debt", async () => { + const f = await fixture(), other = await fixture(), run = await dispatch(f), otherRun = await dispatch(other); + const service = routineExecutionStatusService(db), load = vi.spyOn(service, "load"); + await start(service); await quiet(); load.mockClear(); + await db.update(companies).set({ status: "paused" }).where(eq(companies.id, f.companyId)); + await db.update(issues).set({ status: "done" }).where(eq(issues.id, otherRun.linkedIssueId!)); + await writeExecutionControl(db, DELIVERY_QUEUES.executionStatus, + tx => tx.update(issues).set({ status: "done" }).where(eq(issues.id, run.linkedIssueId!)), { companyId: f.companyId }); + await projected(run.id, "completed"); + expect(load.mock.calls.every(([scope]) => scope?.companyId === f.companyId)).toBe(true); + expect((await saved(otherRun.id)).status).toBe("issue_created"); await quiet(); + await db.transaction(async tx => { await notifyRoutineExecutionStatus(tx, otherRun.linkedIssueId!); }); + await projected(otherRun.id, "completed"); await quiet(); expect(errors).toEqual([]); + }); it("waits for the real outer transaction and forgets a rolled-back nested status write", async () => { const f = await fixture(), run = await dispatch(f); await start(); await quiet(); let release!: () => void, entered!: () => void; diff --git a/server/src/__tests__/routine-execution-status-worker.test.ts b/server/src/__tests__/routine-execution-status-worker.test.ts index b88babbbd98..b62245ce3e2 100644 --- a/server/src/__tests__/routine-execution-status-worker.test.ts +++ b/server/src/__tests__/routine-execution-status-worker.test.ts @@ -1,3 +1,4 @@ +import type { RoutineExecutionStatusScope } from "../services/routine-execution-status-notifications.js"; import { installDatabaseWorkSignals } from "../../../packages/db/src/work-signals.js"; import { afterEach, beforeEach, expect, it, vi } from "vitest"; import { createDeliveryWorkCoordinator } from "../services/delivery-work-coordinator.js"; @@ -18,19 +19,19 @@ afterEach(async () => { await coordinator.stop(); vi.useRealTimers(); }); it("loads actual debt once and leaves no recurring owner work", async () => { const service = { load: vi.fn(async () => []), settle: vi.fn(async () => true) }; await registerRoutineExecutionStatusWork(coordinator, service, () => canRun).ready; - expect(service.load).toHaveBeenCalledExactlyOnceWith(undefined); + expect(service.load).toHaveBeenCalledExactlyOnceWith(undefined, undefined); expect(coordinator.nextWakeAt()).toBeNull(); await vi.advanceTimersByTimeAsync(20 * 60_000); expect(service.load).toHaveBeenCalledOnce(); expect(service.settle).not.toHaveBeenCalled(); expect(errors).toEqual([]); }); it("reconstructs only an invalidated issue and coalesces duplicate requests", async () => { - const service = { load: vi.fn(async (_issueId?: string) => [] as typeof entry[]), settle: vi.fn(async () => true) }; + const service = { load: vi.fn(async (_scope?: RoutineExecutionStatusScope, _afterId?: string) => [] as typeof entry[]), settle: vi.fn(async () => true) }; const worker = registerRoutineExecutionStatusWork(coordinator, service, () => canRun); await worker.ready; service.load.mockResolvedValueOnce([entry]); - worker.invalidate("issue"); worker.invalidate("issue"); + worker.invalidate(JSON.stringify({ issueId: "issue" })); worker.invalidate(JSON.stringify({ issueId: "issue" })); await vi.advanceTimersByTimeAsync(1000); - expect(service.load).toHaveBeenLastCalledWith("issue"); expect(service.load).toHaveBeenCalledTimes(2); + expect(service.load).toHaveBeenLastCalledWith({ issueId: "issue" }, undefined); expect(service.load).toHaveBeenCalledTimes(2); expect(service.settle).toHaveBeenCalledExactlyOnceWith(expect.objectContaining(entry)); expect(coordinator.nextWakeAt()).toBeNull(); expect(errors).toEqual([]); }); @@ -50,7 +51,7 @@ it("retains startup and first work during drain without passive owner calls", as canRun = false; const service = { load: vi.fn(async () => [entry]), settle: vi.fn(async () => true) }; const worker = registerRoutineExecutionStatusWork(coordinator, service, () => canRun); await worker.ready; - worker.invalidate("issue"); + worker.invalidate(JSON.stringify({ issueId: "issue" })); await vi.advanceTimersByTimeAsync(20 * 60_000); expect(service.load).not.toHaveBeenCalled(); expect(service.settle).not.toHaveBeenCalled(); canRun = true; await vi.advanceTimersByTimeAsync(30_000); @@ -69,11 +70,11 @@ it("keeps a failed owner across drain and retries after resume", async () => { it("retains a notification that arrives while the startup reconstruction is in flight", async () => { let release!: () => void; const gate = new Promise(resolve => { release = resolve; }); - const service = { load: vi.fn(async (_issueId?: string) => [] as typeof entry[]), settle: vi.fn(async () => true) }; + const service = { load: vi.fn(async (_scope?: RoutineExecutionStatusScope, _afterId?: string) => [] as typeof entry[]), settle: vi.fn(async () => true) }; service.load.mockImplementationOnce(async () => { await gate; return []; }); const worker = registerRoutineExecutionStatusWork(coordinator, service, () => canRun); await vi.waitFor(() => expect(service.load).toHaveBeenCalledOnce()); - worker.invalidate("issue"); service.load.mockResolvedValueOnce([entry]); release(); await worker.ready; + worker.invalidate(JSON.stringify({ issueId: "issue" })); service.load.mockResolvedValueOnce([entry]); release(); await worker.ready; await vi.advanceTimersByTimeAsync(1000); expect(service.settle).toHaveBeenCalledOnce(); expect(coordinator.nextWakeAt()).toBeNull(); expect(errors).toEqual([]); }); @@ -81,6 +82,26 @@ it("retains a notification that arrives while the startup reconstruction is in f it("cancels retries and subscriptions at shutdown", async () => { const service = { load: vi.fn(async () => [entry]), settle: vi.fn(async () => true).mockRejectedValue(new Error("offline")) }; const worker = registerRoutineExecutionStatusWork(coordinator, service, () => canRun); await worker.ready; - await worker.stop(); worker.invalidate("issue"); await vi.advanceTimersByTimeAsync(20 * 60_000); + await worker.stop(); worker.invalidate(JSON.stringify({ issueId: "issue" })); await vi.advanceTimersByTimeAsync(20 * 60_000); expect(service.settle).toHaveBeenCalledOnce(); expect(coordinator.nextWakeAt()).toBeNull(); }); + + +it("reconstructs a backlog in bounded keyset pages while retaining a scoped hint behind the cursor", async () => { + const firstPage = Array.from({ length: 50 }, (_, i) => ({ ...entry, id: `run-${String(i).padStart(3, "0")}` })); + const late = { ...entry, id: "run-late", issueId: "late-issue" }; + const service = { + load: vi.fn(async (scope?: RoutineExecutionStatusScope, afterId?: string) => scope?.issueId ? [late] + : afterId ? [{ ...entry, id: "run-050" }] : firstPage), + settle: vi.fn(async () => true), + }; + const worker = registerRoutineExecutionStatusWork(coordinator, service, () => canRun); await worker.ready; + expect(service.load).toHaveBeenCalledExactlyOnceWith(undefined, undefined); + worker.invalidate(JSON.stringify({ issueId: "late-issue" })); + await vi.advanceTimersByTimeAsync(3000); + expect(service.load).toHaveBeenCalledWith(undefined, "run-049"); + expect(service.load).toHaveBeenCalledWith({ issueId: "late-issue" }, undefined); + expect(service.settle).toHaveBeenCalledTimes(52); + expect(service.settle).toHaveBeenCalledWith(expect.objectContaining(late)); + expect(coordinator.nextWakeAt()).toBeNull(); expect(errors).toEqual([]); +}); diff --git a/server/src/__tests__/routine-status-reconciliation.test.ts b/server/src/__tests__/routine-status-reconciliation.test.ts index 964e43d6035..343c6fb38b6 100644 --- a/server/src/__tests__/routine-status-reconciliation.test.ts +++ b/server/src/__tests__/routine-status-reconciliation.test.ts @@ -38,11 +38,10 @@ if (!support.supported) console.warn(`Skipping routine status database tests: ${ expect((await db.select().from(routineRuns))[0]).toMatchObject({ status: "issue_created", completedAt: null }); } }); - it.each(["active_issue", "wrong_origin", "wrong_routine", "wrong_run", "wrong_company", "wrong_routine_company", "different_status"])("does not settle an unsupported handoff: %s", async mode => { + it.each(["active_issue", "wrong_origin", "wrong_run", "wrong_company", "wrong_routine_company", "different_status"])("does not settle an unsupported handoff: %s", async mode => { const f = await seed(); if (mode === "active_issue") await db.update(issues).set({ status: "in_progress" }).where(eq(issues.id, f.issue.id)); if (mode === "wrong_origin") await db.update(issues).set({ originKind: "manual" }).where(eq(issues.id, f.issue.id)); - if (mode === "wrong_routine") await db.update(issues).set({ originId: randomUUID() }).where(eq(issues.id, f.issue.id)); if (mode === "wrong_run") await db.update(issues).set({ originRunId: randomUUID() }).where(eq(issues.id, f.issue.id)); if (mode === "wrong_company") { const other = await seed("done"); await db.update(routineRuns).set({ companyId: other.companyId }).where(eq(routineRuns.id, f.run.id)); } if (mode === "wrong_routine_company") { const other = await seed("done"); @@ -55,6 +54,20 @@ if (!support.supported) console.warn(`Skipping routine status database tests: ${ expect((await db.select().from(routineRuns)).find(row => row.id === f.run.id)).toEqual(before.find(row => row.id === f.run.id)); if (mode === "wrong_routine_company") await expect(routineService(db).syncRunStatusForIssue(f.issue.id)).resolves.toBeNull(); }); + it("uses reciprocal durable custody when a retained origin no longer names the current routine declaration", async () => { + const f = await seed("done"); + await db.update(issues).set({ originId: "previous-declaration-origin" }).where(eq(issues.id, f.issue.id)); + expect(await routineService(db).reconcileExecutionRunStatuses()).toEqual({ scanned: 1, settled: 1, deferred: 0 }); + expect((await db.select().from(routineRuns).where(eq(routineRuns.id, f.run.id)))[0].status).toBe("completed"); + }); + it.each(["unknown-status", "other-code"])("preserves a failed invocation with an unrelated transient marker: %s", async mode => { + const f = await seed("in_progress"); + await db.update(routineRuns).set({ status: "failed", failureReason: null, + triggerPayload: { transientFailure: { code: mode === "other-code" ? "unrelated_failure" : "execution_issue_status", + status: mode === "unknown-status" ? "unrecognized" : "blocked" } } }).where(eq(routineRuns.id, f.run.id)); + expect(await routineService(db).reconcileExecutionRunStatuses()).toEqual({ scanned: 0, settled: 0, deferred: 0 }); + expect((await db.select().from(routineRuns).where(eq(routineRuns.id, f.run.id)))[0].status).toBe("failed"); + }); it.each(["resume", "retarget", "routine_transfer"])("rechecks %s after candidate selection and before repair", async mode => { const f = await seed(); const transaction = db.transaction.bind(db); vi.spyOn(db, "transaction").mockImplementationOnce(async callback => { diff --git a/server/src/__tests__/server-startup-feedback-export.test.ts b/server/src/__tests__/server-startup-feedback-export.test.ts index bbb020c865d..caf5565762a 100644 --- a/server/src/__tests__/server-startup-feedback-export.test.ts +++ b/server/src/__tests__/server-startup-feedback-export.test.ts @@ -1,3 +1,4 @@ +import { routineExecutionStatusService } from "../services/routine-run-status.js"; import { nativeHarnessBackupExpirationService } from "../services/native-runtime/native-harness-backup-expiration.js"; import { skillVersionRetentionService } from "../services/skill-version-retention.js"; import { approvalQueueSeedingService } from "../services/approval-queue-seeding-work.js"; @@ -392,6 +393,11 @@ vi.mock("../services/approval-retention.js", async importOriginal => ({ approvalRetentionService: vi.fn(() => ({ sweep: vi.fn(async () => ({ archived: 0 })), next: vi.fn(async () => null) })), })); +vi.mock("../services/routine-run-status.js", async importOriginal => ({ + ...await importOriginal(), + routineExecutionStatusService: vi.fn(() => ({ load: vi.fn(async () => []), settle: vi.fn(async () => false) })), +})); + vi.mock("../services/index.js", () => ({ backfillLegacyToolOAuthTokens: vi.fn(async () => ({ scannedConnections: 0, @@ -1366,6 +1372,27 @@ describe("startServer feedback export wiring", () => { } }); + it.each([{ cloud: false, enabled: true }, { cloud: true, enabled: false }, { cloud: true, enabled: true }])( + "registers routine status only for enabled Cloud scheduling ($cloud/$enabled)", async ({ cloud, enabled }) => { + if (cloud) vi.stubEnv("PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN", "startup-test-token"); + else vi.stubEnv("PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN", undefined); + loadConfigMock.mockReturnValue(buildTestConfig({ heartbeatSchedulerEnabled: enabled })); + const intervalCallbacks: Array<() => void> = []; + const intervals = vi.spyOn(globalThis, "setInterval").mockImplementation(((callback: () => void) => { + intervalCallbacks.push(callback); return 1 as unknown as ReturnType; + }) as typeof setInterval); + try { + await startServer(); await settleStartupHousekeeping(); + expect(deliveryWorkRegisterMock.mock.calls.filter(([queue]) => queue === DELIVERY_QUEUES.routineExecutionStatus)).toHaveLength(cloud && enabled ? 1 : 0); + const service = vi.mocked(routineExecutionStatusService).mock.results.at(-1)?.value; + if (cloud && enabled) expect(service?.load).toHaveBeenCalledExactlyOnceWith(undefined, undefined); + for (let index = 0; index < 4; index++) for (const callback of intervalCallbacks) callback(); + await settleStartupHousekeeping(); + if (cloud && enabled) expect(service?.load).toHaveBeenCalledOnce(); + } finally { intervals.mockRestore(); vi.unstubAllEnvs(); } + }, + ); + it("keeps routine ticks and setup cleanup active when heartbeat scheduling is suppressed", async () => { loadConfigMock.mockReturnValue(buildTestConfig({ heartbeatSchedulerEnabled: true, diff --git a/server/src/services/execution-control-notifications.ts b/server/src/services/execution-control-notifications.ts index 33a99774a48..fcbb892578c 100644 --- a/server/src/services/execution-control-notifications.ts +++ b/server/src/services/execution-control-notifications.ts @@ -2,8 +2,10 @@ import { notifyRoutineExecutionStatus } from "./routine-execution-status-notific import type { Db } from "@paperclipai/db"; import { DELIVERY_QUEUES, notifyDeliveryWork, type DeliveryQueue } from "./delivery-work-notifications.js"; -export async function notifyExecutionRunWork(transaction: object, issueId?: string) { - await notifyRoutineExecutionStatus(transaction, issueId); +export type ExecutionWorkScope = { companyId: string; agentId?: string; runId?: string; issueId?: string }; + +export async function notifyExecutionRunWork(transaction: object, scope?: ExecutionWorkScope) { + if (scope) await notifyRoutineExecutionStatus(transaction, scope.issueId, scope.companyId); for (const queue of [DELIVERY_QUEUES.executionStatus, DELIVERY_QUEUES.executionFinalization, DELIVERY_QUEUES.executionReplacement, DELIVERY_QUEUES.executionDisposition, DELIVERY_QUEUES.issueMonitors]) { await notifyDeliveryWork(transaction, queue); @@ -14,9 +16,10 @@ export function writeExecutionControl( db: Db | Parameters[0]>[0], queue: DeliveryQueue | readonly DeliveryQueue[], write: (tx: Db) => PromiseLike, + scope: ExecutionWorkScope | undefined, ): Promise { return db.transaction(async transaction => { - await notifyRoutineExecutionStatus(transaction); + if (scope) await notifyRoutineExecutionStatus(transaction, scope.issueId, scope.companyId); await notifyDeliveryWork(transaction, DELIVERY_QUEUES.issueMonitors); for (const topic of typeof queue === "string" ? [queue] : queue) await notifyDeliveryWork(transaction, topic); return write(transaction as unknown as Db); diff --git a/server/src/services/execution-control-reconciliation.ts b/server/src/services/execution-control-reconciliation.ts index 2f2a435982d..621fd210cfc 100644 --- a/server/src/services/execution-control-reconciliation.ts +++ b/server/src/services/execution-control-reconciliation.ts @@ -196,7 +196,7 @@ export async function reconcileAbandonedExecutionControl( await notifyNativeEnvironmentSettlement(tx, { companyId: run.companyId, runId: run.id }); await notifyCancelledWorkspacePreservation(tx, { companyId: run.companyId, runId: run.id }); await notifyUnsafeWorkspaceExportRunChange(tx, run); - await notifyExecutionRunWork(tx); + await notifyExecutionRunWork(tx, { companyId: run.companyId }); const [updatedRun] = await tx .update(heartbeatRuns) .set({ diff --git a/server/src/services/execution-recovery-resolution.ts b/server/src/services/execution-recovery-resolution.ts index f3f5a3e4310..a6f80f26ba0 100644 --- a/server/src/services/execution-recovery-resolution.ts +++ b/server/src/services/execution-recovery-resolution.ts @@ -1,3 +1,4 @@ +import { notifyRoutineExecutionStatus } from "./routine-execution-status-notifications.js"; import { notifyNativeStatusReassessment } from "./native-status-reassessment-notifications.js"; import { notifyUnsafeWorkspaceExportRecovery } from "./unsafe-workspace-export-notifications.js"; import { notifyChatResponseAvailability } from "./chat-response-availability.js"; @@ -546,7 +547,7 @@ export async function settleUnrecoverableExecutions( companyId: run.companyId, issueId: task.id, runId: run.id, }).limit(1).for("update"); if (!hold) return; - await notifyExecutionRunWork(tx); + await notifyExecutionRunWork(tx, { companyId: run.companyId }); const note = "Exact session recovery stopped. The recorded workspace recovery holds this execution without replaying its actions."; await tx.update(issueRecoveryActions).set({ status: "cancelled", outcome: "cancelled", resolvedAt: now, updatedAt: now, wakePolicy: null, monitorPolicy: null, @@ -596,6 +597,7 @@ export async function settleUnrecoverableExecutions( if (task.status !== "blocked") await notifyChatResponseAvailability(tx, { companyId: task.companyId, issueId: task.id }); await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors); await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers); + await notifyRoutineExecutionStatus(tx, task.id, task.companyId); const [projected] = await tx .update(issues) .set({ @@ -654,7 +656,7 @@ export async function settleUnrecoverableExecutions( replay: "not_authorized", }, }); - await notifyExecutionRunWork(tx); + await notifyExecutionRunWork(tx, { companyId: run.companyId }); await tx .update(heartbeatRuns) .set({ executionStatusDeliveryId: randomUUID() }) diff --git a/server/src/services/explicit-native-continuation.ts b/server/src/services/explicit-native-continuation.ts index 074ffc3bdf5..084a25e457f 100644 --- a/server/src/services/explicit-native-continuation.ts +++ b/server/src/services/explicit-native-continuation.ts @@ -475,7 +475,7 @@ export async function admitExplicitNativeContinuation(input: { }).where(and(eq(agentWakeupRequests.companyId, companyId), eq(agentWakeupRequests.id, input.queuedCommentInterruptId), eq(agentWakeupRequests.agentId, agentId), eq(agentWakeupRequests.status, "deferred_issue_execution"))); if (cancelledStartupIds.size) await db.transaction(async tx => { - await notifyExecutionRunWork(tx); + await notifyExecutionRunWork(tx, { companyId, agentId }); for (const runId of cancelledStartupIds) { await tx.update(nativeRunFinalizations).set({ phase: "terminal_failure", failureCode: "native_startup_cancelled", nextAttemptAt: null, diff --git a/server/src/services/heartbeat.ts b/server/src/services/heartbeat.ts index eb655fdef25..a4cec7b81b0 100644 --- a/server/src/services/heartbeat.ts +++ b/server/src/services/heartbeat.ts @@ -3724,7 +3724,7 @@ export function heartbeatService( await writeExecutionControl(db, DELIVERY_QUEUES.executionFinalization, tx => tx .update(heartbeatRuns) .set({ executionControlDeadlineAt: null }) - .where(eq(heartbeatRuns.id, run.id))); + .where(eq(heartbeatRuns.id, run.id)), { companyId: run.companyId }); } nativeOwnershipHeld = nativeOwnershipHeld || diff --git a/server/src/services/heartbeat/queue.ts b/server/src/services/heartbeat/queue.ts index a41b486b647..d2786778cf7 100644 --- a/server/src/services/heartbeat/queue.ts +++ b/server/src/services/heartbeat/queue.ts @@ -1,3 +1,4 @@ +import { notifyRoutineExecutionStatus } from "../routine-execution-status-notifications.js"; import { DELIVERY_QUEUES, notifyDeliveryWork } from "../delivery-work-notifications.js"; import { notifyExternalAgentSettlement } from "../../modules/external-agents/index.js"; import { notifyNativeStatusReassessment } from "../native-status-reassessment-notifications.js"; @@ -3919,6 +3920,7 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors); await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers); if (issue.status !== "blocked") await notifyChatResponseAvailability(tx, { companyId: issue.companyId, issueId: issue.id }); + await notifyRoutineExecutionStatus(tx, issue.id, issue.companyId); await tx .update(issues) .set({ diff --git a/server/src/services/heartbeat/run-lifecycle.ts b/server/src/services/heartbeat/run-lifecycle.ts index f796da03a5e..ac4e800f279 100644 --- a/server/src/services/heartbeat/run-lifecycle.ts +++ b/server/src/services/heartbeat/run-lifecycle.ts @@ -1044,8 +1044,8 @@ export function createHeartbeatLifecycle(db: Db, dependencies: HeartbeatLifecycl if (previous && isHeartbeatRunTerminalStatus(status)) await notifyRetainedNativeSessionCleanup(tx, { companyId: previous.companyId, runId: previous.id }); if (isHeartbeatRunTerminalStatus(status)) await notifyDeliveryWork(tx, DELIVERY_QUEUES.accounting, companyId); - await notifyExecutionRunWork(tx, previous?.nativeIssueId ?? previous?.issueId - ?? readNonEmptyString(parseObject(previous?.contextSnapshot).issueId) ?? undefined); + await notifyExecutionRunWork(tx, previous ? { companyId: previous.companyId, agentId: previous.agentId, runId: previous.id, + issueId: previous.nativeIssueId ?? previous.issueId ?? readNonEmptyString(parseObject(previous.contextSnapshot).issueId) ?? undefined } : undefined); if (previous) await notifyChatRunMilestones(tx, { companyId: previous.companyId, runId: previous.id, issueId: previous.nativeIssueId ?? previous.issueId ?? readNonEmptyString(parseObject(previous.contextSnapshot).issueId) ?? undefined }); return write(tx); diff --git a/server/src/services/heartbeat/runtime-execution.ts b/server/src/services/heartbeat/runtime-execution.ts index b5b64abc948..bdbbfa07752 100644 --- a/server/src/services/heartbeat/runtime-execution.ts +++ b/server/src/services/heartbeat/runtime-execution.ts @@ -966,7 +966,7 @@ export async function executeHeartbeatRuntime(db: Db, input: HeartbeatRuntimeExe eq(heartbeatRuns.id, run.id), eq(heartbeatRuns.status, "running"), ), - )); + ), { companyId: run.companyId }); const workspaceFinalizeStatus = hasWorkspaceRestoreFailure(adapterResult.resultJson) ? "failed" : "succeeded"; await recordWorkspaceFinalize(workspaceFinalizeStatus); if (adapterResult.nativeFinalization) { @@ -1042,7 +1042,7 @@ export async function executeHeartbeatRuntime(db: Db, input: HeartbeatRuntimeExe eq(heartbeatRuns.id, run.id), eq(heartbeatRuns.status, "running"), ), - )); + ), { companyId: run.companyId }); if ( issueRef && context.resumeSessionGoalHeartbeat === true && diff --git a/server/src/services/issue-recovery-actions.ts b/server/src/services/issue-recovery-actions.ts index 5fb28bcb6d8..48fb001c0b4 100644 --- a/server/src/services/issue-recovery-actions.ts +++ b/server/src/services/issue-recovery-actions.ts @@ -392,7 +392,7 @@ export function issueRecoveryActionService(db: DbOrTransaction) { const [updated] = await writeExecutionControl(db, [DELIVERY_QUEUES.executionDisposition, DELIVERY_QUEUES.executionContinuation], tx => tx.update(issueRecoveryActions).set({ nextAction: input.nextAction, updatedAt: now }) .where(and(eq(issueRecoveryActions.id, existing.id), inArray(issueRecoveryActions.status, [...ACTIVE_RECOVERY_ACTION_STATUSES]), - sql`${issueRecoveryActions.evidence}->'workspaceOwnerStop' is null`)).returning()); + sql`${issueRecoveryActions.evidence}->'workspaceOwnerStop' is null`)).returning(), { companyId: input.companyId }); if (updated) return toReadModel(updated); } return existing; @@ -467,7 +467,7 @@ export function issueRecoveryActionService(db: DbOrTransaction) { ), ) .returning(); - }); + }, { companyId: input.companyId }); if (!exhausted) { return retryUpsertSourceScoped(input, retryCount); } @@ -539,7 +539,7 @@ export function issueRecoveryActionService(db: DbOrTransaction) { ), ) .returning(); - }); + }, { companyId: input.companyId }); if (!updated) { return retryUpsertSourceScoped(input, retryCount); } @@ -554,7 +554,7 @@ export function issueRecoveryActionService(db: DbOrTransaction) { .insert(issueRecoveryActions) .values(buildInsertValues(input, ownerType, now)) .returning(); - }); + }, { companyId: input.companyId }); return toReadModel(created!); } catch (error) { if (!isUniqueRecoveryActionConflict(error)) throw error; @@ -608,7 +608,7 @@ export function issueRecoveryActionService(db: DbOrTransaction) { }) .where(and(...predicates)) .returning(); - }); + }, { companyId: input.companyId }); return updated ? toReadModel(updated) : null; } diff --git a/server/src/services/issue-review-state-notifications.ts b/server/src/services/issue-review-state-notifications.ts index 15bc2a24287..9ef78e0903d 100644 --- a/server/src/services/issue-review-state-notifications.ts +++ b/server/src/services/issue-review-state-notifications.ts @@ -5,7 +5,7 @@ import { notifyNativeReviewResponse } from "./native-review-response-notificatio import { notifyNativeStatusReassessment } from "./native-status-reassessment-notifications.js"; export async function notifyIssueReviewState(tx: object, scope: { companyId: string; issueId?: string; runId?: string; agentId?: string }) { - await notifyRoutineExecutionStatus(tx, scope.issueId); + await notifyRoutineExecutionStatus(tx, scope.issueId, scope.companyId); await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors); await notifyExternalConversationSettlement(tx, scope); await notifyNativeReviewResponse(tx, scope); diff --git a/server/src/services/issues.ts b/server/src/services/issues.ts index eeebf326267..ece99b16752 100644 --- a/server/src/services/issues.ts +++ b/server/src/services/issues.ts @@ -10607,7 +10607,7 @@ export function issueService(db: Db) { } if (values.monitorNextCheckAt) await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors); await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers); - if (values.originKind === "routine_execution") await notifyRoutineExecutionStatus(tx, values.id); + if (values.originKind === "routine_execution") await notifyRoutineExecutionStatus(tx, values.id, companyId); const [issue] = await tx.insert(issues).values(values).returning(); await attachOwnedDraftImages(tx, issue, issue.description, { userId: issueData.createdByUserId, agentId: issueData.createdByAgentId }); if (issue.visibility === "private" && !inheritedPrivateSubtree && issueData.createdByAgentId) { diff --git a/server/src/services/legacy-execution-recovery.ts b/server/src/services/legacy-execution-recovery.ts index 427704de63f..c462cdf1fca 100644 --- a/server/src/services/legacy-execution-recovery.ts +++ b/server/src/services/legacy-execution-recovery.ts @@ -135,7 +135,7 @@ export async function terminalizeLegacyExecution(input: { }; if (issueId) await notifyUnsafeWorkspaceExportRecovery(tx, { companyId: current.companyId, issueId }); await notifyInstructionCollection(tx, { companyId: current.companyId, runId: current.id }); - await notifyExecutionRunWork(tx); + await notifyExecutionRunWork(tx, { companyId: current.companyId }); let [updated] = await tx .update(heartbeatRuns) .set({ diff --git a/server/src/services/native-runtime/native-failed-workspace-recovery.ts b/server/src/services/native-runtime/native-failed-workspace-recovery.ts index 5c3268f6afc..a1c6c914b33 100644 --- a/server/src/services/native-runtime/native-failed-workspace-recovery.ts +++ b/server/src/services/native-runtime/native-failed-workspace-recovery.ts @@ -65,7 +65,7 @@ export async function preserveNativeFailedWorkspace(db: Db, selected: Pick { const external = resolveExternalTestDatabaseUrl("PAPERCLIP_NATIVE_RECONSTRUCTION_TEST_DATABASE_URL"); if (external) { await applyPendingMigrations(external); - db = createDb(external); + db = createDb(external, { maxConnections: 6 }); return; } temporary = await startEmbeddedPostgresTestDatabase("workspace-reconstruction-"); @@ -144,17 +144,21 @@ describe("reconstruction after confirmed native workspace loss", () => { expect(await queue.nextTimerHeartbeatAt(agent)).toBeNull(); const events: string[] = []; const unsubscribe = subscribeDatabaseWork(db, DELIVERY_QUEUES.agentTimers, event => events.push(event)); + const routineEvents: string[] = []; + const unsubscribeRoutine = subscribeDatabaseWork(db, DELIVERY_QUEUES.routineExecutionStatus, event => routineEvents.push(event)); try { await db.transaction(async tx => { await reconstructNativeWorkspace({ ...f, db: tx as unknown as Db }); expect(events).toContain("intent"); expect(events).not.toContain("settled"); + expect(routineEvents).toContain("intent"); expect(routineEvents).not.toContain("settled"); expect((await db.select().from(issues).where(eq(issues.id, f.issueId)))[0].status).toBe("blocked"); }); expect(events).toContain("settled"); + expect(routineEvents).toContain("settled"); expect((await db.select().from(issues).where(eq(issues.id, f.issueId)))[0].status).toBe("todo"); expect(await queue.nextTimerHeartbeatAt(agent)).toBe(agent.createdAt.getTime() + 600_000); - } finally { unsubscribe(); } + } finally { unsubscribe(); unsubscribeRoutine(); } }); it("pins a historical cancelled workspace through public heartbeat startup without replaying its turn", async () => { diff --git a/server/src/services/native-runtime/native-workspace-reconstruction.ts b/server/src/services/native-runtime/native-workspace-reconstruction.ts index 5273b30216a..34e8b25d10a 100644 --- a/server/src/services/native-runtime/native-workspace-reconstruction.ts +++ b/server/src/services/native-runtime/native-workspace-reconstruction.ts @@ -1,3 +1,4 @@ +import { notifyRoutineExecutionStatus } from "../routine-execution-status-notifications.js"; import { DELIVERY_QUEUES, notifyDeliveryWork } from "../delivery-work-notifications.js"; import { notifyUnsafeWorkspaceExportRecovery } from "../unsafe-workspace-export-notifications.js"; import { and, eq, gt, inArray, ne, notInArray, or, sql } from "drizzle-orm"; @@ -187,6 +188,7 @@ export async function reconstructNativeWorkspace(input: { }).where(eq(issueRecoveryActions.id, source.action.id)); await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors); await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers); + await notifyRoutineExecutionStatus(tx, input.issueId, input.companyId); await tx.update(issues).set({ status: "todo", unblockDescriptor: null, blockedTransitionAt: null, blockedOwnerNotifiedAt: null, updatedAt: now }).where(eq(issues.id, issue.id)); const publication = await persistActivity(db, { companyId: input.companyId, actorType: "user", actorId: input.actorId, diff --git a/server/src/services/routine-execution-status-notifications.ts b/server/src/services/routine-execution-status-notifications.ts index cdd2be826c0..fb9df28dda9 100644 --- a/server/src/services/routine-execution-status-notifications.ts +++ b/server/src/services/routine-execution-status-notifications.ts @@ -1,5 +1,8 @@ import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js"; -export function notifyRoutineExecutionStatus(tx: object, issueId?: string) { - return notifyDeliveryWork(tx, DELIVERY_QUEUES.routineExecutionStatus, issueId ?? "*"); +export type RoutineExecutionStatusScope = { issueId?: string; companyId?: string }; + +export function notifyRoutineExecutionStatus(tx: object, issueId?: string, companyId?: string) { + return notifyDeliveryWork(tx, DELIVERY_QUEUES.routineExecutionStatus, + issueId || companyId ? JSON.stringify({ issueId, companyId } satisfies RoutineExecutionStatusScope) : "*"); } diff --git a/server/src/services/routine-execution-status-work.ts b/server/src/services/routine-execution-status-work.ts index 81c8e566a7c..0a4437f89f2 100644 --- a/server/src/services/routine-execution-status-work.ts +++ b/server/src/services/routine-execution-status-work.ts @@ -1,18 +1,27 @@ import type { createDeliveryWorkCoordinator } from "./delivery-work-coordinator.js"; import { DELIVERY_QUEUES } from "./delivery-work-notifications.js"; import { registerScopedDeadlineWork } from "./scoped-deadline-work.js"; -import type { routineExecutionStatusService } from "./routine-run-status.js"; +import type { RoutineExecutionStatusScope } from "./routine-execution-status-notifications.js"; +import { ROUTINE_EXECUTION_STATUS_PAGE_SIZE, type routineExecutionStatusService } from "./routine-run-status.js"; export function registerRoutineExecutionStatusWork( coordinator: ReturnType, service: ReturnType, canRun: () => boolean, ) { + const cursors = new Map(); return registerScopedDeadlineWork(coordinator, { queue: DELIVERY_QUEUES.routineExecutionStatus, - decodeScope: hint => hint === "*" ? undefined : hint, - matches: (entry, issueId?: string) => issueId === undefined || entry.issueId === issueId, - load: async (issueId?: string) => (await service.load(issueId)).map(row => ({ ...row, dueAt: Date.now() })), + decodeScope: hint => hint === "*" ? undefined : JSON.parse(hint) as RoutineExecutionStatusScope, + matches: (entry, scope?: RoutineExecutionStatusScope) => (!scope?.issueId || entry.issueId === scope.issueId) + && (!scope?.companyId || entry.companyId === scope.companyId), + load: async (scope?: RoutineExecutionStatusScope) => { + const key = JSON.stringify(scope) ?? "*"; + const rows = await service.load(scope, cursors.get(key)); + const complete = rows.length < ROUTINE_EXECUTION_STATUS_PAGE_SIZE; + if (complete) cursors.delete(key); else cursors.set(key, rows.at(-1)!.id); + return { rows: rows.map(row => ({ ...row, dueAt: Date.now() })), complete }; + }, check: async entry => { await service.settle(entry); return null; }, }, canRun); } diff --git a/server/src/services/routine-run-status.ts b/server/src/services/routine-run-status.ts index 79c1ff9a231..2a182def8f9 100644 --- a/server/src/services/routine-run-status.ts +++ b/server/src/services/routine-run-status.ts @@ -1,7 +1,7 @@ -import { notifyRoutineExecutionStatus } from "./routine-execution-status-notifications.js"; +import { notifyRoutineExecutionStatus, type RoutineExecutionStatusScope } from "./routine-execution-status-notifications.js"; import { and, eq, inArray, or, sql } from "drizzle-orm"; import type { Db } from "@paperclipai/db"; -import { issues, pluginManagedResources, routineRuns, routines } from "@paperclipai/db"; +import { issues, routineRuns, routines } from "@paperclipai/db"; const EXECUTION_ISSUE_TRANSIENT_FAILURE_CODE = "execution_issue_status"; const EXECUTION_ISSUE_TRANSIENT_FAILURE_STATUSES = ["blocked", "cancelled"] as const; @@ -141,17 +141,11 @@ export async function syncRoutineRunStatusForIssue(executor: Db, issueId: string } -export function routineExecutionOriginCondition() { - return or(sql`${issues.originId} = ${routineRuns.routineId}::text`, sql`exists ( - select 1 from ${pluginManagedResources} resource - where resource.company_id = ${routineRuns.companyId} - and resource.resource_kind = 'routine' and resource.resource_id = ${routineRuns.routineId} - and resource.defaults_json -> 'issueTemplate' ->> 'originId' = ${issues.originId} - )`); -} +export const ROUTINE_EXECUTION_STATUS_PAGE_SIZE = 50; export function routineExecutionStatusDebtCondition() { - const transientFailure = sql`(${routineRuns.triggerPayload} -> 'transientFailure' ->> 'code' = ${EXECUTION_ISSUE_TRANSIENT_FAILURE_CODE} + const transientFailure = sql`((${routineRuns.triggerPayload} -> 'transientFailure' ->> 'code' = ${EXECUTION_ISSUE_TRANSIENT_FAILURE_CODE} + and ${routineRuns.triggerPayload} -> 'transientFailure' ->> 'status' in ('blocked', 'cancelled')) or ${routineRuns.failureReason} in ('Execution issue moved to blocked', 'Execution issue moved to cancelled'))`; return or( and(eq(routineRuns.status, "issue_created"), inArray(issues.status, ["done", "blocked", "cancelled"])), @@ -165,15 +159,18 @@ export function routineExecutionStatusDebtCondition() { } export function routineExecutionStatusService(db: Db) { - async function load(issueId?: string) { + async function load(scope?: RoutineExecutionStatusScope, afterId?: string) { return db.select({ id: routineRuns.id, companyId: routineRuns.companyId, issueId: issues.id, routineId: routineRuns.routineId }) .from(routineRuns) .innerJoin(routines, and(eq(routines.id, routineRuns.routineId), eq(routines.companyId, routineRuns.companyId))) .innerJoin(issues, and(eq(issues.id, routineRuns.linkedIssueId), eq(issues.companyId, routineRuns.companyId), - eq(issues.originKind, "routine_execution"), sql`${issues.originRunId} = ${routineRuns.id}::text`, routineExecutionOriginCondition())) - .where(and(routineExecutionStatusDebtCondition(), ...(issueId ? [eq(issues.id, issueId)] : []))) - .orderBy(routineRuns.id); + eq(issues.originKind, "routine_execution"), sql`${issues.originRunId} = ${routineRuns.id}::text`)) + .where(and(routineExecutionStatusDebtCondition(), + ...(scope?.issueId ? [eq(issues.id, scope.issueId)] : []), + ...(scope?.companyId ? [eq(routineRuns.companyId, scope.companyId)] : []), + ...(afterId ? [sql`${routineRuns.id} > ${afterId}::uuid`] : []))) + .orderBy(routineRuns.id).limit(ROUTINE_EXECUTION_STATUS_PAGE_SIZE); } async function settle(candidate: Awaited>[number]) { return db.transaction(async tx => { @@ -190,7 +187,7 @@ export function routineExecutionStatusService(db: Db) { .innerJoin(issues, eq(issues.id, routineRuns.linkedIssueId)) .where(and(eq(routineRuns.id, candidate.id), eq(routineRuns.companyId, candidate.companyId), eq(routineRuns.routineId, candidate.routineId), eq(routineRuns.linkedIssueId, issue.id), - routineExecutionOriginCondition(), routineExecutionStatusDebtCondition())).for("update", { of: routineRuns }); + routineExecutionStatusDebtCondition())).for("update", { of: routineRuns }); if (!run) return false; return (await syncRoutineRunStatusForIssue(tx as unknown as Db, issue.id)) !== null; }); diff --git a/server/src/services/routines.ts b/server/src/services/routines.ts index c391207950b..aa5771a9b20 100644 --- a/server/src/services/routines.ts +++ b/server/src/services/routines.ts @@ -2002,6 +2002,8 @@ export function routineService( } + let statusReconcileCursor: string | undefined; + async function syncRunStatusForIssue(issueId: string, executor: Db = db) { return syncRoutineRunStatusForIssue(executor, issueId); } @@ -3246,7 +3248,8 @@ export function routineService( reconcileExecutionRunStatuses: async (): Promise<{ scanned: number; settled: number; deferred: number }> => { const service = routineExecutionStatusService(db); - const candidates = await service.load(); + const candidates = await service.load(undefined, statusReconcileCursor); + statusReconcileCursor = candidates.length === 50 ? candidates.at(-1)!.id : undefined; let settled = 0, deferred = 0; for (const candidate of candidates) { try { diff --git a/server/src/services/system-recovery.ts b/server/src/services/system-recovery.ts index 8dba366e1af..a554a765c4c 100644 --- a/server/src/services/system-recovery.ts +++ b/server/src/services/system-recovery.ts @@ -71,7 +71,7 @@ export async function updateSystemRecovery(db: Db, source: WorkspaceRecoverySour sql`${issueRecoveryActions.evidence}#>>'{systemRecovery,revision}' = ${String(previous.revision)}`)) .returning({ id: issueRecoveryActions.id }); return Boolean(updated); - }); + }, { companyId: source.companyId }); } export async function loadSystemRecovery(db: Db, source: WorkspaceRecoverySourceIdentity) { diff --git a/server/src/services/task-retry.ts b/server/src/services/task-retry.ts index e9f0bd4edbf..17075c34eb9 100644 --- a/server/src/services/task-retry.ts +++ b/server/src/services/task-retry.ts @@ -334,7 +334,7 @@ export async function requestTaskRetry(input: { details: { requestId: active?.id ?? id, sourceRunId: latest.id, attachedToExistingWork: Boolean(active || busy) } }); return { requestId: active?.id ?? id, accepted: true, stale: false, observedRunId: latest.id, chatActionId: intent.chatActionId }; - }); + }, { companyId: input.companyId }); } /** Chat owns admission and authorization. The task receipt follows that From a4ec0d51516cecddf36fde146bdd7979d27dfdd2 Mon Sep 17 00:00:00 2001 From: Dotta Date: Fri, 30 Oct 2026 17:26:01 -0500 Subject: [PATCH 31/70] test(server): preserve Cloud skill verification fixture coverage Co-Authored-By: Paperclip --- .../__tests__/managed-skill-content.test.ts | 91 ++++++++++++++++++- 1 file changed, 90 insertions(+), 1 deletion(-) diff --git a/server/src/__tests__/managed-skill-content.test.ts b/server/src/__tests__/managed-skill-content.test.ts index 677a3fbda65..c59f4a1277a 100644 --- a/server/src/__tests__/managed-skill-content.test.ts +++ b/server/src/__tests__/managed-skill-content.test.ts @@ -1,3 +1,5 @@ +import { createServer } from "node:http"; +import { createS3StorageProvider } from "../storage/s3-provider.js"; import { skillInstallationMaintenance } from "../services/skill-installation-maintenance.js"; import { removeRuntimeAssetTree } from "../services/native-runtime/runtime-asset-retention.js"; import { eq } from "drizzle-orm"; @@ -78,6 +80,93 @@ describe.skipIf(!support.supported)("managed skills use committed versions", () }); it("loads persisted Cloud configuration for read-only maintenance without seeding it", async () => { + const companyId = await company(); + const objects = new Map(); + const server = createServer((request, response) => { + void (async () => { + const key = new URL(request.url!, "http://fixture").pathname; + if (request.method === "PUT") { + const chunks: Buffer[] = []; + for await (const chunk of request) chunks.push(Buffer.from(chunk)); + let body = Buffer.concat(chunks); + if (request.headers["content-encoding"]?.includes("aws-chunked")) { + const decoded: Buffer[] = []; + let offset = 0; + while (offset < body.length) { + const end = body.indexOf("\r\n", offset); + const size = Number.parseInt(body.subarray(offset, end).toString().split(";")[0]!, 16); + if (size === 0) break; + decoded.push(body.subarray(end + 2, end + 2 + size)); offset = end + 2 + size + 2; + } + body = Buffer.concat(decoded); + } + objects.set(key, body); response.writeHead(200, { etag: '"fixture"' }).end(); return; + } + const body = objects.get(key); + if (!body) { response.writeHead(404).end(); return; } + response.writeHead(200, { "content-length": body.length, "content-type": "application/octet-stream", etag: '"fixture"' }); + response.end(request.method === "HEAD" ? undefined : body); + })().catch(() => { response.writeHead(500).end(); }); + }); + await new Promise(resolve => { server.listen(0, "127.0.0.1", resolve); }); + const address = server.address(); if (!address || typeof address === "string") throw new Error("Expected fixture address"); + const endpoint = `http://127.0.0.1:${address.port}`; + const credentials = { AWS_ACCESS_KEY_ID: "fixture-access-key", AWS_SECRET_ACCESS_KEY: "fixture-secret-key", AWS_SESSION_TOKEN: "" }; + const previousCredentials = Object.fromEntries(Object.keys(credentials).map(key => [key, process.env[key]])); + Object.assign(process.env, credentials); + vi.mocked(storageModule.getStorageService).mockReturnValue(createStorageService(createS3StorageProvider({ + bucket: "managed-skill-fixture", region: "us-east-1", endpoint, forcePathStyle: true, + }))); + const key = "cloud-runtime-config:v1"; + try { + await companySkillService(db).createLocalSkill(companyId, { name: "Cloud CLI", markdown: "# Cloud CLI" }); + const run = (phase: string) => promisify(execFile)(process.execPath, [ + "--import", fileURLToPath(import.meta.resolve("tsx")), + fileURLToPath(new URL("../../scripts/migrate-skill-packages.ts", import.meta.url)), + `--phase=${phase}`, `--company=${companyId}`, "--limit=50", + ], { env: { ...process.env, ...credentials, DATABASE_URL: database.connectionString, + PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN: "disposable-cloud-cli-fixture", + PAPERCLIP_SECRETS_MASTER_KEY: "a".repeat(64), PAPERCLIP_STORAGE_PROVIDER: "s3", + PAPERCLIP_STORAGE_S3_BUCKET: "managed-skill-fixture", PAPERCLIP_STORAGE_S3_REGION: "us-east-1", + PAPERCLIP_STORAGE_S3_ENDPOINT: endpoint, PAPERCLIP_STORAGE_S3_FORCE_PATH_STYLE: "true", PAPERCLIP_STORAGE_S3_PREFIX: "" }, + timeout: 60_000, maxBuffer: 1024 * 1024 }); + await expect(run("eligibility")).rejects.toThrow("Initialize Cloud runtime configuration"); + expect(await db.select().from(instanceSettings).where(eq(instanceSettings.singletonKey, key))).toHaveLength(0); + await db.insert(instanceSettings).values({ singletonKey: key, general: { version: 1, revision: 0, config: null } }); + const eligibility = JSON.parse((await run("eligibility")).stdout.trim().split("\n").at(-1)!); + expect(eligibility.results).toEqual([expect.objectContaining({ status: "version_backed" })]); + const before = [...objects.entries()].map(([key, body]) => [key, Buffer.from(body)] as const); + const verification = JSON.parse((await run("verification")).stdout.trim().split("\n").at(-1)!); + expect(verification.results).toEqual([expect.objectContaining({ status: "remote_storage" })]); + expect([...objects.entries()]).toEqual(before); + const archiveKey = [...objects.keys()].find(key => key.endsWith("/files.tar.gz"))!; + const archive = objects.get(archiveKey)!; + const corrupted = Buffer.from(archive); corrupted[0] = corrupted[0]! ^ 1; + objects.set(archiveKey, corrupted); + const corrupt = JSON.parse((await run("verification")).stdout.trim().split("\n").at(-1)!); + expect(corrupt.results).toEqual([expect.objectContaining({ status: "corrupt_package" })]); + objects.set(archiveKey, archive); + const manifestKey = [...objects.keys()].find(key => key.endsWith("/manifest.json"))!; + const manifest = objects.get(manifestKey)!; objects.delete(manifestKey); + const unavailable = JSON.parse((await run("verification")).stdout.trim().split("\n").at(-1)!); + expect(unavailable.results).toEqual([expect.objectContaining({ status: "unavailable_or_corrupt_package" })]); + objects.set(manifestKey, manifest); + const recovered = JSON.parse((await run("verification")).stdout.trim().split("\n").at(-1)!); + expect(recovered.results).toEqual([expect.objectContaining({ status: "remote_storage" })]); + expect([...objects.entries()].sort()).toEqual([...before].sort()); + expect(await db.select().from(instanceSettings).where(eq(instanceSettings.singletonKey, key))) + .toEqual([expect.objectContaining({ general: { version: 1, revision: 0, config: null } })]); + } finally { + await db.delete(instanceSettings).where(eq(instanceSettings.singletonKey, key)); + vi.mocked(storageModule.getStorageService).mockReturnValue(createStorageService(createLocalDiskStorageProvider(path.join(root, "objects")))); + for (const [key, value] of Object.entries(previousCredentials)) { + if (value === undefined) delete process.env[key]; else process.env[key] = value; + } + server.closeAllConnections(); await new Promise(resolve => { server.close(() => resolve()); }); + } + }, 180_000); + + it("reports a provider mismatch for local versions under persisted Cloud configuration", async () => { const companyId = await company(); const skill = await companySkillService(db).createLocalSkill(companyId, { name: "Cloud CLI", markdown: "# Cloud CLI" }); const local = await skillInstallationMaintenance(db).verifyVersions({ companyId, limit: 50 }); @@ -90,7 +179,7 @@ describe.skipIf(!support.supported)("managed skills use committed versions", () ], { env: { ...process.env, DATABASE_URL: database.connectionString, PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN: "disposable-cloud-cli-fixture", PAPERCLIP_STORAGE_PROVIDER: "s3", PAPERCLIP_STORAGE_S3_BUCKET: "managed-skill-cli-fixture", - PAPERCLIP_STORAGE_S3_REGION: "us-east-1", PAPERCLIP_SECRETS_MASTER_KEY: "12345678901234567890123456789012" }, + PAPERCLIP_STORAGE_S3_REGION: "us-east-1", PAPERCLIP_SECRETS_MASTER_KEY: "a".repeat(64) }, timeout: 60_000, maxBuffer: 1024 * 1024 }); await expect(run("eligibility")).rejects.toThrow("Initialize Cloud runtime configuration"); expect(await db.select().from(instanceSettings).where(eq(instanceSettings.singletonKey, key))).toHaveLength(0); From cc3bc43ada773cdd66d650853a8209df1fcefc02 Mon Sep 17 00:00:00 2001 From: Dotta Date: Fri, 30 Oct 2026 17:26:36 -0500 Subject: [PATCH 32/70] test(server): retain current master skill fixture Co-Authored-By: Paperclip --- .../__tests__/managed-skill-content.test.ts | 91 +------------------ 1 file changed, 1 insertion(+), 90 deletions(-) diff --git a/server/src/__tests__/managed-skill-content.test.ts b/server/src/__tests__/managed-skill-content.test.ts index c59f4a1277a..677a3fbda65 100644 --- a/server/src/__tests__/managed-skill-content.test.ts +++ b/server/src/__tests__/managed-skill-content.test.ts @@ -1,5 +1,3 @@ -import { createServer } from "node:http"; -import { createS3StorageProvider } from "../storage/s3-provider.js"; import { skillInstallationMaintenance } from "../services/skill-installation-maintenance.js"; import { removeRuntimeAssetTree } from "../services/native-runtime/runtime-asset-retention.js"; import { eq } from "drizzle-orm"; @@ -80,93 +78,6 @@ describe.skipIf(!support.supported)("managed skills use committed versions", () }); it("loads persisted Cloud configuration for read-only maintenance without seeding it", async () => { - const companyId = await company(); - const objects = new Map(); - const server = createServer((request, response) => { - void (async () => { - const key = new URL(request.url!, "http://fixture").pathname; - if (request.method === "PUT") { - const chunks: Buffer[] = []; - for await (const chunk of request) chunks.push(Buffer.from(chunk)); - let body = Buffer.concat(chunks); - if (request.headers["content-encoding"]?.includes("aws-chunked")) { - const decoded: Buffer[] = []; - let offset = 0; - while (offset < body.length) { - const end = body.indexOf("\r\n", offset); - const size = Number.parseInt(body.subarray(offset, end).toString().split(";")[0]!, 16); - if (size === 0) break; - decoded.push(body.subarray(end + 2, end + 2 + size)); offset = end + 2 + size + 2; - } - body = Buffer.concat(decoded); - } - objects.set(key, body); response.writeHead(200, { etag: '"fixture"' }).end(); return; - } - const body = objects.get(key); - if (!body) { response.writeHead(404).end(); return; } - response.writeHead(200, { "content-length": body.length, "content-type": "application/octet-stream", etag: '"fixture"' }); - response.end(request.method === "HEAD" ? undefined : body); - })().catch(() => { response.writeHead(500).end(); }); - }); - await new Promise(resolve => { server.listen(0, "127.0.0.1", resolve); }); - const address = server.address(); if (!address || typeof address === "string") throw new Error("Expected fixture address"); - const endpoint = `http://127.0.0.1:${address.port}`; - const credentials = { AWS_ACCESS_KEY_ID: "fixture-access-key", AWS_SECRET_ACCESS_KEY: "fixture-secret-key", AWS_SESSION_TOKEN: "" }; - const previousCredentials = Object.fromEntries(Object.keys(credentials).map(key => [key, process.env[key]])); - Object.assign(process.env, credentials); - vi.mocked(storageModule.getStorageService).mockReturnValue(createStorageService(createS3StorageProvider({ - bucket: "managed-skill-fixture", region: "us-east-1", endpoint, forcePathStyle: true, - }))); - const key = "cloud-runtime-config:v1"; - try { - await companySkillService(db).createLocalSkill(companyId, { name: "Cloud CLI", markdown: "# Cloud CLI" }); - const run = (phase: string) => promisify(execFile)(process.execPath, [ - "--import", fileURLToPath(import.meta.resolve("tsx")), - fileURLToPath(new URL("../../scripts/migrate-skill-packages.ts", import.meta.url)), - `--phase=${phase}`, `--company=${companyId}`, "--limit=50", - ], { env: { ...process.env, ...credentials, DATABASE_URL: database.connectionString, - PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN: "disposable-cloud-cli-fixture", - PAPERCLIP_SECRETS_MASTER_KEY: "a".repeat(64), PAPERCLIP_STORAGE_PROVIDER: "s3", - PAPERCLIP_STORAGE_S3_BUCKET: "managed-skill-fixture", PAPERCLIP_STORAGE_S3_REGION: "us-east-1", - PAPERCLIP_STORAGE_S3_ENDPOINT: endpoint, PAPERCLIP_STORAGE_S3_FORCE_PATH_STYLE: "true", PAPERCLIP_STORAGE_S3_PREFIX: "" }, - timeout: 60_000, maxBuffer: 1024 * 1024 }); - await expect(run("eligibility")).rejects.toThrow("Initialize Cloud runtime configuration"); - expect(await db.select().from(instanceSettings).where(eq(instanceSettings.singletonKey, key))).toHaveLength(0); - await db.insert(instanceSettings).values({ singletonKey: key, general: { version: 1, revision: 0, config: null } }); - const eligibility = JSON.parse((await run("eligibility")).stdout.trim().split("\n").at(-1)!); - expect(eligibility.results).toEqual([expect.objectContaining({ status: "version_backed" })]); - const before = [...objects.entries()].map(([key, body]) => [key, Buffer.from(body)] as const); - const verification = JSON.parse((await run("verification")).stdout.trim().split("\n").at(-1)!); - expect(verification.results).toEqual([expect.objectContaining({ status: "remote_storage" })]); - expect([...objects.entries()]).toEqual(before); - const archiveKey = [...objects.keys()].find(key => key.endsWith("/files.tar.gz"))!; - const archive = objects.get(archiveKey)!; - const corrupted = Buffer.from(archive); corrupted[0] = corrupted[0]! ^ 1; - objects.set(archiveKey, corrupted); - const corrupt = JSON.parse((await run("verification")).stdout.trim().split("\n").at(-1)!); - expect(corrupt.results).toEqual([expect.objectContaining({ status: "corrupt_package" })]); - objects.set(archiveKey, archive); - const manifestKey = [...objects.keys()].find(key => key.endsWith("/manifest.json"))!; - const manifest = objects.get(manifestKey)!; objects.delete(manifestKey); - const unavailable = JSON.parse((await run("verification")).stdout.trim().split("\n").at(-1)!); - expect(unavailable.results).toEqual([expect.objectContaining({ status: "unavailable_or_corrupt_package" })]); - objects.set(manifestKey, manifest); - const recovered = JSON.parse((await run("verification")).stdout.trim().split("\n").at(-1)!); - expect(recovered.results).toEqual([expect.objectContaining({ status: "remote_storage" })]); - expect([...objects.entries()].sort()).toEqual([...before].sort()); - expect(await db.select().from(instanceSettings).where(eq(instanceSettings.singletonKey, key))) - .toEqual([expect.objectContaining({ general: { version: 1, revision: 0, config: null } })]); - } finally { - await db.delete(instanceSettings).where(eq(instanceSettings.singletonKey, key)); - vi.mocked(storageModule.getStorageService).mockReturnValue(createStorageService(createLocalDiskStorageProvider(path.join(root, "objects")))); - for (const [key, value] of Object.entries(previousCredentials)) { - if (value === undefined) delete process.env[key]; else process.env[key] = value; - } - server.closeAllConnections(); await new Promise(resolve => { server.close(() => resolve()); }); - } - }, 180_000); - - it("reports a provider mismatch for local versions under persisted Cloud configuration", async () => { const companyId = await company(); const skill = await companySkillService(db).createLocalSkill(companyId, { name: "Cloud CLI", markdown: "# Cloud CLI" }); const local = await skillInstallationMaintenance(db).verifyVersions({ companyId, limit: 50 }); @@ -179,7 +90,7 @@ describe.skipIf(!support.supported)("managed skills use committed versions", () ], { env: { ...process.env, DATABASE_URL: database.connectionString, PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN: "disposable-cloud-cli-fixture", PAPERCLIP_STORAGE_PROVIDER: "s3", PAPERCLIP_STORAGE_S3_BUCKET: "managed-skill-cli-fixture", - PAPERCLIP_STORAGE_S3_REGION: "us-east-1", PAPERCLIP_SECRETS_MASTER_KEY: "a".repeat(64) }, + PAPERCLIP_STORAGE_S3_REGION: "us-east-1", PAPERCLIP_SECRETS_MASTER_KEY: "12345678901234567890123456789012" }, timeout: 60_000, maxBuffer: 1024 * 1024 }); await expect(run("eligibility")).rejects.toThrow("Initialize Cloud runtime configuration"); expect(await db.select().from(instanceSettings).where(eq(instanceSettings.singletonKey, key))).toHaveLength(0); From 0fac69e1679270f48700d98cbae6cce8558d9554 Mon Sep 17 00:00:00 2001 From: Dotta Date: Fri, 30 Oct 2026 17:04:31 -0500 Subject: [PATCH 33/70] fix(server): scope run retention hints to affected companies Co-Authored-By: Paperclip --- server/src/__tests__/execution-control-work.test.ts | 4 ++-- server/src/services/execution-control-notifications.ts | 9 ++++++--- .../src/services/execution-control-reconciliation.ts | 2 +- server/src/services/execution-recovery-resolution.ts | 4 ++-- server/src/services/explicit-native-continuation.ts | 2 +- server/src/services/heartbeat.ts | 2 +- server/src/services/heartbeat/run-lifecycle.ts | 2 +- server/src/services/heartbeat/runtime-execution.ts | 4 ++-- server/src/services/issue-recovery-actions.ts | 10 +++++----- server/src/services/legacy-execution-recovery.ts | 2 +- .../native-runtime/native-failed-workspace-recovery.ts | 2 +- .../services/native-runtime/native-restart-recovery.ts | 2 +- .../services/native-runtime/native-run-finalizer.ts | 4 ++-- .../services/native-runtime/native-run-projection.ts | 2 +- .../services/native-runtime/native-safe-replacement.ts | 4 ++-- .../services/native-runtime/native-session-executor.ts | 6 +++--- .../services/native-runtime/native-superseded-retry.ts | 2 +- server/src/services/system-recovery.ts | 2 +- server/src/services/task-retry.ts | 2 +- 19 files changed, 35 insertions(+), 32 deletions(-) diff --git a/server/src/__tests__/execution-control-work.test.ts b/server/src/__tests__/execution-control-work.test.ts index 0046d8574aa..14cd9456eb5 100644 --- a/server/src/__tests__/execution-control-work.test.ts +++ b/server/src/__tests__/execution-control-work.test.ts @@ -383,11 +383,11 @@ describe.skipIf(!support.supported)("execution-control durable work", () => { const due = new Date(Date.now() + 120_000), earlier = new Date(Date.now() + 60_000); for (const at of [due, earlier]) { await writeExecutionControl(db, DELIVERY_QUEUES.executionFinalization, tx => tx.update(heartbeatRuns) - .set({ executionControlDeadlineAt: at }).where(eq(heartbeatRuns.id, run.id))); + .set({ executionControlDeadlineAt: at }).where(eq(heartbeatRuns.id, run.id)), { companyId }); await settle(); expect(c.nextWakeAt()).toBe(at.getTime()); } await writeExecutionControl(db, DELIVERY_QUEUES.executionFinalization, tx => tx.update(heartbeatRuns) - .set({ executionControlDeadlineAt: null }).where(eq(heartbeatRuns.id, run.id))); + .set({ executionControlDeadlineAt: null }).where(eq(heartbeatRuns.id, run.id)), { companyId }); await quiet(c); }); it("delivers a reconciled continuation when marking and resolving commit together", async () => { diff --git a/server/src/services/execution-control-notifications.ts b/server/src/services/execution-control-notifications.ts index c7b26a8bd72..eb2ea380c0b 100644 --- a/server/src/services/execution-control-notifications.ts +++ b/server/src/services/execution-control-notifications.ts @@ -2,8 +2,10 @@ import { notifyStatusCardIssueChange } from "./status-card-notifications.js"; import type { Db } from "@paperclipai/db"; import { DELIVERY_QUEUES, notifyDeliveryWork, type DeliveryQueue } from "./delivery-work-notifications.js"; -export async function notifyExecutionRunWork(transaction: object) { - await notifyStatusCardIssueChange(transaction); +export type ExecutionWorkScope = { companyId: string; agentId?: string; runId?: string; issueId?: string }; + +export async function notifyExecutionRunWork(transaction: object, scope?: ExecutionWorkScope) { + if (scope) await notifyStatusCardIssueChange(transaction, scope.companyId); for (const queue of [DELIVERY_QUEUES.executionStatus, DELIVERY_QUEUES.executionFinalization, DELIVERY_QUEUES.executionReplacement, DELIVERY_QUEUES.executionDisposition, DELIVERY_QUEUES.issueMonitors]) { await notifyDeliveryWork(transaction, queue); @@ -14,9 +16,10 @@ export function writeExecutionControl( db: Db | Parameters[0]>[0], queue: DeliveryQueue | readonly DeliveryQueue[], write: (tx: Db) => PromiseLike, + scope: ExecutionWorkScope | undefined, ): Promise { return db.transaction(async transaction => { - await notifyStatusCardIssueChange(transaction); + if (scope) await notifyStatusCardIssueChange(transaction, scope.companyId); await notifyDeliveryWork(transaction, DELIVERY_QUEUES.issueMonitors); for (const topic of typeof queue === "string" ? [queue] : queue) await notifyDeliveryWork(transaction, topic); return write(transaction as unknown as Db); diff --git a/server/src/services/execution-control-reconciliation.ts b/server/src/services/execution-control-reconciliation.ts index 2f2a435982d..621fd210cfc 100644 --- a/server/src/services/execution-control-reconciliation.ts +++ b/server/src/services/execution-control-reconciliation.ts @@ -196,7 +196,7 @@ export async function reconcileAbandonedExecutionControl( await notifyNativeEnvironmentSettlement(tx, { companyId: run.companyId, runId: run.id }); await notifyCancelledWorkspacePreservation(tx, { companyId: run.companyId, runId: run.id }); await notifyUnsafeWorkspaceExportRunChange(tx, run); - await notifyExecutionRunWork(tx); + await notifyExecutionRunWork(tx, { companyId: run.companyId }); const [updatedRun] = await tx .update(heartbeatRuns) .set({ diff --git a/server/src/services/execution-recovery-resolution.ts b/server/src/services/execution-recovery-resolution.ts index f3f5a3e4310..d56e7d926f1 100644 --- a/server/src/services/execution-recovery-resolution.ts +++ b/server/src/services/execution-recovery-resolution.ts @@ -546,7 +546,7 @@ export async function settleUnrecoverableExecutions( companyId: run.companyId, issueId: task.id, runId: run.id, }).limit(1).for("update"); if (!hold) return; - await notifyExecutionRunWork(tx); + await notifyExecutionRunWork(tx, { companyId: run.companyId }); const note = "Exact session recovery stopped. The recorded workspace recovery holds this execution without replaying its actions."; await tx.update(issueRecoveryActions).set({ status: "cancelled", outcome: "cancelled", resolvedAt: now, updatedAt: now, wakePolicy: null, monitorPolicy: null, @@ -654,7 +654,7 @@ export async function settleUnrecoverableExecutions( replay: "not_authorized", }, }); - await notifyExecutionRunWork(tx); + await notifyExecutionRunWork(tx, { companyId: run.companyId }); await tx .update(heartbeatRuns) .set({ executionStatusDeliveryId: randomUUID() }) diff --git a/server/src/services/explicit-native-continuation.ts b/server/src/services/explicit-native-continuation.ts index 074ffc3bdf5..084a25e457f 100644 --- a/server/src/services/explicit-native-continuation.ts +++ b/server/src/services/explicit-native-continuation.ts @@ -475,7 +475,7 @@ export async function admitExplicitNativeContinuation(input: { }).where(and(eq(agentWakeupRequests.companyId, companyId), eq(agentWakeupRequests.id, input.queuedCommentInterruptId), eq(agentWakeupRequests.agentId, agentId), eq(agentWakeupRequests.status, "deferred_issue_execution"))); if (cancelledStartupIds.size) await db.transaction(async tx => { - await notifyExecutionRunWork(tx); + await notifyExecutionRunWork(tx, { companyId, agentId }); for (const runId of cancelledStartupIds) { await tx.update(nativeRunFinalizations).set({ phase: "terminal_failure", failureCode: "native_startup_cancelled", nextAttemptAt: null, diff --git a/server/src/services/heartbeat.ts b/server/src/services/heartbeat.ts index eb655fdef25..a4cec7b81b0 100644 --- a/server/src/services/heartbeat.ts +++ b/server/src/services/heartbeat.ts @@ -3724,7 +3724,7 @@ export function heartbeatService( await writeExecutionControl(db, DELIVERY_QUEUES.executionFinalization, tx => tx .update(heartbeatRuns) .set({ executionControlDeadlineAt: null }) - .where(eq(heartbeatRuns.id, run.id))); + .where(eq(heartbeatRuns.id, run.id)), { companyId: run.companyId }); } nativeOwnershipHeld = nativeOwnershipHeld || diff --git a/server/src/services/heartbeat/run-lifecycle.ts b/server/src/services/heartbeat/run-lifecycle.ts index 00a27c1faf2..9b9eaad3061 100644 --- a/server/src/services/heartbeat/run-lifecycle.ts +++ b/server/src/services/heartbeat/run-lifecycle.ts @@ -1044,7 +1044,7 @@ export function createHeartbeatLifecycle(db: Db, dependencies: HeartbeatLifecycl if (previous && isHeartbeatRunTerminalStatus(status)) await notifyRetainedNativeSessionCleanup(tx, { companyId: previous.companyId, runId: previous.id }); if (isHeartbeatRunTerminalStatus(status)) await notifyDeliveryWork(tx, DELIVERY_QUEUES.accounting, companyId); - await notifyExecutionRunWork(tx); + await notifyExecutionRunWork(tx, companyId ? { companyId } : undefined); if (previous) await notifyChatRunMilestones(tx, { companyId: previous.companyId, runId: previous.id, issueId: previous.nativeIssueId ?? previous.issueId ?? readNonEmptyString(parseObject(previous.contextSnapshot).issueId) ?? undefined }); return write(tx); diff --git a/server/src/services/heartbeat/runtime-execution.ts b/server/src/services/heartbeat/runtime-execution.ts index b5b64abc948..bdbbfa07752 100644 --- a/server/src/services/heartbeat/runtime-execution.ts +++ b/server/src/services/heartbeat/runtime-execution.ts @@ -966,7 +966,7 @@ export async function executeHeartbeatRuntime(db: Db, input: HeartbeatRuntimeExe eq(heartbeatRuns.id, run.id), eq(heartbeatRuns.status, "running"), ), - )); + ), { companyId: run.companyId }); const workspaceFinalizeStatus = hasWorkspaceRestoreFailure(adapterResult.resultJson) ? "failed" : "succeeded"; await recordWorkspaceFinalize(workspaceFinalizeStatus); if (adapterResult.nativeFinalization) { @@ -1042,7 +1042,7 @@ export async function executeHeartbeatRuntime(db: Db, input: HeartbeatRuntimeExe eq(heartbeatRuns.id, run.id), eq(heartbeatRuns.status, "running"), ), - )); + ), { companyId: run.companyId }); if ( issueRef && context.resumeSessionGoalHeartbeat === true && diff --git a/server/src/services/issue-recovery-actions.ts b/server/src/services/issue-recovery-actions.ts index 5fb28bcb6d8..48fb001c0b4 100644 --- a/server/src/services/issue-recovery-actions.ts +++ b/server/src/services/issue-recovery-actions.ts @@ -392,7 +392,7 @@ export function issueRecoveryActionService(db: DbOrTransaction) { const [updated] = await writeExecutionControl(db, [DELIVERY_QUEUES.executionDisposition, DELIVERY_QUEUES.executionContinuation], tx => tx.update(issueRecoveryActions).set({ nextAction: input.nextAction, updatedAt: now }) .where(and(eq(issueRecoveryActions.id, existing.id), inArray(issueRecoveryActions.status, [...ACTIVE_RECOVERY_ACTION_STATUSES]), - sql`${issueRecoveryActions.evidence}->'workspaceOwnerStop' is null`)).returning()); + sql`${issueRecoveryActions.evidence}->'workspaceOwnerStop' is null`)).returning(), { companyId: input.companyId }); if (updated) return toReadModel(updated); } return existing; @@ -467,7 +467,7 @@ export function issueRecoveryActionService(db: DbOrTransaction) { ), ) .returning(); - }); + }, { companyId: input.companyId }); if (!exhausted) { return retryUpsertSourceScoped(input, retryCount); } @@ -539,7 +539,7 @@ export function issueRecoveryActionService(db: DbOrTransaction) { ), ) .returning(); - }); + }, { companyId: input.companyId }); if (!updated) { return retryUpsertSourceScoped(input, retryCount); } @@ -554,7 +554,7 @@ export function issueRecoveryActionService(db: DbOrTransaction) { .insert(issueRecoveryActions) .values(buildInsertValues(input, ownerType, now)) .returning(); - }); + }, { companyId: input.companyId }); return toReadModel(created!); } catch (error) { if (!isUniqueRecoveryActionConflict(error)) throw error; @@ -608,7 +608,7 @@ export function issueRecoveryActionService(db: DbOrTransaction) { }) .where(and(...predicates)) .returning(); - }); + }, { companyId: input.companyId }); return updated ? toReadModel(updated) : null; } diff --git a/server/src/services/legacy-execution-recovery.ts b/server/src/services/legacy-execution-recovery.ts index 427704de63f..c462cdf1fca 100644 --- a/server/src/services/legacy-execution-recovery.ts +++ b/server/src/services/legacy-execution-recovery.ts @@ -135,7 +135,7 @@ export async function terminalizeLegacyExecution(input: { }; if (issueId) await notifyUnsafeWorkspaceExportRecovery(tx, { companyId: current.companyId, issueId }); await notifyInstructionCollection(tx, { companyId: current.companyId, runId: current.id }); - await notifyExecutionRunWork(tx); + await notifyExecutionRunWork(tx, { companyId: current.companyId }); let [updated] = await tx .update(heartbeatRuns) .set({ diff --git a/server/src/services/native-runtime/native-failed-workspace-recovery.ts b/server/src/services/native-runtime/native-failed-workspace-recovery.ts index 5c3268f6afc..a1c6c914b33 100644 --- a/server/src/services/native-runtime/native-failed-workspace-recovery.ts +++ b/server/src/services/native-runtime/native-failed-workspace-recovery.ts @@ -65,7 +65,7 @@ export async function preserveNativeFailedWorkspace(db: Db, selected: Pick>'{systemRecovery,revision}' = ${String(previous.revision)}`)) .returning({ id: issueRecoveryActions.id }); return Boolean(updated); - }); + }, { companyId: source.companyId }); } export async function loadSystemRecovery(db: Db, source: WorkspaceRecoverySourceIdentity) { diff --git a/server/src/services/task-retry.ts b/server/src/services/task-retry.ts index e9f0bd4edbf..17075c34eb9 100644 --- a/server/src/services/task-retry.ts +++ b/server/src/services/task-retry.ts @@ -334,7 +334,7 @@ export async function requestTaskRetry(input: { details: { requestId: active?.id ?? id, sourceRunId: latest.id, attachedToExistingWork: Boolean(active || busy) } }); return { requestId: active?.id ?? id, accepted: true, stale: false, observedRunId: latest.id, chatActionId: intent.chatActionId }; - }); + }, { companyId: input.companyId }); } /** Chat owns admission and authorization. The task receipt follows that From 875d763c1d2a3396fc51a8f610a8421ea0bfc12f Mon Sep 17 00:00:00 2001 From: Dotta Date: Fri, 30 Oct 2026 17:24:55 -0500 Subject: [PATCH 34/70] fix: back off expired deadlines during domain drain Keep a future in-memory resume wake when domain dispatch is paused independently of coordinator admission. Preserve the saved obligation without rescheduling its expired deadline into an immediate skipped-attempt loop. Co-Authored-By: Paperclip --- server/src/__tests__/deadline-work.test.ts | 47 ++++++++++++++++++++++ server/src/services/deadline-work.ts | 2 +- 2 files changed, 48 insertions(+), 1 deletion(-) diff --git a/server/src/__tests__/deadline-work.test.ts b/server/src/__tests__/deadline-work.test.ts index 7657a7e9d99..c00daeb2f00 100644 --- a/server/src/__tests__/deadline-work.test.ts +++ b/server/src/__tests__/deadline-work.test.ts @@ -122,3 +122,50 @@ it("does not release an ambiguous database intent while dispatch is drained", as draining = false; await vi.advanceTimersByTimeAsync(30_000); expect(run).toHaveBeenCalledTimes(2); expect(coordinator.nextWakeAt()).toBeNull(); }); + +it("backs off an expired deadline when only the domain is drained and resumes promptly", async () => { + vi.useFakeTimers(); + let paused = false; + const owner = installDatabaseWorkSignals({ execute: vi.fn(async () => []) }); + const run = vi.fn(async () => {}); + const domainCanRun = vi.fn(() => !paused); + const deadline = Date.now() + 1_000; + const next = vi.fn(async () => run.mock.calls.length === 1 ? deadline : null); + coordinator = createDeliveryWorkCoordinator({ owner, canRun: () => true, onError: vi.fn() }); + const worker = registerDeadlineWork(coordinator, DELIVERY_QUEUES.agentTimers, { + run, next, canRun: domainCanRun, retryMs: 30_000, + }); + await worker.ready; + paused = true; + const before = idleWorkSnapshot(); + await vi.advanceTimersByTimeAsync(1_000); + expect(coordinator.nextWakeAt()).toBe(Date.now() + 30_000); + const attempts = domainCanRun.mock.calls.length; + await vi.advanceTimersByTimeAsync(90_000); + expect(domainCanRun.mock.calls.length - attempts).toBeLessThanOrEqual(12); + // Four attempts (expiry plus three retries), each begins and ends tracked work. + expect(idleWorkSnapshot().generation - before.generation).toBe(8); + expect(idleWorkSnapshot().active).toBe(before.active); + expect(run).toHaveBeenCalledOnce(); expect(next).toHaveBeenCalledOnce(); expect(owner.execute).not.toHaveBeenCalled(); + paused = false; + await vi.advanceTimersByTimeAsync(30_000); + expect(run).toHaveBeenCalledTimes(2); expect(coordinator.nextWakeAt()).toBeNull(); +}); + +it("retains a prompt paused wake when a new request preempts a far-future deadline", async () => { + vi.useFakeTimers(); + let paused = false; + const owner = installDatabaseWorkSignals({ execute: vi.fn(async () => []) }); + const run = vi.fn(async () => {}); + const deadline = Date.now() + 86_400_000; + coordinator = createDeliveryWorkCoordinator({ owner, canRun: () => true, onError: vi.fn() }); + const worker = registerDeadlineWork(coordinator, DELIVERY_QUEUES.agentTimers, { + run, next: async () => run.mock.calls.length === 1 ? deadline : null, canRun: () => !paused, retryMs: 30_000, + }); + await worker.ready; paused = true; worker.wake(); + await vi.advanceTimersByTimeAsync(0); + expect(coordinator.nextWakeAt()).toBe(Date.now() + 30_000); + expect(owner.execute).not.toHaveBeenCalled(); + paused = false; await vi.advanceTimersByTimeAsync(30_000); + expect(run).toHaveBeenCalledTimes(2); expect(coordinator.nextWakeAt()).toBeNull(); +}); diff --git a/server/src/services/deadline-work.ts b/server/src/services/deadline-work.ts index 15ba76abdae..208b208e051 100644 --- a/server/src/services/deadline-work.ts +++ b/server/src/services/deadline-work.ts @@ -52,7 +52,7 @@ export function registerDeadlineWork( // the domain to probe its database while dispatch is closed. hasPending: async () => task.canRun() && requested !== reconciled, nextRunAt: () => !task.canRun() && requested !== reconciled - ? Math.min(next ?? Infinity, Date.now() + task.retryMs) + ? Date.now() + task.retryMs : next, }); return { ...worker, wake: () => { requested++; worker.wake(); } }; From 9a5c429043cd5927499aa16cb24cbdc6375ec7ac Mon Sep 17 00:00:00 2001 From: Dotta Date: Fri, 30 Oct 2026 17:33:00 -0500 Subject: [PATCH 35/70] fix(server): scope lifecycle hints from selected run fields Co-Authored-By: Paperclip --- server/src/services/heartbeat/run-lifecycle.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/services/heartbeat/run-lifecycle.ts b/server/src/services/heartbeat/run-lifecycle.ts index ac4e800f279..a1c4fa20fa3 100644 --- a/server/src/services/heartbeat/run-lifecycle.ts +++ b/server/src/services/heartbeat/run-lifecycle.ts @@ -1044,7 +1044,7 @@ export function createHeartbeatLifecycle(db: Db, dependencies: HeartbeatLifecycl if (previous && isHeartbeatRunTerminalStatus(status)) await notifyRetainedNativeSessionCleanup(tx, { companyId: previous.companyId, runId: previous.id }); if (isHeartbeatRunTerminalStatus(status)) await notifyDeliveryWork(tx, DELIVERY_QUEUES.accounting, companyId); - await notifyExecutionRunWork(tx, previous ? { companyId: previous.companyId, agentId: previous.agentId, runId: previous.id, + await notifyExecutionRunWork(tx, previous ? { companyId: previous.companyId, runId: previous.id, issueId: previous.nativeIssueId ?? previous.issueId ?? readNonEmptyString(parseObject(previous.contextSnapshot).issueId) ?? undefined } : undefined); if (previous) await notifyChatRunMilestones(tx, { companyId: previous.companyId, runId: previous.id, issueId: previous.nativeIssueId ?? previous.issueId ?? readNonEmptyString(parseObject(previous.contextSnapshot).issueId) ?? undefined }); From 2cc2f9ae8cbf355ffba46596b38ce8eb7a04270f Mon Sep 17 00:00:00 2001 From: Dotta Date: Fri, 30 Oct 2026 17:39:30 -0500 Subject: [PATCH 36/70] fix(server): serialize direct routine status hooks Co-Authored-By: Paperclip --- doc/plans/2026-10-30-routine-status-sleep.md | 4 ++- .../routine-execution-status-work.test.ts | 34 +++++++++++++++++++ server/src/services/routine-run-status.ts | 8 ++++- 3 files changed, 44 insertions(+), 2 deletions(-) diff --git a/doc/plans/2026-10-30-routine-status-sleep.md b/doc/plans/2026-10-30-routine-status-sleep.md index 271d0b544a7..e852bf94861 100644 --- a/doc/plans/2026-10-30-routine-status-sleep.md +++ b/doc/plans/2026-10-30-routine-status-sleep.md @@ -18,7 +18,9 @@ and belong to a routine in that company. This is the existing routine status owner contract. Origin labels can change when a plugin declaration changes or is removed; retained reciprocal links preserve historical execution custody. Repair locks the routine and task and rechecks the linked invocation before writing. -It preserves skipped/coalesced history and unrelated dispatch failures. A task +Direct route hooks also lock the execution task and linked invocation in their +own transaction, nested under caller transactions when present. An older hook +cannot write after a newer task status has committed. It preserves skipped/coalesced history and unrelated dispatch failures. A task that resumes clears an execution-task transient failure through the same owner used by native completion and issue routes. diff --git a/server/src/__tests__/routine-execution-status-work.test.ts b/server/src/__tests__/routine-execution-status-work.test.ts index 53d7e5c0225..e6bd818852c 100644 --- a/server/src/__tests__/routine-execution-status-work.test.ts +++ b/server/src/__tests__/routine-execution-status-work.test.ts @@ -174,6 +174,40 @@ const support = external ? { supported: true } : await getEmbeddedPostgresTestSu await db.transaction(async tx => { await notifyRoutineExecutionStatus(tx, otherRun.linkedIssueId!); }); await projected(otherRun.id, "completed"); await quiet(); expect(errors).toEqual([]); }); + it("serializes a direct status hook with later task changes before projecting the routine", async () => { + const f = await fixture(), run = await dispatch(f); await start(); await quiet(); + await db.update(issues).set({ status: "done" }).where(eq(issues.id, run.linkedIssueId!)); + let entered!: () => void, release!: () => void; + const enteredGate = new Promise(resolve => { entered = resolve; }); + const releaseGate = new Promise(resolve => { release = resolve; }); + const holding = db.transaction(async tx => { + await tx.select().from(routineRuns).where(eq(routineRuns.id, run.id)).for("update"); + entered(); await releaseGate; + }); + let direct: Promise | undefined, changing: Promise | undefined; + let directPid: number | undefined; + try { + await enteredGate; direct = f.service.syncRunStatusForIssue(run.linkedIssueId!); + await vi.waitFor(async () => { + const rows = await db.execute(sql`select pid from pg_stat_activity + where datname = current_database() and wait_event_type = 'Lock' + and (query like 'update "routine_runs"%' or query like 'select %"routine_runs"%')`); + expect(rows).toHaveLength(1); directPid = Number(rows[0]!.pid); + }, { timeout: 5000 }); + await expect(db.transaction(tx => tx.select({ id: issues.id }).from(issues) + .where(eq(issues.id, run.linkedIssueId!)).for("update", { noWait: true }))) + .rejects.toMatchObject({ cause: expect.objectContaining({ code: "55P03" }) }); + changing = issueService(db).update(run.linkedIssueId!, { status: "blocked" }); + await vi.waitFor(async () => { + const rows = await db.execute(sql`select count(*)::int as count from pg_stat_activity + where datname = current_database() and wait_event_type = 'Lock' + and ${directPid}::int = any(pg_blocking_pids(pid))`); + expect(Number(rows[0]?.count)).toBeGreaterThan(0); + }, { timeout: 5000 }); + release(); await holding; await direct; await changing; + await projected(run.id, "failed"); await quiet(); expect(errors).toEqual([]); + } finally { release(); await holding; await direct; await changing; } + }); it("waits for the real outer transaction and forgets a rolled-back nested status write", async () => { const f = await fixture(), run = await dispatch(f); await start(); await quiet(); let release!: () => void, entered!: () => void; diff --git a/server/src/services/routine-run-status.ts b/server/src/services/routine-run-status.ts index 2a182def8f9..46a2bae4cf5 100644 --- a/server/src/services/routine-run-status.ts +++ b/server/src/services/routine-run-status.ts @@ -51,6 +51,10 @@ export async function finalizeRoutineRun(executor: Db, runId: string, patch: Par } export async function syncRoutineRunStatusForIssue(executor: Db, issueId: string) { + return executor.transaction(tx => syncLockedRoutineRunStatusForIssue(tx as unknown as Db, issueId)); +} + +async function syncLockedRoutineRunStatusForIssue(executor: Db, issueId: string) { const issue = await executor .select({ id: issues.id, @@ -60,7 +64,8 @@ export async function syncRoutineRunStatusForIssue(executor: Db, issueId: string originRunId: issues.originRunId, }) .from(issues) - .where(eq(issues.id, issueId)) + .where(and(eq(issues.id, issueId), eq(issues.originKind, "routine_execution"))) + .for("update") .then((rows) => rows[0] ?? null); if (!issue || issue.originKind !== "routine_execution" || !issue.originRunId) return null; const run = await executor @@ -75,6 +80,7 @@ export async function syncRoutineRunStatusForIssue(executor: Db, issueId: string .innerJoin(routines, and(eq(routines.id, routineRuns.routineId), eq(routines.companyId, issue.companyId))) .where(and(eq(routineRuns.id, issue.originRunId), eq(routineRuns.companyId, issue.companyId), eq(routineRuns.linkedIssueId, issue.id))) + .for("update", { of: routineRuns }) .then((rows) => rows[0] ?? null); if (!run || ["received", "skipped", "coalesced"].includes(run.status)) return null; if (issue.status === "done") { From 228196a1101febe66bfcd68fa0186d2ca299d0c1 Mon Sep 17 00:00:00 2001 From: Dotta Date: Fri, 30 Oct 2026 17:47:09 -0500 Subject: [PATCH 37/70] test: prove timer deadlines survive settled drain holds Expect committed producers to release idle admission while retaining a resume deadline, then verify actual dispatch and final quiet after resume. Co-Authored-By: Paperclip --- server/src/__tests__/agent-timer-deadlines.test.ts | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/server/src/__tests__/agent-timer-deadlines.test.ts b/server/src/__tests__/agent-timer-deadlines.test.ts index 4c4c0d324a9..f925fb8dfd7 100644 --- a/server/src/__tests__/agent-timer-deadlines.test.ts +++ b/server/src/__tests__/agent-timer-deadlines.test.ts @@ -597,8 +597,10 @@ describe.skipIf(!support.supported)("agent timer owned deadlines", () => { await agentService(db).update(f.agent.id, { runtimeConfig: { heartbeat: { enabled: true, intervalSec: 600 } } }); await vi.advanceTimersByTimeAsync(2000); - - expect(idleWorkSnapshot().active).toBe(1); + // The producer committed: retain its resume deadline without keeping the + // instance busy throughout the drain. Uncertain writes still hold admission. + expect(idleWorkSnapshot().active).toBe(0); + expect(coordinator!.nextWakeAt()).toBeGreaterThan(Date.now()); const select = vi.spyOn(db, "select"), execute = vi.spyOn(db, "execute"), transaction = vi.spyOn(db, "transaction"); try { await vi.advanceTimersByTimeAsync(60_000); @@ -608,6 +610,11 @@ describe.skipIf(!support.supported)("agent timer owned deadlines", () => { stopTaskDrain(); await vi.advanceTimersByTimeAsync(1000); await vi.waitFor(() => expect(coordinator!.nextWakeAt()).toBe(now.getTime() + 600_000), { timeout: 3000 }); + await vi.advanceTimersByTimeAsync(coordinator!.nextWakeAt()! - Date.now()); + await vi.waitFor(() => expect(f.enqueueWakeup).toHaveBeenCalledOnce()); + expect(f.enqueueWakeup).toHaveBeenCalledWith(f.agent.id, expect.objectContaining({ source: "timer", reason: "heartbeat_timer" })); + await agentService(db).update(f.agent.id, { runtimeConfig: { heartbeat: { enabled: false } } }); + await quiet(); }); it("does not inspect timer rows during startup drain, then reconstructs them on resume", async () => { From 938e22aa21cbd6c887f2451d9a339b2db44d11f4 Mon Sep 17 00:00:00 2001 From: Dotta Date: Fri, 30 Oct 2026 17:47:09 -0500 Subject: [PATCH 38/70] test: prove timer deadlines survive settled drain holds Expect committed producers to release idle admission while retaining a resume deadline, then verify actual dispatch and final quiet after resume. Co-Authored-By: Paperclip --- server/src/__tests__/agent-timer-deadlines.test.ts | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/server/src/__tests__/agent-timer-deadlines.test.ts b/server/src/__tests__/agent-timer-deadlines.test.ts index 4c4c0d324a9..f925fb8dfd7 100644 --- a/server/src/__tests__/agent-timer-deadlines.test.ts +++ b/server/src/__tests__/agent-timer-deadlines.test.ts @@ -597,8 +597,10 @@ describe.skipIf(!support.supported)("agent timer owned deadlines", () => { await agentService(db).update(f.agent.id, { runtimeConfig: { heartbeat: { enabled: true, intervalSec: 600 } } }); await vi.advanceTimersByTimeAsync(2000); - - expect(idleWorkSnapshot().active).toBe(1); + // The producer committed: retain its resume deadline without keeping the + // instance busy throughout the drain. Uncertain writes still hold admission. + expect(idleWorkSnapshot().active).toBe(0); + expect(coordinator!.nextWakeAt()).toBeGreaterThan(Date.now()); const select = vi.spyOn(db, "select"), execute = vi.spyOn(db, "execute"), transaction = vi.spyOn(db, "transaction"); try { await vi.advanceTimersByTimeAsync(60_000); @@ -608,6 +610,11 @@ describe.skipIf(!support.supported)("agent timer owned deadlines", () => { stopTaskDrain(); await vi.advanceTimersByTimeAsync(1000); await vi.waitFor(() => expect(coordinator!.nextWakeAt()).toBe(now.getTime() + 600_000), { timeout: 3000 }); + await vi.advanceTimersByTimeAsync(coordinator!.nextWakeAt()! - Date.now()); + await vi.waitFor(() => expect(f.enqueueWakeup).toHaveBeenCalledOnce()); + expect(f.enqueueWakeup).toHaveBeenCalledWith(f.agent.id, expect.objectContaining({ source: "timer", reason: "heartbeat_timer" })); + await agentService(db).update(f.agent.id, { runtimeConfig: { heartbeat: { enabled: false } } }); + await quiet(); }); it("does not inspect timer rows during startup drain, then reconstructs them on resume", async () => { From a12f68ef134d10b69357ec6d193ccf56b0db971f Mon Sep 17 00:00:00 2001 From: Dotta Date: Fri, 30 Oct 2026 17:50:38 -0500 Subject: [PATCH 39/70] fix: preserve scoped status-card work across mutations and recovery Complete transactional source coverage, cached feature gating, startup scope reconstruction, off-page relative query expiry and busy-lock retry. Verify producer, persistence, failure, budget resume, compatibility and startup paths with isolated PostgreSQL and focused tests. Co-Authored-By: Paperclip --- doc/plans/2026-10-30-status-cards-sleep.md | 113 ++++++++-- .../__tests__/company-search-service.test.ts | 14 +- .../server-startup-feedback-export.test.ts | 30 +++ .../status-card-evaluation-work.test.ts | 209 +++++++++++++++++- .../status-card-notifications.test.ts | 31 +++ .../modules/wake-queue/adapters/postgres.ts | 2 + .../adapters/queued-comment-postgres.ts | 3 + server/src/routes/execution-workspaces.ts | 6 +- server/src/services/agent-conversations.ts | 3 + server/src/services/budgets.ts | 3 + .../telegram/attachment-failure.ts | 2 + server/src/services/company-search.ts | 15 +- server/src/services/documents.ts | 5 + server/src/services/execution-workspaces.ts | 5 +- server/src/services/heartbeat/queue.ts | 3 + server/src/services/heartbeat/scheduling.ts | 6 + .../src/services/issue-thread-interactions.ts | 63 +++--- server/src/services/issues.ts | 34 ++- ...ative-status-reassessment-notifications.ts | 4 +- server/src/services/projects.ts | 3 + server/src/services/status-card-evaluation.ts | 14 +- server/src/services/status-cards.ts | 53 ++--- server/src/services/task-watchdogs.ts | 6 +- server/src/services/workspace-runtime.ts | 22 +- 24 files changed, 524 insertions(+), 125 deletions(-) create mode 100644 server/src/__tests__/status-card-notifications.test.ts diff --git a/doc/plans/2026-10-30-status-cards-sleep.md b/doc/plans/2026-10-30-status-cards-sleep.md index dcd6c5ca226..2af51ff65cf 100644 --- a/doc/plans/2026-10-30-status-cards-sleep.md +++ b/doc/plans/2026-10-30-status-cards-sleep.md @@ -1,33 +1,102 @@ # Status-card evaluation deadlines -Baseline: `76fde891ccc29283b3f5c8dcdce089c5b3d937d1`. +Status-card evaluation uses existing cards, updates and issues as durable truth. +`index.ts` registers a typed worker. The status-card owner controls evaluation, +generation claims, assignment through the ordinary issue wake API, and failed +assignment cancellation. Shared delivery infrastructure controls transaction +settlement, coalescing, retries, admission drain, standby and shutdown. -Status-card ownership covers evaluation policy, card claims, generation, ordinary assignment wakeups, and failed-assignment cancellation. `index.ts` registers a typed capability. Shared delivery/deadline infrastructure owns transaction settlement, coalescing, admission/drain, retries and shutdown. Existing cards, updates and issues remain durable truth; no timer table or sleep-safety exemption is introduced. +Manual cards have no automatic obligation. Interval cards retain the requested +evaluation cadence. Reactive cards evaluate committed source changes and retain +only an actual debounce, hourly rate, daily budget, active-hour or saved-query +membership deadline. Unchanged cards disarm. A changed experimental setting +invalidates the worker's cached feature gate. Other source writes do not reread +settings. Startup reconstructs every disarmed reactive scope together with any +concurrent company-scoped hint, without moving saved eligibility/retry deadlines. -Manual cards have no automatic obligation. Interval cards retain their requested evaluation cadence. Reactive cards evaluate committed source changes, retain the actual debounce/rate/budget/active-hours deadline while relevant changes remain, and disarm once unchanged. Saved relative-search windows retain their actual membership expiry; the duration interpretation stays with company search. Startup reconstructs existing work without advancing business timestamps. +Company search reports the earliest matching `updatedWithin` expiry through an +optional internal callback in its existing aggregate query. It includes rows +outside the returned page: an expiring priority-ranked row ahead of an offset +can change the watched set. The callback exposes only a normalized deadline; +ordinary search consumers and API responses retain their existing contract. +Overlapping query scopes can require an evaluation when one query expires even +if another still retains the same issue; no generation occurs without a filtered +card change. No recurring scan replaces the expired query obligation. -Scheduled reactive source events can preempt a future evaluation. A feature-owned transaction advisory lock serializes scheduler evaluation across connections while existing card/issue services perform their own committed writes. The lock holder has no card row lock, avoiding self-deadlock. Existing next-evaluation CAS, generating-issue ownership and issue idempotency keys remain required, including for manual refreshes and older server instances that do not use the advisory lock. The persisted five-minute claim remains the failure retry after a scheduled refresh cannot finish. Internal timers do not establish a Cloud wake alarm. +## Claims and recovery -## Producer audit (in progress) +A feature-owned PostgreSQL transaction advisory lock serializes scheduled +per-card evaluation across connections. Its transaction holds no card row lock +while existing issue/card services commit through their own transactions. The +lock remains held through actual refresh settlement. Existing card deadline CAS, +generating-issue ownership and issue idempotency keys remain in force. Manual +refresh and older instances continue to use those existing guards. A busy lock +retains a company-scoped retry; consuming a hint is not successful evaluation. +A failed scheduled refresh retains the existing five-minute durable claim. -| Source | Owning committed boundary | State | -| --- | --- | --- | -| Card create/config/archive/delete, compile, query/summary writeback, evaluation | status-card mutation transactions | wired; ordinary lifecycle exercised | -| Experimental feature flag | instance settings update transaction | wired; disable exercised | -| Company pause/archive/reactivate/delete | company owning transactions | wired; lifecycle acceptance remains | -| Ordinary issue create/update/remove | issue service transactions | wired; create/change exercised | -| Comments create/delete/tombstone | common insert and issue mutation transactions | wired; comment acceptance remains | -| Review state / chat-run milestones | issue-review-state notification boundary | wired; exact caller coverage audit remains | -| Execution-control writes/run settlement | execution-control transaction helpers | wired; exact caller coverage audit remains | -| Agent deletion/membership reassignment, issue tree transitions, title/prefix | owner transaction | wired; representative acceptance remains | -| Workspace selection/binding, plugin/provider workspace, native recovery/reclaim, active watchdog, retry | owner transactions where uncovered by shared notifications | partially wired; remaining per-write audit required | -| Raw workspace cleanup/routes, conversation activation/settlement, queue paths | direct writes and adjacent owner notifications | exact coverage incomplete; not acceptance-ready | -| Imports/independent writers/replicas | supported service writes and startup reconstruction | full ingress contract audit remains | +Once a generation exists, the normal issue assignment authority remains its +owner. Assignment failure cancels that issue and finalizes the card's failed +ledger entry. The user's ordinary refresh action can retry it. Restart recovery +must not repeat an already completed generation. No new timer table, automatic +write replay, execution-authority exemption or sleep-safety relaxation is added. -Process-local notifications cover writes using the installed database owner. They do not notify an unrelated SQL writer or another server process; those supported paths need explicit ingress or reconstruction coverage. No claim of cross-replica delivery, host suspension, or fleet savings is made by the current tests. +## Supported producer coverage -## Acceptance evidence and remaining gates +All hints are registered before the relevant write within its owning transaction. +Nested service writes share outer commit ownership. Company-scoped hints batch +reactive reconciliation; ordinary execution writes without known scope retain +all existing execution consumers without sending an instance-wide card hint. -The original status-card PostgreSQL suite passed25/25. New real PostgreSQL tests exercise ordinary card compilation/writeback, source-change debounce, persisted assignment at a controlled provider boundary, result persistence, quiet SQL after completion, earlier/cancelled deadlines, nested outer commit/rollback and drain/resume. The provider boundary is simulated; no model runs. Additional race, restart, failure, active-hour/rate/budget persistence, multi-company and producer-class coverage remain before PR delivery. +| Producer | Notification boundary and coverage | +| --- | --- | +| Card create, configuration, archive/restore, delete, compile, query/summary result, evaluation claim | Status-card mutation transactions; stalled issue finalization signals before releasing the card claim | +| Experimental status-card setting | Instance experimental-settings transaction; full reconstruction is appropriate for this instance-wide setting | +| Company pause/archive/reactivate/delete | Company lifecycle/deletion transactions and budget-owned company hard-stop/resume transactions | +| Issue create/update/delete and label changes | Ordinary issue transactions; label deletion signals its known company before cascades; title and prefix changes signal their existing owner transactions | +| Issue checkout/adoption/release/reset and environment-selection recency | Existing issue transaction, or a transaction around the previously standalone write; company comes from the existing issue/run read | +| Comment create, edit, delete, tombstone, import and recency | Common comment insert; outer root comment transaction; queued-comment and Telegram edit transactions; existing metadata-upgrade path; remove/tombstone transactions | +| Issue import, comment import, document import | One notification per affected company in the existing batch transaction; no per-row notification lookup | +| Document create/update/restore/delete and search membership | Document owner transactions; only the existing company is passed, without copying search predicates into producers | +| Review, interaction and execution transitions | Existing issue-review-state and native-status-reassessment boundaries; scoped execution-control helpers; interaction recency touches own a transaction when called after settlement | +| Run admission, retries, monitors, watchdog/recovery and native settlement/reclaim | Existing scoped review/milestone/status/execution-control hooks; explicit notifications at the uncovered issue execution bind, legacy claim, monitor claim/release and review-participant recovery transaction | +| Agent deletion/access reassignment and project deletion/privacy cascades | Existing company-scoped owner transactions before foreign-key or recursive issue changes | +| Workspace selection/binding, plugin/provider workspace and cleanup/audit touches | Existing owner transactions, including standalone cleanup and audit recency writes now enclosed with their intent | +| Skill-test harness expiry and task-watchdog fingerprint touch | Existing owner mutation transaction; no feature policy is moved into the writer | -Shared deadline regressions reproduced and corrected a settled-write admission hold during drain. The correction releases only settled holds, retains an in-memory resume wake, and leaves ambiguous transaction intent fenced by the database coordinator. Unit coverage includes first write during drain, failure entering drain, preserved future deadlines, warm standby and ambiguous commit settlement. +The audit includes direct issue/comment mutations, raw recursive privacy updates, +bulk insert helpers, label/project/agent cascades, linked document search content +and timestamp-dependent search membership. Metadata-only writes that do not +change the fingerprint, matching fields or visibility do not create an evaluation +obligation: examples include run identity linkage, workspace IDs without a +recency change, document locks and source-trust receipts without searched content +changes. Routine and status-summary documents that have no issue-document link +do not change issue search membership. + +This follows the existing [single-active-writer contract](../idle-sleep-safety.md). +Supported services, import/recovery writers, caller-owned transactions and startup +reconstruction participate. Independent SQL writers or another active server +process require explicit wake ingress; process-local hints do not establish +cross-replica notification delivery. The advisory lock supplies competing +scheduler exclusion, not cross-process event transport. Internal deadlines do +not establish a Cloud wake alarm or prove host/database suspension. + +## Verification + +Real PostgreSQL acceptance uses ordinary card create/compile and authorized +summarizer writeback. The assignment boundary is controlled: it persists a wake, +and the test explicitly performs the authorized result write; no model runs. +Checks cover empty/manual/reactive/disabled quiet SQL, first source change, +debounce and durable wake/result, earlier/cancelled deadlines, outer commit and +rollback, drain/resume, competing connections, busy locks, restart eligibility, +active hours/rate/budget gates, multiple companies, budget-owned company resume, +ignored activity, document/comment membership, relative query expiry including +an off-page candidate, failed assignment cleanup/manual retry and the persisted +five-minute internal-failure retry across restart/drain. + +Shared coordinator regressions also cover warm standby, ambiguous transaction +intent, failure entering drain and different domain/coordinator gates. The paused +worker releases only settled holds and retains a future in-memory resume wake. +An expired saved deadline cannot schedule an immediate skipped-attempt loop. +Existing startup, status-card, company-search, execution and monitor suites cover +the production wiring and unchanged consumers. Exact commands and results belong +in the PR verification record; this document does not claim deployed savings. diff --git a/server/src/__tests__/company-search-service.test.ts b/server/src/__tests__/company-search-service.test.ts index 5d9e53260fb..7dffe3d6a90 100644 --- a/server/src/__tests__/company-search-service.test.ts +++ b/server/src/__tests__/company-search-service.test.ts @@ -1,3 +1,4 @@ +import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js"; import { randomUUID } from "node:crypto"; import { sql } from "drizzle-orm"; import { afterAll, afterEach, beforeAll, describe, expect, it } from "vitest"; @@ -28,12 +29,13 @@ import { import { issueReadSqlCondition, projectReadSqlCondition } from "../services/authorization.js"; import { issueService } from "../services/issues.js"; -const embeddedPostgresSupport = await getEmbeddedPostgresTestSupport(); +const externalDatabaseUrl = resolveExternalTestDatabaseUrl("PAPERCLIP_COMPANY_SEARCH_TEST_DATABASE_URL"); +const embeddedPostgresSupport = externalDatabaseUrl ? { supported: true } : await getEmbeddedPostgresTestSupport(); const describeEmbeddedPostgres = embeddedPostgresSupport.supported ? describe : describe.skip; if (!embeddedPostgresSupport.supported) { console.warn( - `Skipping embedded Postgres company search tests on this host: ${embeddedPostgresSupport.reason ?? "unsupported environment"}`, + `Skipping embedded Postgres company search tests on this host: ${("reason" in embeddedPostgresSupport ? embeddedPostgresSupport.reason : "unsupported environment")}`, ); } @@ -80,8 +82,11 @@ describeEmbeddedPostgres("companySearchService", () => { let tempDb: Awaited> | null = null; beforeAll(async () => { - tempDb = await startEmbeddedPostgresTestDatabase("paperclip-company-search-"); - db = createDb(tempDb.connectionString); + if (!externalDatabaseUrl) tempDb = await startEmbeddedPostgresTestDatabase("paperclip-company-search-"); + db = createDb(externalDatabaseUrl ?? tempDb!.connectionString, { maxConnections: 6 }); + // External suites can reuse this dedicated database sequentially. Match + // the fresh embedded fixture before this suite creates its own rows. + if (externalDatabaseUrl) await db.execute(sql`TRUNCATE TABLE companies CASCADE`); svc = companySearchService(db); await db.execute(sql.raw("CREATE EXTENSION IF NOT EXISTS pg_trgm")); }, 20_000); @@ -100,6 +105,7 @@ describeEmbeddedPostgres("companySearchService", () => { }); afterAll(async () => { + await db?.$client.end({ timeout: 1 }); await tempDb?.cleanup(); }); diff --git a/server/src/__tests__/server-startup-feedback-export.test.ts b/server/src/__tests__/server-startup-feedback-export.test.ts index bbb020c865d..3008ee21f9c 100644 --- a/server/src/__tests__/server-startup-feedback-export.test.ts +++ b/server/src/__tests__/server-startup-feedback-export.test.ts @@ -1,3 +1,4 @@ +import { statusCardEvaluationService, registerStatusCardEvaluationWork } from "../services/status-card-evaluation.js"; import { nativeHarnessBackupExpirationService } from "../services/native-runtime/native-harness-backup-expiration.js"; import { skillVersionRetentionService } from "../services/skill-version-retention.js"; import { approvalQueueSeedingService } from "../services/approval-queue-seeding-work.js"; @@ -527,6 +528,13 @@ vi.mock("../services/native-runtime/native-question-bridge.js", () => ({ validateNativeQuestionResponseInput: vi.fn(), })); +vi.mock("../services/status-card-evaluation.js", () => ({ + statusCardEvaluationService: vi.fn(() => ({ run: vi.fn(async () => {}), next: vi.fn(async () => null), changed: vi.fn() })), + registerStatusCardEvaluationWork: vi.fn((_coordinator, service, canRun) => ({ + ready: canRun() ? service.run(new AbortController().signal) : Promise.resolve(), wake: vi.fn(), stop: vi.fn(async () => {}), + })), +})); + vi.mock("../services/secret-proposals.js", () => ({ createSecretProposalsService: vi.fn(() => ({ expiryDeadlines: vi.fn(async () => []), @@ -790,6 +798,28 @@ describe("startServer feedback export wiring", () => { expect(deliveryWorkErrors.filter(error => error.queue === DELIVERY_QUEUES.skillVersionRetention)).toEqual([]); }); + it.each([false, true])("registers status-card evaluation only with maintenance and never ticks it from the heartbeat loop (%s)", async enabled => { + loadConfigMock.mockReturnValue(buildTestConfig({ heartbeatSchedulerEnabled: enabled, heartbeatSchedulerIntervalMs: 1000 })); + let tick: (() => void) | undefined; + const timer = vi.spyOn(globalThis, "setInterval").mockImplementation(((callback: () => void, interval: number) => { + if (interval === 1000) tick = callback; + return 1 as unknown as ReturnType; + }) as typeof setInterval); + try { + await startServer(); await settleStartupHousekeeping(); + expect(registerStatusCardEvaluationWork).toHaveBeenCalledTimes(enabled ? 1 : 0); + expect(statusCardEvaluationService).toHaveBeenCalledTimes(enabled ? 1 : 0); + if (enabled) { + const service = vi.mocked(statusCardEvaluationService).mock.results.at(-1)!.value; + expect(service.run).toHaveBeenCalledOnce(); + expect(tick).toBeDefined(); + for (let i = 0; i < 3; i++) { tick!(); await settleStartupHousekeeping(); } + expect(service.run).toHaveBeenCalledOnce(); + } + expect(deliveryWorkErrors.filter(error => error.queue === DELIVERY_QUEUES.statusCardEvaluation)).toEqual([]); + } finally { timer.mockRestore(); } + }); + it.each([false, true])("registers response recovery and its shutdown subscription only when maintenance is enabled (%s)", async enabled => { loadConfigMock.mockReturnValue(buildTestConfig({ heartbeatSchedulerEnabled: enabled })); await startServer(); await settleStartupHousekeeping(); diff --git a/server/src/__tests__/status-card-evaluation-work.test.ts b/server/src/__tests__/status-card-evaluation-work.test.ts index 427ad3385eb..4f369cd76e9 100644 --- a/server/src/__tests__/status-card-evaluation-work.test.ts +++ b/server/src/__tests__/status-card-evaluation-work.test.ts @@ -1,7 +1,9 @@ +import { budgetService } from "../services/budgets.js"; +import { documentService } from "../services/documents.js"; import { randomUUID } from "node:crypto"; import { eq, sql } from "drizzle-orm"; import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; -import { activityLog, agentWakeupRequests, agents, companies, companyMemberships, createDb, documents, +import { activityLog, agentWakeupRequests, agents, approvals, budgetIncidents, budgetPolicies, costEvents, companies, companyMemberships, createDb, documents, heartbeatRuns, instanceSettings, issueComments, issues, statusCards, statusCardUpdates, documentRevisions } from "@paperclipai/db"; import { companySearchQuerySchema, statusCardRefreshPolicySchema } from "@paperclipai/shared"; import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js"; @@ -22,6 +24,7 @@ const support = external ? { supported: true } : await getEmbeddedPostgresTestSu let draining = false; let companyId: string, agentId: string; const errors: unknown[] = []; + let compiledQuery = companySearchQuerySchema.parse({ scope: "issues", status: ["blocked"] }); const actor = { agentId: null, userId: "board-user" }; const policy = (mode: "manual" | "reactive" | "interval" = "reactive") => statusCardRefreshPolicySchema.parse({ mode, debounceSeconds: 60, intervalMinutes: 15, @@ -29,7 +32,8 @@ const support = external ? { supported: true } : await getEmbeddedPostgresTestSu // Controlled provider boundary: assignment persists a queued wake, and tests // explicitly run the authorized summarizer writeback. No model/provider runs. const wakeup = vi.fn(async (id: string, options: { payload?: Record | null; reason?: string | null }) => { - return db.insert(agentWakeupRequests).values({ companyId, agentId: id, source: "assignment", + const [agent] = await db.select({ companyId: agents.companyId }).from(agents).where(eq(agents.id, id)); + return db.insert(agentWakeupRequests).values({ companyId: agent!.companyId, agentId: id, source: "assignment", reason: options.reason, payload: options.payload, status: "queued" }).returning(); }); @@ -38,7 +42,8 @@ const support = external ? { supported: true } : await getEmbeddedPostgresTestSu db = createDb(external ?? temporary!.connectionString, { maxConnections: 6, idleTimeoutSeconds: 0, maxLifetimeSeconds: 0 }); }, 30_000); beforeEach(async () => { - companyId = randomUUID(); agentId = randomUUID(); draining = false; errors.length = 0; wakeup.mockClear(); + companyId = randomUUID(); agentId = randomUUID(); draining = false; errors.length = 0; wakeup.mockReset(); + compiledQuery = companySearchQuerySchema.parse({ scope: "issues", status: ["blocked"] }); await db.insert(companies).values({ id: companyId, name: "Status", issuePrefix: `S${companyId.slice(0, 6)}` }); await db.insert(agents).values({ id: agentId, companyId, name: "Summarizer", role: "general", status: "idle", adapterType: "process", adapterConfig: {} }); await instanceSettingsService(db).updateExperimental({ enableStatusCards: true }); @@ -48,6 +53,7 @@ const support = external ? { supported: true } : await getEmbeddedPostgresTestSu afterEach(async () => { await coordinator?.stop(); coordinator = undefined; vi.useRealTimers(); vi.restoreAllMocks(); + await db.delete(budgetIncidents); await db.delete(budgetPolicies); await db.delete(approvals); await db.delete(costEvents); await db.delete(agentWakeupRequests); await db.delete(statusCardUpdates); await db.delete(statusCards); await db.delete(documentRevisions); await db.delete(documents); await db.delete(activityLog); await db.delete(issueComments); await db.delete(issues); await db.delete(heartbeatRuns); @@ -80,14 +86,14 @@ const support = external ? { supported: true } : await getEmbeddedPostgresTestSu await db.update(issues).set({ checkoutRunId: run!.id }).where(eq(issues.id, generationIssueId)); const writer = { agentId, runId: run!.id }; const cards = statusCardService(db); - if (compile) await cards.writeQuery(cardId, { queries: [companySearchQuerySchema.parse({ scope: "issues", status: ["blocked"] })], + if (compile) await cards.writeQuery(cardId, { queries: [compiledQuery], title: "Blockers", changeSummary: "Compiled", generationIssueId }, writer); await cards.writeSummary(cardId, { markdown: "Current blockers", changeSummary: "Updated", generationIssueId }, writer); await issueService(db).update(generationIssueId, { status: "done" }); } - async function card(mode: "manual" | "reactive" | "interval" = "reactive") { + async function card(mode: "manual" | "reactive" | "interval" = "reactive", refreshPolicy = policy(mode)) { const cards = statusCardService(db); - const created = await cards.create(companyId, { interestPrompt: "Blockers", titlePinned: false, agentId, refreshPolicy: policy(mode) }, actor); + const created = await cards.create(companyId, { interestPrompt: "Blockers", titlePinned: false, agentId, refreshPolicy }, actor); const compile = await cards.requestCompile(created.id, actor); await writeback(created.id, compile.generatingIssue.id, true); return cards.getById(created.id).then(row => row!); @@ -131,6 +137,19 @@ const support = external ? { supported: true } : await getEmbeddedPostgresTestSu expect(wakeup).not.toHaveBeenCalled(); expect(errors).toEqual([]); }); + it("restores overdue debt after the feature is disabled across its saved deadline", async () => { + await start(); const created = await card(); await quiet(); + await issueService(db).create(companyId, { title: "Disabled blocker", status: "blocked", priority: "medium" }); + await vi.waitFor(async () => expect((await statusCardService(db).getById(created.id))?.pendingChangeCount).toBeGreaterThan(0)); + const pending = (await statusCardService(db).getById(created.id))!; + await instanceSettingsService(db).updateExperimental({ enableStatusCards: false }); await quiet(); + expect(wakeup).not.toHaveBeenCalled(); + expect((await statusCardService(db).getById(created.id))!.nextEvalAt).toEqual(pending.nextEvalAt); + await instanceSettingsService(db).updateExperimental({ enableStatusCards: true }); + await vi.waitFor(() => expect(wakeup).toHaveBeenCalledOnce()); + await writeback(created.id, (await statusCardService(db).getById(created.id))!.generatingIssueId!); await quiet(); + }); + it("does not react to rolled-back or nested uncommitted card configuration", async () => { await start(); const created = await card("manual"); await quiet(); await expect(db.transaction(async tx => { @@ -169,6 +188,184 @@ const support = external ? { supported: true } : await getEmbeddedPostgresTestSu .toHaveLength(1); }); + it.each(["debounce", "hours", "rate", "budget"] as const)("restores pending %s eligibility after restart without advancing it or acting early", async gate => { + vi.setSystemTime(new Date("2026-11-02T18:00:00Z")); + const refreshPolicy = { ...policy(), ...(gate === "hours" ? { activeHours: { start: "09:00", end: "17:00", timezone: "UTC" } } : {}), + ...(gate === "rate" ? { maxUpdatesPerHour: 1 } : {}), ...(gate === "budget" ? { dailyTokenCap: 100 } : {}) }; + const created = await card("reactive", refreshPolicy); + if (gate === "rate" || gate === "budget") await db.insert(statusCardUpdates).values({ cardId: created.id, kind: "full", trigger: "reactive", + status: "ok", startedAt: new Date(Date.now() - (gate === "rate" ? 30 * 60_000 : 60_000)), finishedAt: new Date(), inputTokens: gate === "budget" ? 100 : 0 }); + await issueService(db).create(companyId, { title: "Saved blocker", status: "blocked", priority: "medium", createdByUserId: "board-user" }); + const service = statusCardEvaluationService(db, { wakeup }); + await service.run(new AbortController().signal); + const pending = (await statusCardService(db).getById(created.id))!; + expect(pending.pendingChangeCount).toBeGreaterThan(0); expect(pending.nextEvalAt!.getTime()).toBeGreaterThan(Date.now()); + const restarted = statusCardEvaluationService(db, { wakeup }); + await restarted.run(new AbortController().signal); + expect(await restarted.next()).toBe(pending.nextEvalAt!.getTime()); + expect((await statusCardService(db).getById(created.id))!.nextEvalAt).toEqual(pending.nextEvalAt); + vi.setSystemTime(pending.nextEvalAt!.getTime() - 1); await restarted.run(new AbortController().signal); + expect(wakeup).not.toHaveBeenCalled(); + vi.setSystemTime(pending.nextEvalAt!); await restarted.run(new AbortController().signal); + expect(wakeup).toHaveBeenCalledOnce(); + }); + + it("unions startup reconstruction with a scoped source change in another company", async () => { + const firstCompany = companyId; + const first = await card(); + await issueService(db).create(companyId, { title: "First blocker", status: "blocked", priority: "medium" }); + companyId = randomUUID(); agentId = randomUUID(); + await db.insert(companies).values({ id: companyId, name: "Other", issuePrefix: `X${companyId.slice(0, 6)}` }); + await db.insert(agents).values({ id: agentId, companyId, name: "Other summarizer", role: "general", status: "idle", adapterType: "process", adapterConfig: {} }); + const second = await card(); + await issueService(db).create(companyId, { title: "Second blocker", status: "blocked", priority: "medium" }); + const service = statusCardEvaluationService(db, { wakeup }); + service.changed(new Set([`changes:${companyId}`])); + await service.run(new AbortController().signal); + const cards = statusCardService(db); + expect((await cards.getById(first.id))?.companyId).toBe(firstCompany); + expect((await cards.getById(first.id))?.pendingChangeCount).toBeGreaterThan(0); + expect((await cards.getById(second.id))?.pendingChangeCount).toBeGreaterThan(0); + await vi.advanceTimersByTimeAsync(60_000); await service.run(new AbortController().signal); + expect(wakeup).toHaveBeenCalledTimes(2); + expect(new Set(await db.select().from(agentWakeupRequests).then(rows => rows.map(row => row.companyId)))) + .toEqual(new Set([firstCompany, companyId])); + }); + + it("keeps ignored issue activity quiet and reacts to human comments and document search membership", async () => { + await start(); await card(); await quiet(); + await issueService(db).create(companyId, { title: "Ignored todo", status: "todo", priority: "medium" }); + await quiet(); expect(wakeup).not.toHaveBeenCalled(); + compiledQuery = companySearchQuerySchema.parse({ scope: "all", q: "launch" }); + const searchCard = await card(); await quiet(); + const target = await issueService(db).create(companyId, { title: "Ordinary task", status: "todo", priority: "medium" }); + await quiet(); + await documentService(db).upsertIssueDocument({ issueId: target.id, key: "plan", format: "markdown", body: "launch evidence", createdByUserId: "board-user" }); + await vi.waitFor(async () => expect((await statusCardService(db).getById(searchCard.id))?.pendingChangeCount).toBeGreaterThan(0)); + await vi.advanceTimersByTimeAsync(60_000); await settled(); expect(wakeup).toHaveBeenCalledOnce(); + await writeback(searchCard.id, (await statusCardService(db).getById(searchCard.id))!.generatingIssueId!); await quiet(); + await issueService(db).addComment(target.id, "A human launch update", { userId: "board-user" }); + await vi.waitFor(async () => expect((await statusCardService(db).getById(searchCard.id))?.pendingChangeCount).toBeGreaterThan(0)); + }); + + it("retains relative query membership expiry across restart and emits the eventual removal", async () => { + const now = new Date(); + const target = await issueService(db).create(companyId, { title: "Expiring blocker", status: "blocked", priority: "medium" }); + await db.update(issues).set({ updatedAt: now }).where(eq(issues.id, target.id)); + compiledQuery = companySearchQuerySchema.parse({ scope: "issues", status: ["blocked"], updatedWithin: "1h" }); + const created = await card(); + const expires = now.getTime() + 3_600_001; + expect(created.nextEvalAt?.getTime()).toBe(expires); + const restarted = statusCardEvaluationService(db, { wakeup }); + await restarted.run(new AbortController().signal); + expect(await restarted.next()).toBe(expires); + vi.setSystemTime(expires - 1); await restarted.run(new AbortController().signal); expect(wakeup).not.toHaveBeenCalled(); + vi.setSystemTime(expires); await restarted.run(new AbortController().signal); + const pending = (await statusCardService(db).getById(created.id))!; + expect(pending.pendingChangeCount).toBe(1); expect(pending.nextEvalAt!.getTime()).toBe(expires + 60_000); + vi.setSystemTime(pending.nextEvalAt!); await restarted.run(new AbortController().signal); + expect(wakeup).toHaveBeenCalledOnce(); + }); + + it("uses expiry of an off-page candidate to refresh offset-ranked query membership", async () => { + const now = new Date(); + const tasks = await Promise.all((["critical", "high", "medium"] as const).map(priority => issueService(db).create(companyId, { + title: `${priority} blocker`, priority, status: "blocked", + }))); + await db.update(issues).set({ updatedAt: new Date(now.getTime() - 59 * 60_000) }).where(eq(issues.id, tasks[0]!.id)); + for (const task of tasks.slice(1)) await db.update(issues).set({ updatedAt: now }).where(eq(issues.id, task.id)); + compiledQuery = companySearchQuerySchema.parse({ scope: "issues", status: ["blocked"], updatedWithin: "1h", sort: "priority", limit: 1, offset: 1 }); + const created = await card(); + expect(Object.keys(created.fingerprint!)).toEqual([tasks[1]!.id]); + const expires = now.getTime() + 60_001; + expect(created.nextEvalAt?.getTime()).toBe(expires); + const service = statusCardEvaluationService(db, { wakeup }); + vi.setSystemTime(expires); await service.run(new AbortController().signal); + const pending = (await statusCardService(db).getById(created.id))!; + expect(pending.pendingChangeCount).toBe(2); expect(wakeup).not.toHaveBeenCalled(); + vi.setSystemTime(pending.nextEvalAt!); await service.run(new AbortController().signal); + expect(wakeup).toHaveBeenCalledOnce(); + const [update] = await db.select().from(statusCardUpdates).where(eq(statusCardUpdates.generationIssueId, (await statusCardService(db).getById(created.id))!.generatingIssueId!)); + expect(new Set(update!.changes.map(change => change.issueId))).toEqual(new Set([tasks[1]!.id, tasks[2]!.id])); + }); + + it("retains a scoped retry while another evaluator owns the card lock", async () => { + const created = await card(); + await issueService(db).create(companyId, { title: "Locked blocker", status: "blocked", priority: "medium" }); + const acquired = Promise.withResolvers(), release = Promise.withResolvers(); + const holder = db.transaction(async tx => { + await tx.execute(sql`select pg_advisory_xact_lock(hashtextextended(${created.id}, 0))`); + acquired.resolve(); await release.promise; + }); + await acquired.promise; + const service = statusCardEvaluationService(db, { wakeup }); + service.changed(new Set([`changes:${companyId}`])); + try { + await service.run(new AbortController().signal); + expect(await service.next()).toBeGreaterThan(Date.now()); + expect((await statusCardService(db).getById(created.id))!.nextEvalAt).toBeNull(); + expect(wakeup).not.toHaveBeenCalled(); + } finally { release.resolve(); await holder; } + await vi.advanceTimersByTimeAsync(30_000); await service.run(new AbortController().signal); + const pending = (await statusCardService(db).getById(created.id))!; + expect(pending.pendingChangeCount).toBeGreaterThan(0); + vi.setSystemTime(pending.nextEvalAt!); await service.run(new AbortController().signal); + expect(wakeup).toHaveBeenCalledOnce(); + }); + + it("retains the durable five-minute retry after evaluation failure across restart and drain", async () => { + const created = await card(); + await db.update(statusCards).set({ queries: [{ scope: "invalid" } as never], nextEvalAt: new Date() }).where(eq(statusCards.id, created.id)); + const service = statusCardEvaluationService(db, { wakeup }); + await service.run(new AbortController().signal); + const failed = (await statusCardService(db).getById(created.id))!; + const retryAt = failed.nextEvalAt!.getTime(); + expect(retryAt).toBeGreaterThan(Date.now() + 299_000); expect(wakeup).not.toHaveBeenCalled(); + await start(); expect(coordinator!.nextWakeAt()).toBe(retryAt); + draining = true; await vi.advanceTimersByTimeAsync(5 * 60_000); await settled(); expect(wakeup).not.toHaveBeenCalled(); + // Repair the controlled malformed stored-query fault; no tested result is precreated. + await db.update(statusCards).set({ queries: [compiledQuery] }).where(eq(statusCards.id, created.id)); + draining = false; await vi.advanceTimersByTimeAsync(30_000); await quiet(); + expect((await statusCardService(db).getById(created.id))!.nextEvalAt).toBeNull(); + }); + + it("cancels failed assignments without rerunning a completed generation and settles after retry", async () => { + await start(); const created = await card(); await quiet(); + wakeup.mockRejectedValueOnce(new Error("assignment unavailable")); + await issueService(db).create(companyId, { title: "Assignment blocker", status: "blocked", priority: "medium" }); + await vi.waitFor(async () => expect((await statusCardService(db).getById(created.id))?.pendingChangeCount).toBeGreaterThan(0)); + await vi.advanceTimersByTimeAsync(60_000); await settled(); + const failed = (await statusCardService(db).getById(created.id))!; + expect(failed).toMatchObject({ state: "error", generatingIssueId: null, nextEvalAt: null }); + expect(errors.length).toBeGreaterThan(0); + const history = await db.select().from(statusCardUpdates).where(eq(statusCardUpdates.cardId, created.id)); + const ledger = history.find(row => row.status === "failed")!; + expect(ledger.finishedAt).toBeInstanceOf(Date); + expect((await db.select().from(issues).where(eq(issues.id, ledger.generationIssueId!)))[0]?.status).toBe("cancelled"); + const count = wakeup.mock.calls.length; await quiet(); expect(wakeup).toHaveBeenCalledTimes(count); + // An explicit ordinary refresh is still the user's supported retry action. + const retry = await statusCardService(db).requestRefresh(created.id, { trigger: "manual", actor }); + expect(retry.generatingIssue).toBeTruthy(); await writeback(created.id, retry.generatingIssue!.id); await quiet(); + expect((await statusCardService(db).getById(created.id))!.state).toBe("active"); + }); + + it("budget-owned company resume reactivates reactive work committed while paused", async () => { + await start(); const created = await card(); await quiet(); + const budgets = budgetService(db); + await budgets.upsertPolicy(companyId, { scopeType: "company", scopeId: companyId, amount: 100, hardStopEnabled: true, notifyEnabled: false }, "board-user"); + const [cost] = await db.insert(costEvents).values({ companyId, agentId, provider: "openai", biller: "openai", billingType: "metered_api", model: "test", + costCents: 200, occurredAt: new Date() }).returning(); + await budgets.evaluateCostEvent(cost!); + expect((await db.select().from(companies).where(eq(companies.id, companyId)))[0]?.status).toBe("paused"); + await quiet(); + await issueService(db).create(companyId, { title: "Paused blocker", status: "blocked", priority: "medium" }); await quiet(); + expect(wakeup).not.toHaveBeenCalled(); + await budgets.upsertPolicy(companyId, { scopeType: "company", scopeId: companyId, amount: 1000 }, "board-user"); + expect((await db.select().from(companies).where(eq(companies.id, companyId)))[0]?.status).toBe("active"); + await vi.waitFor(async () => expect((await statusCardService(db).getById(created.id))?.pendingChangeCount).toBeGreaterThan(0)); + await vi.advanceTimersByTimeAsync(60_000); await settled(); expect(wakeup).toHaveBeenCalledOnce(); + }); + it("preserves the first write during drain until resume", async () => { await start(); const created = await card("manual"); await quiet(); draining = true; diff --git a/server/src/__tests__/status-card-notifications.test.ts b/server/src/__tests__/status-card-notifications.test.ts new file mode 100644 index 00000000000..0566ebdd394 --- /dev/null +++ b/server/src/__tests__/status-card-notifications.test.ts @@ -0,0 +1,31 @@ +import { expect, it, vi } from "vitest"; +import { installDatabaseWorkSignals, subscribeDatabaseWork } from "../../../packages/db/src/work-signals.js"; +import { DELIVERY_QUEUES } from "../services/delivery-work-notifications.js"; +import { notifyExecutionRunWork, writeExecutionControl } from "../services/execution-control-notifications.js"; + +it.each([false, true])("keeps execution queue consumers intact and emits only a known status-card company scope (%s)", async scoped => { + const owner = installDatabaseWorkSignals({ execute: vi.fn(async () => []), transaction: async (fn: (tx: object) => Promise) => + fn({ execute: async () => [{ xid: "1" }] }) }); + const queues = [DELIVERY_QUEUES.executionStatus, DELIVERY_QUEUES.executionFinalization, DELIVERY_QUEUES.executionReplacement, + DELIVERY_QUEUES.executionDisposition, DELIVERY_QUEUES.issueMonitors]; + const settled: Array<{ queue: string; hints?: string[] }> = []; + const unsubscribe = [...queues, DELIVERY_QUEUES.statusCardEvaluation, DELIVERY_QUEUES.workspaceRecovery].map(queue => subscribeDatabaseWork(owner, queue, (event, hints) => { + if (event === "settled") settled.push({ queue, hints: hints ? [...hints] : undefined }); + })); + try { + const scope = scoped ? { companyId: "changed-company", runId: "changed-run" } : undefined; + await owner.transaction(async tx => { + await notifyExecutionRunWork(tx, scope); + expect(settled).toEqual([]); + }); + expect(new Set(settled.map(row => row.queue))).toEqual(new Set([...queues, ...(scoped ? [DELIVERY_QUEUES.statusCardEvaluation] : [])])); + expect(settled.find(row => row.queue === DELIVERY_QUEUES.statusCardEvaluation)?.hints) + .toEqual(scoped ? ["changes:changed-company"] : undefined); + settled.length = 0; + const write = vi.fn(async () => "written"); + expect(await writeExecutionControl(owner as never, DELIVERY_QUEUES.workspaceRecovery, write, scope)).toBe("written"); + expect(write).toHaveBeenCalledOnce(); + expect(new Set(settled.map(row => row.queue))).toEqual(new Set([DELIVERY_QUEUES.issueMonitors, DELIVERY_QUEUES.workspaceRecovery, + ...(scoped ? [DELIVERY_QUEUES.statusCardEvaluation] : [])])); + } finally { for (const stop of unsubscribe) stop(); } +}); diff --git a/server/src/modules/wake-queue/adapters/postgres.ts b/server/src/modules/wake-queue/adapters/postgres.ts index b888ee51fb3..2b7f0a16e26 100644 --- a/server/src/modules/wake-queue/adapters/postgres.ts +++ b/server/src/modules/wake-queue/adapters/postgres.ts @@ -1,3 +1,4 @@ +import { notifyStatusCardIssueChange } from "../../../services/status-card-notifications.js"; import { notifyExternalConversationSettlement } from "../../../services/external-conversation-settlement-notifications.js"; import { notifyNativeStatusReassessment } from "../../../services/native-status-reassessment-notifications.js"; import { notifyUnsafeWorkspaceExportRecovery } from "../../../services/unsafe-workspace-export-notifications.js"; @@ -610,6 +611,7 @@ function buildTransaction(tx: Db, deps: WakeQueuePostgresAdapterDeps, db: Db, ru }, async queueReviewParticipantRecoveryRun({ companyId, issue, finishingRun, recoveryAgent, sessionBefore, now }) { + await notifyStatusCardIssueChange(tx, companyId); const executionState = parseIssueExecutionState(issue.executionState); const wakeupRequest = await tx .insert(agentWakeupRequests) diff --git a/server/src/modules/wake-queue/adapters/queued-comment-postgres.ts b/server/src/modules/wake-queue/adapters/queued-comment-postgres.ts index 184c799f82d..a10419f0c41 100644 --- a/server/src/modules/wake-queue/adapters/queued-comment-postgres.ts +++ b/server/src/modules/wake-queue/adapters/queued-comment-postgres.ts @@ -1,3 +1,4 @@ +import { notifyStatusCardIssueChange } from "../../../services/status-card-notifications.js"; import { notifyExternalAgentSettlement } from "../../external-agents/index.js"; import { notifyNativeStatusReassessment } from "../../../services/native-status-reassessment-notifications.js"; import { notifyUnsafeWorkspaceExportRecovery } from "../../../services/unsafe-workspace-export-notifications.js"; @@ -55,6 +56,7 @@ export type QueuedCommentQueuePostgresAdapterDeps = { function buildTransaction(tx: Db, companyId: string, issueId: string, deps: QueuedCommentQueuePostgresAdapterDeps): QueuedCommentQueueTransaction { return { async updateCommentBody({ issueId, commentId, body, updatedAt }) { + await notifyStatusCardIssueChange(tx, companyId); const updated = await tx .update(issueComments) .set({ body, updatedAt }) @@ -65,6 +67,7 @@ function buildTransaction(tx: Db, companyId: string, issueId: string, deps: Queu }, async touchIssueUpdatedAt({ issueId, updatedAt }) { + await notifyStatusCardIssueChange(tx, companyId); await tx.update(issues).set({ updatedAt }).where(and(eq(issues.id, issueId), eq(issues.companyId, companyId))); }, diff --git a/server/src/routes/execution-workspaces.ts b/server/src/routes/execution-workspaces.ts index e87fbbeea6b..7be8e505a94 100644 --- a/server/src/routes/execution-workspaces.ts +++ b/server/src/routes/execution-workspaces.ts @@ -1,3 +1,4 @@ +import { notifyStatusCardIssueChange } from "../services/status-card-notifications.js"; import { removePluginWorkspace } from "../services/plugin-workspaces.js"; import { canActorReadWorkspaceOperation } from "../services/heartbeat-run-privacy.js"; import { canActorReadExecutionWorkspace, executionWorkspaceReadSqlCondition } from "../services/authorization.js"; @@ -1245,7 +1246,9 @@ export function executionWorkspaceRoutes(db: Db, opts: { pluginWorkerManager?: P await runtimeLeases.release({ executionWorkspaceId: existing.id, force: true }); if (existing.mode === "shared_workspace") { - await db + await db.transaction(async tx => { + await notifyStatusCardIssueChange(tx, existing.companyId); + await tx .update(issues) .set({ executionWorkspaceId: null, @@ -1257,6 +1260,7 @@ export function executionWorkspaceRoutes(db: Db, opts: { pluginWorkerManager?: P eq(issues.executionWorkspaceId, existing.id), ), ); + }); } try { diff --git a/server/src/services/agent-conversations.ts b/server/src/services/agent-conversations.ts index af44a0355d8..6dc783568e2 100644 --- a/server/src/services/agent-conversations.ts +++ b/server/src/services/agent-conversations.ts @@ -1,3 +1,4 @@ +import { notifyStatusCardIssueChange } from "./status-card-notifications.js"; import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js"; import { notifyUnsafeWorkspaceExportRecovery } from "./unsafe-workspace-export-notifications.js"; import { FAST_RESPONSE_AGENT_GUIDANCE, fastResponseHistoryBody } from "@paperclipai/shared"; @@ -269,6 +270,7 @@ export async function prepareConversationTurn( context.conversationReplayThroughCommentId = replayThrough?.id ?? commentId; } } + await notifyStatusCardIssueChange(tx, issue.companyId); await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors); await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers); await tx @@ -389,6 +391,7 @@ export async function settleConversationTurn( issue.conversationState === conversationState ) return true; + await notifyStatusCardIssueChange(tx, issue.companyId); await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors); await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers); if (status === "in_review") await notifyUnsafeWorkspaceExportRecovery(tx, { companyId: issue.companyId, issueId: issue.id }); diff --git a/server/src/services/budgets.ts b/server/src/services/budgets.ts index 6777124ee90..baf667c43c9 100644 --- a/server/src/services/budgets.ts +++ b/server/src/services/budgets.ts @@ -1,3 +1,4 @@ +import { notifyStatusCardIssueChange } from "./status-card-notifications.js"; import { notifyApprovalWork } from "./approval-work-notifications.js"; import { notifyCompanyRetentionWork } from "./retention-work-notifications.js"; import { addCents, compareCents, normalizeCents } from "@paperclipai/shared"; @@ -301,6 +302,7 @@ export function budgetServiceInTransaction(db: Db, publications: ActivityPublica const scope = await resolveScopeRecord(db, "company", policy.scopeId); if (scope.paused) return; await notifyCompanyRetentionWork(db, policy.companyId); + await notifyStatusCardIssueChange(db, policy.companyId); await notifyDeliveryWork(db, DELIVERY_QUEUES.agentTimers); await notifyDeliveryWork(db, DELIVERY_QUEUES.routineSchedules); await notifyDeliveryWork(db, DELIVERY_QUEUES.issueMonitors); @@ -361,6 +363,7 @@ export function budgetServiceInTransaction(db: Db, publications: ActivityPublica const scope = await resolveScopeRecord(db, "company", policy.scopeId); if (scope.pauseReason !== "budget") return; await notifyCompanyRetentionWork(db, policy.companyId); + await notifyStatusCardIssueChange(db, policy.companyId); await notifyDeliveryWork(db, DELIVERY_QUEUES.agentTimers); await notifyDeliveryWork(db, DELIVERY_QUEUES.routineSchedules); await notifyDeliveryWork(db, DELIVERY_QUEUES.issueMonitors); diff --git a/server/src/services/chat-channels/telegram/attachment-failure.ts b/server/src/services/chat-channels/telegram/attachment-failure.ts index 58d5d4a2677..49e2497a055 100644 --- a/server/src/services/chat-channels/telegram/attachment-failure.ts +++ b/server/src/services/chat-channels/telegram/attachment-failure.ts @@ -1,3 +1,4 @@ +import { notifyStatusCardIssueChange } from "../../status-card-notifications.js"; import { createProviderEffects, type ProviderEffectTarget } from "../provider-effects.js"; import type { EndpointRow, RuntimeContext, InboundRuntimeContext } from "../runtime-types.js"; @@ -82,6 +83,7 @@ export function createTelegramAttachmentFailure(db: Db, options: TelegramAttachm // Paperclip-visible failure. The message link stays authoritative for // exact provider redelivery, while no agent wakeup is enqueued for an // empty turn that contains no usable bytes. + await notifyStatusCardIssueChange(tx, input.endpoint.companyId); await tx .update(issueComments) .set({ body: visibleFailure, updatedAt: new Date() }) diff --git a/server/src/services/company-search.ts b/server/src/services/company-search.ts index bcf2a550d3f..e6595beebd8 100644 --- a/server/src/services/company-search.ts +++ b/server/src/services/company-search.ts @@ -548,7 +548,7 @@ export function companySearchService(db: Db) { search: async ( companyId: string, query: CompanySearchQuery, - options?: { issueReadCondition?: SQL; projectReadCondition?: SQL }, + options?: { issueReadCondition?: SQL; projectReadCondition?: SQL; onNextMembershipChange?: (at: Date | null) => void }, ): Promise => { const taskSearch = parseTaskSearch(query.q); const normalizedQuery = taskSearch.normalizedQuery; @@ -559,6 +559,7 @@ export function companySearchService(db: Db) { const limit = query.limit; const offset = query.offset; if (!hasSearchText && !issueOnlyFiltersActive(query)) { + options?.onNextMembershipChange?.(null); return { query: query.q, normalizedQuery, @@ -727,6 +728,13 @@ export function companySearchService(db: Db) { } } + const membershipWindow = companySearchUpdatedWithinMs(query.updatedWithin); + if (options?.onNextMembershipChange && membershipWindow !== null && scopeIncludesIssues(scope)) { + // Includes rows outside the visible page: their expiry can move an + // offset/priority-ranked result into the watched set without a write. + branches.push(sql`SELECT 'membership-expiry' AS kind, min(m.updated_at)::text AS value, + 0::int AS count ${countTail} FROM matched m ${branchWhere([scopeCond, ...facetsAll])}`); + } const resultRows = await db.execute(sql` ${taskSearchCtes(companyId, taskSearch, scope !== "issues", and(...issueFilters), options?.issueReadCondition)} ${sql.join(branches, sql` UNION ALL `)} @@ -738,6 +746,7 @@ export function companySearchService(db: Db) { totals: { current: 0, unfiltered: 0, omit: {} }, }; const issueRowsRaw: IssueSearchRow[] = []; + let nextMembershipChange: Date | null = null; for (const row of resultRows) { if (row.kind === "result") { issueRowsRaw.push({ @@ -762,6 +771,9 @@ export function companySearchService(db: Db) { }); continue; } + if (row.kind === "membership-expiry" && row.value && membershipWindow !== null) { + nextMembershipChange = new Date(new Date(row.value).getTime() + membershipWindow + 1); + } const count = Number(row.count ?? 0); if (row.kind === "type:issue") aggregates.typeCounts.issue = count; else if (row.kind === "type:comment") aggregates.typeCounts.comment = count; @@ -782,6 +794,7 @@ export function companySearchService(db: Db) { aggregates.totals.omit[row.kind.slice("total:omit:".length) as CompanySearchIssueFilterKey] = count; } } + options?.onNextMembershipChange?.(nextMembershipChange); return { rows: await enrichIssueSnippets(issueRowsRaw), aggregates }; } diff --git a/server/src/services/documents.ts b/server/src/services/documents.ts index 493c495e007..79e8e14309c 100644 --- a/server/src/services/documents.ts +++ b/server/src/services/documents.ts @@ -1,3 +1,4 @@ +import { notifyStatusCardIssueChange } from "./status-card-notifications.js"; import { randomUUID } from "node:crypto"; import { and, asc, desc, eq, sql } from "drizzle-orm"; import type { Db } from "@paperclipai/db"; @@ -238,6 +239,7 @@ export function documentService(db: Db) { for (let attempt = 0; attempt < maxAttempts; attempt += 1) { try { return await db.transaction(async (tx) => { + await notifyStatusCardIssueChange(tx, issue.companyId); const now = new Date(); const existing = await tx .select({ @@ -592,6 +594,7 @@ export function documentService(db: Db) { }); } await db.transaction(async (tx) => { + for (const companyId of new Set(rows.map(row => row.companyId))) await notifyStatusCardIssueChange(tx, companyId); await insertRowsInChunks(tx, documents, documentRows); await insertRowsInChunks(tx, documentRevisions, revisionRows); await insertRowsInChunks(tx, issueDocuments, linkRows); @@ -662,6 +665,7 @@ export function documentService(db: Db) { }) .returning(); + await notifyStatusCardIssueChange(tx, existing.companyId); await tx .update(documents) .set({ @@ -818,6 +822,7 @@ export function documentService(db: Db) { } await tx.delete(issueDocuments).where(eq(issueDocuments.documentId, existing.id)); + await notifyStatusCardIssueChange(tx, existing.companyId); await tx.delete(documents).where(eq(documents.id, existing.id)); return { diff --git a/server/src/services/execution-workspaces.ts b/server/src/services/execution-workspaces.ts index caaae186ad8..15ff32f292e 100644 --- a/server/src/services/execution-workspaces.ts +++ b/server/src/services/execution-workspaces.ts @@ -2530,13 +2530,16 @@ export function executionWorkspaceService(db: Db, opts: ExecutionWorkspaceServic forceWorktreeRemoval: false, }); if (cleanup.cleaned && workspace.mode === "shared_workspace") { - await db + await db.transaction(async tx => { + await notifyStatusCardIssueChange(tx, workspace.companyId); + await tx .update(issues) .set({ executionWorkspaceId: null, updatedAt: now() }) .where(and( eq(issues.companyId, workspace.companyId), eq(issues.executionWorkspaceId, workspace.id), )); + }); } const cleanupReason = [ISSUE_TERMINAL_WORKSPACE_CLEANUP_REASON, ...cleanup.warnings].join(" | "); if (!cleanup.cleaned || cleanup.warnings.length > 0) { diff --git a/server/src/services/heartbeat/queue.ts b/server/src/services/heartbeat/queue.ts index a41b486b647..4d1e423d014 100644 --- a/server/src/services/heartbeat/queue.ts +++ b/server/src/services/heartbeat/queue.ts @@ -1,3 +1,4 @@ +import { notifyStatusCardIssueChange } from "../status-card-notifications.js"; import { DELIVERY_QUEUES, notifyDeliveryWork } from "../delivery-work-notifications.js"; import { notifyExternalAgentSettlement } from "../../modules/external-agents/index.js"; import { notifyNativeStatusReassessment } from "../native-status-reassessment-notifications.js"; @@ -1267,6 +1268,7 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende } async function bindClaimedIssueExecution(tx: Db, ownsIssue: boolean, claimedRun: typeof heartbeatRuns.$inferSelect | null | undefined) { if (!claimedRun || !issueId || !ownsIssue) return; + await notifyStatusCardIssueChange(tx, run.companyId); await tx.update(issues).set({ executionRunId: claimedRun.id, executionAgentNameKey: normalizeAgentNameKey(agent.name), @@ -3758,6 +3760,7 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende .from(agents) .where(eq(agents.id, legacyRun.agentId)) .then((rows) => rows[0] ?? null); + await notifyStatusCardIssueChange(tx, issue.companyId); await tx .update(issues) .set({ diff --git a/server/src/services/heartbeat/scheduling.ts b/server/src/services/heartbeat/scheduling.ts index dece8b41ae9..b0e7f308297 100644 --- a/server/src/services/heartbeat/scheduling.ts +++ b/server/src/services/heartbeat/scheduling.ts @@ -1,3 +1,4 @@ +import { notifyStatusCardIssueChange } from "../status-card-notifications.js"; import { DELIVERY_QUEUES, notifyDeliveryWork } from "../delivery-work-notifications.js"; import { notifyIssueReviewState } from "../issue-review-state-notifications.js"; import { insertIssueComment } from "../issue-comment-writes.js"; @@ -595,6 +596,7 @@ export function createHeartbeatScheduling( try { await db.transaction(async (tx) => { await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors); + await notifyStatusCardIssueChange(tx, claimed.companyId); const released = await tx.update(issues) .set({ monitorWakeRequestedAt: null, updatedAt: input.now }) .where(issueMonitorClaimCondition(claimed)).returning({ id: issues.id }); @@ -628,6 +630,7 @@ export function createHeartbeatScheduling( // failed, and never clobber a monitor that was replaced in the meantime. await db.transaction(async tx => { await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors); + await notifyStatusCardIssueChange(tx, claimed.companyId); await tx.update(issues).set({ monitorWakeRequestedAt: null, updatedAt: input.now }) .where(issueMonitorClaimCondition(claimed)); }); @@ -827,6 +830,7 @@ export function createHeartbeatScheduling( } const consumed = await db.transaction(async tx => { + await notifyStatusCardIssueChange(tx, claimed.companyId); await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors); return tx.update(issues) .set({ @@ -977,6 +981,7 @@ export function createHeartbeatScheduling( const staleClaimThreshold = new Date(now.getTime() - 5 * 60 * 1000); const claimed = await db.transaction(async (tx) => { + await notifyStatusCardIssueChange(tx, issue.companyId); await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors); const [updated] = await tx .update(issues) @@ -1050,6 +1055,7 @@ export function createHeartbeatScheduling( for (const due of dueMonitors) { if (!canContinue()) break; const claimed = await db.transaction(async (tx) => { + await notifyStatusCardIssueChange(tx, due.companyId); await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors); const [updated] = await tx .update(issues) diff --git a/server/src/services/issue-thread-interactions.ts b/server/src/services/issue-thread-interactions.ts index 59c14618342..f4d5abbc5e2 100644 --- a/server/src/services/issue-thread-interactions.ts +++ b/server/src/services/issue-thread-interactions.ts @@ -1,3 +1,4 @@ +import { notifyStatusCardIssueChange } from "./status-card-notifications.js"; import { notifyNativeStatusReassessment } from "./native-status-reassessment-notifications.js"; import { notifyIssueReviewState } from "./issue-review-state-notifications.js"; import { notifyChatConfirmationResolution } from "./chat-confirmation-notifications.js"; @@ -376,7 +377,7 @@ type ResolvedInteractionResult = { }; type IssueThreadInteractionRow = typeof issueThreadInteractions.$inferSelect; -type IssueTouchDb = Pick; +type IssueTouchDb = Pick; function isNativeCompletionReview( row: Pick, @@ -801,11 +802,11 @@ function hydrateInteraction( } } -async function touchIssue(db: IssueTouchDb, issueId: string) { - await db - .update(issues) - .set({ updatedAt: new Date() }) - .where(eq(issues.id, issueId)); +async function touchIssue(db: IssueTouchDb, issueId: string, companyId: string) { + await db.transaction(async tx => { + await notifyStatusCardIssueChange(tx, companyId); + await tx.update(issues).set({ updatedAt: new Date() }).where(eq(issues.id, issueId)); + }); } function isTerminalIssueStatus(status: string) { @@ -2013,7 +2014,7 @@ async function expireStaleRequestConfirmationTarget( "Interaction has already been resolved", ); } - await touchIssue(tx, args.row.issueId); + await touchIssue(tx, args.row.issueId, args.row.companyId); const expired = hydrateInteraction(updated); await enqueueTerminalIssueInteractionChatPublications(tx, expired); return expired; @@ -2468,7 +2469,7 @@ export function issueThreadInteractionService( }; } } else { - await touchIssue(tx, args.issue.id); + await touchIssue(tx, args.issue.id, args.issue.companyId); } // Snapshot the continuation target only after every lifecycle mutation @@ -2682,7 +2683,7 @@ export function issueThreadInteractionService( tx, ); } else { - await touchIssue(tx, args.issue.id); + await touchIssue(tx, args.issue.id, args.issue.companyId); } await enqueueTerminalIssueInteractionChatPublications( tx as unknown as Db, @@ -2864,7 +2865,7 @@ export function issueThreadInteractionService( await enqueueTerminalIssueInteractionChatPublications(tx as unknown as Db, hydrateInteraction(previous)); } } - await touchIssue(tx, issue.id); + await touchIssue(tx, issue.id, issue.companyId); inserted = true; await enqueueIssueInteractionChatPublications(tx as unknown as Db, hydrateInteraction(row)); await notifyDeliveryWork(tx, DELIVERY_QUEUES.connection); @@ -2908,7 +2909,7 @@ export function issueThreadInteractionService( ) .returning(); if (!updated) throw interactionAlreadyResolvedError(); - await touchIssue(db, issue.id); + await touchIssue(db, issue.id, issue.companyId); return hydrateInteraction(updated) as ConnectionIntentInteraction; }, resolveConnectionIntent: async ( @@ -2972,7 +2973,7 @@ export function issueThreadInteractionService( return row; }); if (!updated) throw interactionAlreadyResolvedError(); - await touchIssue(db, issue.id); + await touchIssue(db, issue.id, issue.companyId); const interaction = hydrateInteraction( updated, ) as ConnectionIntentInteraction; @@ -3351,7 +3352,7 @@ export function issueThreadInteractionService( }) .where(eq(issueThreadInteractions.id, current.id)) .returning(); - await touchIssue(tx, issue.id); + await touchIssue(tx, issue.id, issue.companyId); return row; }); return hydrateInteraction(updated); @@ -3414,10 +3415,8 @@ export function issueThreadInteractionService( if (updated) cancelled.push(updated); } - for (const issueId of new Set( - cancelled.map((interaction) => interaction.issueId), - )) { - await touchIssue(db, issueId); + for (const [issueId, companyId] of new Map(cancelled.map(interaction => [interaction.issueId, interaction.companyId]))) { + await touchIssue(db, issueId, companyId); } await emitResolvedInteractionsTelemetry(db, cancelled); return cancelled; @@ -3506,10 +3505,8 @@ export function issueThreadInteractionService( } if (expired.length > 0) { - for (const issueId of new Set( - expired.map((interaction) => interaction.issueId), - )) { - await touchIssue(db, issueId); + for (const [issueId, companyId] of new Map(expired.map(interaction => [interaction.issueId, interaction.companyId]))) { + await touchIssue(db, issueId, companyId); } await emitResolvedInteractionsTelemetry(db, expired); } @@ -4052,7 +4049,7 @@ export function issueThreadInteractionService( return interaction; } - await touchIssue(db, issue.id); + await touchIssue(db, issue.id, issue.companyId); if (superseded.length > 0) { await emitResolvedInteractionsTelemetry( db, @@ -4357,7 +4354,7 @@ export function issueThreadInteractionService( .where(eq(issueThreadInteractions.id, interactionId)) .returning(); - await touchIssue(tx, issue.id); + await touchIssue(tx, issue.id, issue.companyId); await mutationOptions.afterResolveInTransaction?.(tx, hydrateInteraction(updated)); await enqueueTerminalIssueInteractionChatPublications(tx as unknown as Db, hydrateInteraction(updated)); current.status = updated.status; @@ -4549,7 +4546,7 @@ export function issueThreadInteractionService( throw interactionAlreadyResolvedError(); } - await touchIssue(tx, issue.id); + await touchIssue(tx, issue.id, issue.companyId); await mutationOptions.afterResolveInTransaction?.(tx, hydrateInteraction(updated)); if (complete) await enqueueTerminalIssueInteractionChatPublications(tx as unknown as Db, hydrateInteraction(updated)); return { @@ -4622,7 +4619,7 @@ export function issueThreadInteractionService( throw interactionAlreadyResolvedError(); } - await touchIssue(db, issue.id); + await touchIssue(db, issue.id, issue.companyId); const rejected = hydrateInteraction(updated); await emitInteractionResolvedTelemetry(db, rejected); return rejected; @@ -4720,7 +4717,7 @@ export function issueThreadInteractionService( } if (expired.length > 0) { - await touchIssue(db, issue.id); + await touchIssue(db, issue.id, issue.companyId); await emitResolvedInteractionsTelemetry(db, expired); } return expired; @@ -4952,7 +4949,7 @@ export function issueThreadInteractionService( } if (expired.length > 0) { - await touchIssue(db, issue.id); + await touchIssue(db, issue.id, issue.companyId); await emitResolvedInteractionsTelemetry(db, expired); } return expired; @@ -5049,7 +5046,7 @@ export function issueThreadInteractionService( } if (expired.length > 0) { - await touchIssue(db, issue.id); + await touchIssue(db, issue.id, issue.companyId); await emitResolvedInteractionsTelemetry(db, expired); } return expired; @@ -5189,7 +5186,7 @@ export function issueThreadInteractionService( if (updated) expired.push(hydrateInteraction(updated)); } if (expired.length > 0) { - await touchIssue(db, issue.id); + await touchIssue(db, issue.id, issue.companyId); await emitResolvedInteractionsTelemetry(db, expired); } return expired; @@ -5294,7 +5291,7 @@ export function issueThreadInteractionService( return row; }); - await touchIssue(db, issue.id); + await touchIssue(db, issue.id, issue.companyId); const withdrawn = hydrateInteraction(updated); await emitInteractionResolvedTelemetry(db, withdrawn); return withdrawn; @@ -5417,7 +5414,7 @@ export function issueThreadInteractionService( return row; }); - await touchIssue(db, issue.id); + await touchIssue(db, issue.id, issue.companyId); const answered = hydrateInteraction(updated); await emitInteractionResolvedTelemetry(db, answered); return answered; @@ -5553,7 +5550,7 @@ export function issueThreadInteractionService( return row; }); - await touchIssue(db, issue.id); + await touchIssue(db, issue.id, issue.companyId); const skipped = hydrateInteraction(updated); await emitInteractionResolvedTelemetry(db, skipped); return skipped; @@ -5630,7 +5627,7 @@ export function issueThreadInteractionService( return row; }); - await touchIssue(db, issue.id); + await touchIssue(db, issue.id, issue.companyId); const cancelled = hydrateInteraction(updated); await emitInteractionResolvedTelemetry(db, cancelled); return cancelled; diff --git a/server/src/services/issues.ts b/server/src/services/issues.ts index 8787ac867a8..f58d0e62789 100644 --- a/server/src/services/issues.ts +++ b/server/src/services/issues.ts @@ -7730,6 +7730,7 @@ export function issueService(db: Db) { } const now = new Date(); + await notifyStatusCardIssueChange(tx, lockedIssue.companyId); const adopted = await tx .update(issues) .set({ @@ -7783,13 +7784,14 @@ export function issueService(db: Db) { sql`select ${heartbeatRuns.id} from ${heartbeatRuns} where ${heartbeatRuns.id} = ${input.actorRunId} for update`, ); const actorRun = await tx - .select({ status: heartbeatRuns.status }) + .select({ status: heartbeatRuns.status, companyId: heartbeatRuns.companyId }) .from(heartbeatRuns) .where(eq(heartbeatRuns.id, input.actorRunId)) .then((rows) => rows[0] ?? null); if (!actorRun || TERMINAL_HEARTBEAT_RUN_STATUSES.has(actorRun.status)) return null; + await notifyStatusCardIssueChange(tx, actorRun.companyId); const now = new Date(); const adopted = await tx .update(issues) @@ -10892,6 +10894,7 @@ export function issueService(db: Db) { } } + await notifyStatusCardIssueChange(tx, companyId); await insertRowsInChunks(tx, issues, issueRows); await insertRowsInChunks(tx, issueLabels, labelRows); }); @@ -10908,6 +10911,7 @@ export function issueService(db: Db) { const censorUsernameInLogs = (await instanceSettings.getGeneral()) .censorUsernameInLogs; await db.transaction(async (tx) => { + for (const companyId of new Set(rows.map(row => row.companyId))) await notifyStatusCardIssueChange(tx, companyId); const commentRows = rows.map((row) => { const createdAt = row.createdAt ? new Date(row.createdAt) : null; return { @@ -11922,7 +11926,9 @@ export function issueService(db: Db) { ); if (settings?.environmentId !== environmentId) continue; - await db + await db.transaction(async tx => { + await notifyStatusCardIssueChange(tx, companyId); + await tx .update(issues) .set({ executionWorkspaceSettings: { @@ -11932,6 +11938,7 @@ export function issueService(db: Db) { updatedAt: new Date(), }) .where(eq(issues.id, row.id)); + }); cleared += 1; } @@ -12088,6 +12095,7 @@ export function issueService(db: Db) { ) : isNull(issues.executionRunId); const updated = await db.transaction(async tx => { + await notifyStatusCardIssueChange(tx, issueCompany.companyId); await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors); await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers); return tx @@ -12140,7 +12148,9 @@ export function issueService(db: Db) { current.executionRunId === checkoutRunId) && checkoutRunId ) { - const adopted = await db + const adopted = await db.transaction(async tx => { + await notifyStatusCardIssueChange(tx, issueCompany.companyId); + return tx .update(issues) .set({ checkoutRunId, @@ -12161,6 +12171,7 @@ export function issueService(db: Db) { ) .returning() .then((rows) => rows[0] ?? null); + }); if (adopted) return adopted; } @@ -12217,6 +12228,7 @@ export function issueService(db: Db) { } const staleExecutionRunId = current.executionRunId; const adopted = await db.transaction(async tx => { + await notifyStatusCardIssueChange(tx, issueCompany.companyId); await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors); await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers); return tx @@ -12524,6 +12536,7 @@ export function issueService(db: Db) { patch.assigneeAgentId = null; } + await notifyStatusCardIssueChange(tx, existing.companyId); await notifyTerminalWorkspaceWork(tx, `issue:${id}`); if (existing.checkoutRunId || existing.executionRunId || options.clearAssignee) { await notifyExternalConversationSettlement(tx, { companyId: existing.companyId, issueId: id }); @@ -12581,12 +12594,12 @@ export function issueService(db: Db) { return created; }, - deleteLabel: async (id: string) => - db - .delete(labels) - .where(eq(labels.id, id)) - .returning() - .then((rows) => rows[0] ?? null), + deleteLabel: async (id: string) => db.transaction(async tx => { + const [label] = await tx.select({ companyId: labels.companyId }).from(labels).where(eq(labels.id, id)); + if (!label) return null; + await notifyStatusCardIssueChange(tx, label.companyId); + return tx.delete(labels).where(eq(labels.id, id)).returning().then(rows => rows[0] ?? null); + }), listComments: async ( issueId: string, @@ -12838,7 +12851,7 @@ export function issueService(db: Db) { }, dbOrTx: any = db, ): Promise { - if (dbOrTx === db && (actor.runId || actor.userId)) { + if (dbOrTx === db) { const append = () => measureRunTransaction(db, "response", async (tx) => { // Serialize run-authored comments on the issue so a provider retry @@ -13081,6 +13094,7 @@ export function issueService(db: Db) { currentUserRedactionOptions.enabled, ); } + await notifyStatusCardIssueChange(dbOrTx, issue.companyId); comment = shouldUpgradeExternalAuthorization || shouldUpgradeAttachmentAuthorization diff --git a/server/src/services/native-status-reassessment-notifications.ts b/server/src/services/native-status-reassessment-notifications.ts index fb845523bb4..33cf199736f 100644 --- a/server/src/services/native-status-reassessment-notifications.ts +++ b/server/src/services/native-status-reassessment-notifications.ts @@ -1,8 +1,10 @@ +import { notifyStatusCardIssueChange } from "./status-card-notifications.js"; import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js"; export type NativeStatusReassessmentScope = { companyId: string; issueId?: string; runId?: string; agentId?: string }; -export function notifyNativeStatusReassessment(tx: object, scope: NativeStatusReassessmentScope) { +export async function notifyNativeStatusReassessment(tx: object, scope: NativeStatusReassessmentScope) { + await notifyStatusCardIssueChange(tx, scope.companyId); return notifyDeliveryWork(tx, DELIVERY_QUEUES.nativeStatusReassessment, JSON.stringify({ companyId: scope.companyId, issueId: scope.issueId, runId: scope.runId, agentId: scope.agentId })); } diff --git a/server/src/services/projects.ts b/server/src/services/projects.ts index 28cae661e9b..45891a218a6 100644 --- a/server/src/services/projects.ts +++ b/server/src/services/projects.ts @@ -1,3 +1,4 @@ +import { notifyStatusCardIssueChange } from "./status-card-notifications.js"; import { notifyChatInboundWakeNotices } from "./chat-inbound-wake-notice-notifications.js"; import { listProjectWorkspaceProviders, resolveProjectWorkspaceProvider } from "./project-workspace-provider.js"; import { retireRemovedPluginWorkspaceDependents } from "./plugin-workspaces.js"; @@ -1013,6 +1014,7 @@ export function projectService(db: Db, options: { captureWorkspaceUpdates?: bool const row = await db.transaction(async (tx) => { if (updates.visibility !== undefined) { + await notifyStatusCardIssueChange(tx, existingProject.companyId); await notifyChatInboundWakeNotices(tx, { companyId: existingProject.companyId }); await tx.execute(sql`select pg_advisory_xact_lock(hashtextextended(${`issue-privacy-tree:${existingProject.companyId}`}, 0))`); } @@ -1086,6 +1088,7 @@ export function projectService(db: Db, options: { captureWorkspaceUpdates?: bool await tx.select({ id: projectWorkspaces.id }).from(projectWorkspaces).where(eq(projectWorkspaces.projectId, id)).for("update"); await recordProjectLifecycleEvent(tx as unknown as Db, project.companyId, id, "delete"); await notifyDeliveryWork(tx, DELIVERY_QUEUES.skillSourceSync); + await notifyStatusCardIssueChange(tx, project.companyId); const [row] = await tx.delete(projects).where(eq(projects.id, id)).returning(); return row ? { ...row, urlKey: deriveProjectUrlKey(row.name, row.id) } : null; }), diff --git a/server/src/services/status-card-evaluation.ts b/server/src/services/status-card-evaluation.ts index 143df32344a..3e33c0d0885 100644 --- a/server/src/services/status-card-evaluation.ts +++ b/server/src/services/status-card-evaluation.ts @@ -18,17 +18,18 @@ export function statusCardEvaluationService(db: Db, heartbeat: IssueAssignmentWa let reconstructReactive = true; let allChanges = false; const changes = new Set(); - let enabled = false; + let enabled: boolean | null = null; + let retryAt: number | null = null; return { changed(hints?: ReadonlySet) { for (const hint of hints ?? ["*"]) { - if (hint === "*" || hint === "changes:*") allChanges = true; + if (hint === "*" || hint === "changes:*") { allChanges = true; if (hint === "*") enabled = null; } else if (hint.startsWith("changes:")) changes.add(hint.slice("changes:".length)); } }, async run(signal: AbortSignal) { - enabled = (await settings.getExperimental()).enableStatusCards === true; + if (enabled === null) enabled = (await settings.getExperimental()).enableStatusCards === true; if (!enabled || signal.aborted) return; const reconstruct = reconstructReactive; const evaluateAllChanges = allChanges; @@ -37,7 +38,9 @@ export function statusCardEvaluationService(db: Db, heartbeat: IssueAssignmentWa reconstructReactive = false; allChanges = false; try { - const result = await cards.tickDueStatusCards(new Date(), evaluateAllChanges ? "changed" : reconstruct && companiesChanged.size === 0 ? "all" : companiesChanged); + const result = await cards.tickDueStatusCards(new Date(), evaluateAllChanges ? "changed" : companiesChanged, reconstruct); + retryAt = result.deferredCompanyIds.size ? Date.now() + 30_000 : null; + for (const companyId of result.deferredCompanyIds) changes.add(companyId); const assignments = await Promise.allSettled(result.enqueued.map(async ({ cardId, generatingIssue }) => { // Once generated, dispatch or cancel even if shutdown begins: leaving // an unassigned claimed card is not a completed sweep. @@ -74,7 +77,8 @@ export function statusCardEvaluationService(db: Db, heartbeat: IssueAssignmentWa .where(and(eq(companies.status, "active"), isNull(statusCards.archivedAt), isNull(statusCards.generatingIssueId), isNotNull(statusCards.nextEvalAt))) .orderBy(asc(statusCards.nextEvalAt)).limit(1); - return next?.at?.getTime() ?? null; + const deadline = next?.at?.getTime() ?? null; + return retryAt === null ? deadline : Math.min(deadline ?? Infinity, retryAt); }, }; } diff --git a/server/src/services/status-cards.ts b/server/src/services/status-cards.ts index 84bbf7b2cd4..19e7ac75140 100644 --- a/server/src/services/status-cards.ts +++ b/server/src/services/status-cards.ts @@ -25,7 +25,7 @@ import { conflict, forbidden, notFound, unprocessable } from "../errors.js"; import { logger } from "../middleware/logger.js"; import { readBuiltInAgentMarker } from "../lib/built-in-agent-metadata.js"; import { builtInAgentService } from "./built-in-agents.js"; -import { companySearchService, companySearchUpdatedWithinMs } from "./company-search.js"; +import { companySearchService } from "./company-search.js"; import { issueService } from "./issues.js"; import { SUMMARIZER_BUILT_IN_KEY } from "./summary-slots.js"; import { @@ -47,7 +47,6 @@ import { type StatusCardActor = { agentId: string | null; userId: string | null }; type StatusCardWriter = { agentId: string | null; runId: string | null }; type StatusCardRow = typeof statusCards.$inferSelect; -type WatchedIssue = CompanySearchIssueSummary & { scopeExpiresAt?: Date | null }; type StatusCardReadOptions = { issueReadCondition?: SQL }; const TERMINAL_ISSUE_STATUSES = new Set(["done", "cancelled"]); @@ -590,22 +589,15 @@ export function statusCardService( return loadIssueSummaries(card.companyId, card.mentionedIssueIds ?? [], options); } - async function executeQueries(card: StatusCardRow, options: StatusCardReadOptions = {}) { - const issueMap = new Map(); + async function executeQueries(card: StatusCardRow, options: StatusCardReadOptions = {}, onNextMembershipChange?: (at: Date | null) => void) { + const issueMap = new Map(); for (const storedQuery of card.queries) { const query = companySearchQuerySchema.parse(storedQuery); const response = await searchSvc.search(card.companyId, query, { - issueReadCondition: options.issueReadCondition, + issueReadCondition: options.issueReadCondition, onNextMembershipChange, }); for (const result of response.results) { - if (result.type !== "issue" || !result.issue) continue; - const duration = companySearchUpdatedWithinMs(query.updatedWithin); - const expiry = duration === null ? null : new Date(new Date(result.issue.updatedAt).getTime() + duration + 1); - const previous = issueMap.get(result.issue.id); - issueMap.set(result.issue.id, { ...result.issue, scopeExpiresAt: previous - ? previous.scopeExpiresAt == null || expiry === null ? null - : new Date(Math.max(previous.scopeExpiresAt.getTime(), expiry.getTime())) - : expiry }); + if (result.type === "issue" && result.issue) issueMap.set(result.issue.id, result.issue); } } // Issues mentioned in the latest summary join the watched set alongside the @@ -615,11 +607,7 @@ export function statusCardService( (card.mentionedIssueIds ?? []).filter((issueId) => !issueMap.has(issueId)), options, ); - for (const issue of mentioned) issueMap.set(issue.id, { ...issue, scopeExpiresAt: null }); - for (const id of card.mentionedIssueIds ?? []) { - const issue = issueMap.get(id); - if (issue) issue.scopeExpiresAt = null; - } + for (const issue of mentioned) issueMap.set(issue.id, issue); const snapshot = [...issueMap.values()]; if (snapshot.length === 0) return snapshot; const latestHumanComments = await db @@ -643,11 +631,10 @@ export function statusCardService( return snapshot.map((issue) => ({ ...issue, latestHumanCommentAt: commentByIssueId.get(issue.id) ?? null })); } - function nextEvaluationAt(card: StatusCardRow, snapshot: WatchedIssue[], now: Date) { + function nextEvaluationAt(card: StatusCardRow, membershipAt: Date | null, now: Date) { const interval = nextStatusCardEvaluationAt(card.refreshPolicy, now); - if (card.refreshPolicy.mode !== "reactive" || !card.refreshPolicy.triggers.membershipChanges) return interval; - const expiries = snapshot.flatMap(issue => issue.scopeExpiresAt && issue.scopeExpiresAt > now ? [issue.scopeExpiresAt.getTime()] : []); - return expiries.length ? new Date(Math.min(...expiries)) : null; + return card.refreshPolicy.mode === "reactive" && (card.refreshPolicy.triggers.membershipChanges || card.refreshPolicy.triggers.anyUpdate) + ? membershipAt : interval; } async function requestRefresh(cardId: string, input: { @@ -670,13 +657,14 @@ export function statusCardService( } const now = input.now ?? new Date(); - const snapshot = await executeQueries(card); + let membershipAt: Date | null = null; + const snapshot = await executeQueries(card, {}, at => { if (at && (!membershipAt || at < membershipAt)) membershipAt = at; }); const fingerprint = buildStatusCardFingerprint(snapshot); const allChanges = diffStatusCardFingerprint(card.fingerprint as StatusCardFingerprint | null, fingerprint); const changes = filterStatusCardChanges(allChanges, card.refreshPolicy); const trigger = input.trigger ?? "manual"; const forceRun = trigger === "manual" || trigger === "restore"; - const nextEvalAt = nextEvaluationAt(card, snapshot, now); + const nextEvalAt = nextEvaluationAt(card, membershipAt, now); if (!forceRun && changes.length === 0) { const [next] = await mutate(card.companyId, tx => tx.update(statusCards).set({ pendingChangeCount: 0, @@ -797,9 +785,12 @@ export function statusCardService( return { card: next, generatingIssue: generationIssue!, alreadyGenerating: deduplicated, enqueued: true, kind, changes }; } - async function tickDueStatusCards(now = new Date(), reactiveCompanies?: ReadonlySet | "all" | "changed") { - const reactiveScope = reactiveCompanies === "all" ? isNull(statusCards.nextEvalAt) : reactiveCompanies === "changed" ? sql`true` : reactiveCompanies?.size + async function tickDueStatusCards(now = new Date(), reactiveCompanies?: ReadonlySet | "changed", reconstruct = false) { + const changedScope = reactiveCompanies === "changed" ? sql`true` : reactiveCompanies?.size ? inArray(statusCards.companyId, [...reactiveCompanies]) : sql`false`; + // Startup restores every disarmed reactive scope. A concurrent committed + // source hint can additionally preempt its company's saved future deadline. + const reactiveScope = or(reconstruct ? isNull(statusCards.nextEvalAt) : undefined, changedScope); const due = await db.select({ card: statusCards }).from(statusCards) .innerJoin(companies, eq(companies.id, statusCards.companyId)) .where(and(eq(companies.status, "active"), isNull(statusCards.archivedAt), isNull(statusCards.generatingIssueId), @@ -808,6 +799,7 @@ export function statusCardService( .then(rows => rows.map(row => row.card)); const enqueued: Array<{ cardId: string; generatingIssue: typeof issues.$inferSelect }> = []; let evaluated = 0; + const deferredCompanyIds = new Set(); for (const candidate of due) { // nextEvalAt is both a retry lease and an evaluation deadline. A source // event may preempt a genuine future deadline, so serialize scheduler @@ -817,7 +809,7 @@ export function statusCardService( const [lock] = await lockTx.execute<{ acquired: boolean }>( sql`select pg_try_advisory_xact_lock(hashtextextended(${candidate.id}, 0)) as acquired`, ); - if (!lock?.acquired) return; + if (!lock?.acquired) { deferredCompanyIds.add(candidate.companyId); return; } const claimUntil = new Date(now.getTime() + 5 * 60 * 1000); const [claimed] = await mutate(candidate.companyId, tx => tx.update(statusCards).set({ nextEvalAt: claimUntil }) .where(and(eq(statusCards.id, candidate.id), isNull(statusCards.archivedAt), isNull(statusCards.generatingIssueId), @@ -836,7 +828,7 @@ export function statusCardService( } }); } - return { evaluated, enqueued }; + return { evaluated, enqueued, deferredCompanyIds }; } async function writeSummary(cardId: string, input: WriteStatusCardSummary, actor: StatusCardWriter) { @@ -866,7 +858,8 @@ export function statusCardService( const mentionedIssueIds = await resolveMentionedIssueIds(current.companyId, input.markdown); // Current watched membership: compiled-query matches plus the issues this // summary mentions. - const watchedSnapshot = await executeQueries({ ...current, mentionedIssueIds }); + let membershipAt: Date | null = null; + const watchedSnapshot = await executeQueries({ ...current, mentionedIssueIds }, {}, at => { if (at && (!membershipAt || at < membershipAt)) membershipAt = at; }); const watchedNow = buildStatusCardFingerprint(watchedSnapshot); let snapshot: StatusCardFingerprint; if (payloadFingerprint) { @@ -940,7 +933,7 @@ export function statusCardService( lastChangeAt: null, nextEvalAt: current.refreshPolicy.mode === "reactive" && filterStatusCardChanges(diffStatusCardFingerprint(snapshot, watchedNow), current.refreshPolicy).length > 0 - ? now : nextEvaluationAt(current, watchedSnapshot, now), + ? now : nextEvaluationAt(current, membershipAt, now), updatedAt: now, }).where(and(eq(statusCards.id, current.id), eq(statusCards.generatingIssueId, input.generationIssueId))).returning(); if (!next) throw conflict("Status-card generation was superseded by a newer task"); diff --git a/server/src/services/task-watchdogs.ts b/server/src/services/task-watchdogs.ts index cf8d92d39df..84c7f92a8a6 100644 --- a/server/src/services/task-watchdogs.ts +++ b/server/src/services/task-watchdogs.ts @@ -1,3 +1,4 @@ +import { notifyStatusCardIssueChange } from "./status-card-notifications.js"; import { createHash } from "node:crypto"; import { and, asc, eq, gte, inArray, isNull, or, sql } from "drizzle-orm"; import type { Db } from "@paperclipai/db"; @@ -1398,10 +1399,13 @@ export function taskWatchdogService(db: Db, deps: TaskWatchdogServiceDeps = {}) }) ?? fallback : fallback; if (!shouldReopen && watchdogIssue.originFingerprint !== input.classification.stopFingerprint) { - await db + await db.transaction(async tx => { + await notifyStatusCardIssueChange(tx, input.watchdog.companyId); + await tx .update(issues) .set({ originFingerprint: input.classification.stopFingerprint, updatedAt: new Date() }) .where(and(eq(issues.companyId, input.watchdog.companyId), eq(issues.id, watchdogIssue.id))); + }); watchdogIssue.originFingerprint = input.classification.stopFingerprint; } await issuesSvc.addComment( diff --git a/server/src/services/workspace-runtime.ts b/server/src/services/workspace-runtime.ts index 600d5e32a42..2bb9239eaf3 100644 --- a/server/src/services/workspace-runtime.ts +++ b/server/src/services/workspace-runtime.ts @@ -1,3 +1,4 @@ +import { notifyStatusCardIssueChange } from "./status-card-notifications.js"; import { insertIssueComment } from "./issue-comment-writes.js"; import { restoreGitWorkspaceContents } from "./git-workspace-contents.js"; import { savedGitWorkspaceRestoration, ensureSavedGitCommit, GitCacheRestorationFailure } from "./git-workspace-restoration.js"; @@ -1636,10 +1637,11 @@ async function writeDirtyQuarantineAuditComments(input: { let sourceAuditCommentId: string | null = null; let claimantAuditCommentId: string | null = null; const now = new Date(); - if (input.evidence.sourceIssueId) { + const sourceIssueId = input.evidence.sourceIssueId; + if (sourceIssueId) { const [sourceComment] = await insertIssueComment(input.db, { companyId: input.companyId, - issueId: input.evidence.sourceIssueId, + issueId: sourceIssueId, authorAgentId: null, authorUserId: null, authorType: "system", @@ -1647,10 +1649,10 @@ async function writeDirtyQuarantineAuditComments(input: { body, }); sourceAuditCommentId = sourceComment?.id ?? null; - await input.db - .update(issues) - .set({ updatedAt: now }) - .where(eq(issues.id, input.evidence.sourceIssueId)); + await input.db.transaction(async tx => { + await notifyStatusCardIssueChange(tx, input.companyId); + await tx.update(issues).set({ updatedAt: now }).where(eq(issues.id, sourceIssueId)); + }); } const claimantIssueId = input.evidence.contention?.claimedByIssueId ?? null; @@ -1665,10 +1667,10 @@ async function writeDirtyQuarantineAuditComments(input: { body, }); claimantAuditCommentId = claimantComment?.id ?? null; - await input.db - .update(issues) - .set({ updatedAt: now }) - .where(eq(issues.id, claimantIssueId)); + await input.db.transaction(async tx => { + await notifyStatusCardIssueChange(tx, input.companyId); + await tx.update(issues).set({ updatedAt: now }).where(eq(issues.id, claimantIssueId)); + }); } return { sourceAuditCommentId, claimantAuditCommentId }; From 097ac579da3a19154d28ba980906d3a23080363e Mon Sep 17 00:00:00 2001 From: Dotta Date: Fri, 30 Oct 2026 17:59:04 -0500 Subject: [PATCH 40/70] perf(server): batch remaining attention retention page evidence Load company-partitioned decision/recovery links, run/agent visibility and newer-run probes once per page; share one blocker graph reconstruction and exact SQL timestamps across page entries. Keep fresh per-entry archive locks and revalidation. Add full100-source query budgets and late visibility/supersession safety controls. Co-Authored-By: Paperclip --- .../2026-10-30-attention-retention-sleep.md | 2 +- .../attention-retention-work.test.ts | 85 +++++++++++++++ server/src/services/attention-retention.ts | 103 ++++++++++++++---- 3 files changed, 169 insertions(+), 21 deletions(-) diff --git a/doc/plans/2026-10-30-attention-retention-sleep.md b/doc/plans/2026-10-30-attention-retention-sleep.md index 1b71dc71261..2e0373fc975 100644 --- a/doc/plans/2026-10-30-attention-retention-sleep.md +++ b/doc/plans/2026-10-30-attention-retention-sleep.md @@ -25,7 +25,7 @@ The feed and worker share confirmation collapse, newer-run probes, issue visibil ## Existing mutation boundaries reactivate work -Committed hints identify a company, source kind, and source ID where those values are known. Retention's own writes use exact source hints; blocker retention changes invalidate the company/source because multiple blocked roots can project one terminal subject. Reconstruction pages and due batches remain bounded. Review attention and interaction invokability are computed in company batches per candidate page; a full 100-candidate review page uses at most 20 statements and 100 confirmations use at most six. Ordinary execution, decision and budget mutations pass their affected company; exact decision sources are included. +Committed hints identify a company, source kind, and source ID where those values are known. Retention's own writes use exact source hints; blocker retention changes invalidate the company/source because multiple blocked roots can project one terminal subject. Reconstruction pages and due batches remain bounded. Review attention and interaction invokability are computed in company batches per candidate page; a full 100-candidate review page uses at most 20 statements and 100 confirmations use at most six. Decisions, recovery source links, runs, eligible agents, issue visibility, newer-run probes and blocker graphs also load in company batches per page. A 100-source decision or recovery page uses at most three statements; 100 failed runs use at most six; 100 blocker roots use at most 30 graph/source statements plus retention-state reads for their emitted entries. These evidence maps exist only for one load. Archive checks still lock and freshly re-read sources, terminal subjects and related eligibility. Ordinary execution, decision and budget mutations pass their affected company; exact decision sources are included. Producer coverage includes decision creation and changes; interaction creation, resolution, expiry and supersession; issue creation, updates, blocker relations, checkout, adoption and release; agent hire, configuration, lifecycle, error clearing and deletion; budget incidents and company pause/resume; exhausted-run events and retry-run creation; native status, review, finalization and reconstruction; external conversation transitions; and imported blocker relations. Existing service transactions retain outer commit ownership. Savepoint release does not publish work before outer commit. diff --git a/server/src/__tests__/attention-retention-work.test.ts b/server/src/__tests__/attention-retention-work.test.ts index 889030d2c22..fd1a01e8582 100644 --- a/server/src/__tests__/attention-retention-work.test.ts +++ b/server/src/__tests__/attention-retention-work.test.ts @@ -171,6 +171,91 @@ const delay = DEFAULT_DECISION_ARCHIVE_DAYS * 86_400_000; expect(statements.filter(query => query.includes('from "agents"'))).toHaveLength(1); }); + it("batches decision and recovery source links and rechecks a late hidden origin", async () => { + const origins = await db.insert(issues).values(Array.from({ length: 100 }, () => ({ companyId, + title: randomUUID(), status: "todo", updatedAt: old() }))).returning(); + const [run] = await db.insert(heartbeatRuns).values({ companyId, agentId, status: "succeeded" }).returning(); + const decisionRows = await db.insert(decisions).values(origins.map(origin => ({ companyId, + originAgentId: agentId, originIssueId: origin.id, originRunId: run.id, title: "Decision", body: "Choose", + options: [], signedSpec: "fixture", targetSnapshots: {}, expiresAt: new Date(Date.now() + 86_400_000), updatedAt: old() }))).returning(); + const recoveryRows = await db.insert(issueRecoveryActions).values(origins.map(origin => ({ companyId, + sourceIssueId: origin.id, kind: "execution", ownerType: "board", cause: "fixture", fingerprint: "fixture", + nextAction: "Choose", updatedAt: old() }))).returning(); + const service = attentionRetentionService(db); + const pages = []; + for (const sourceKind of ["decision", "recovery_action"] as const) { + statements.length = 0; + const page = await service.load({ companyId, sourceKind }); + expect(page.rows).toHaveLength(100); expect(page.complete).toBe(false); + expect(statements.length).toBeLessThanOrEqual(3); + pages.push(page.rows!); + } + await db.update(issues).set({ hiddenAt: new Date() }).where(eq(issues.id, origins[99].id)); + const hiddenDecision = pages[0].find(entry => entry.sourceId === decisionRows[99].id)!; + const hiddenRecovery = pages[1].find(entry => entry.sourceId === recoveryRows[99].id)!; + expect(await service.check(hiddenDecision)).toBeNull(); expect(await service.check(hiddenRecovery)).toBeNull(); + expect(await state("decision", hiddenDecision.sourceId)).toBeNull(); expect(await state("recovery_action", hiddenRecovery.sourceId)).toBeNull(); + expect(await db.select().from(decisionArchiveNotificationOutbox)).toHaveLength(0); + }); + + it("batches 100 failed runs and rechecks late visibility and supersession before archive", async () => { + const origins = await db.insert(issues).values(Array.from({ length: 100 }, () => ({ companyId, + title: randomUUID(), status: "todo", updatedAt: old() }))).returning(); + const runs = await db.insert(heartbeatRuns).values(origins.map(origin => ({ companyId, agentId, + status: "failed", contextSnapshot: { issueId: origin.id }, createdAt: old(), finishedAt: old() }))).returning(); + await db.insert(heartbeatRunEvents).values(runs.map(run => ({ companyId, agentId, runId: run.id, + seq: 1, eventType: "lifecycle", message: "Bounded retry exhausted: batch fixture" }))); + const service = attentionRetentionService(db), scope = { companyId, sourceKind: "failed_run" as const }; + statements.length = 0; + const page = await service.load(scope); + expect(page.rows).toHaveLength(100); expect(page.complete).toBe(false); + expect(statements.length).toBeLessThanOrEqual(6); + const hiddenRun = runs[98], supersededRun = runs[99]; + const hidden = page.rows!.find(entry => entry.sourceId === hiddenRun.id)!; + const superseded = page.rows!.find(entry => entry.sourceId === supersededRun.id)!; + await db.update(issues).set({ hiddenAt: new Date() }).where(eq(issues.id, origins[98].id)); + await db.insert(heartbeatRuns).values({ companyId, agentId, status: "succeeded", + contextSnapshot: { issueId: origins[99].id }, createdAt: new Date(old().getTime() + 60_000), finishedAt: old() }); + expect(await service.check(hidden)).toBeNull(); expect(await service.check(superseded)).toBeNull(); + expect(await state("failed_run", hiddenRun.id)).toBeNull(); expect(await state("failed_run", supersededRun.id)).toBeNull(); + expect(await db.select().from(decisionArchiveNotificationOutbox)).toHaveLength(0); + expect((await service.load(scope)).complete).toBe(true); + const refreshed = await service.load(scope); + expect(refreshed.rows).toHaveLength(98); + expect(refreshed.rows!.some(entry => [hiddenRun.id, supersededRun.id].includes(entry.sourceId))).toBe(false); + }); + + it("batches 100 blocker graphs with exact terminal timestamps and fresh terminal visibility", async () => { + const roots = await db.insert(issues).values(Array.from({ length: 100 }, () => ({ companyId, + title: randomUUID(), status: "blocked", updatedAt: old(), blockedTransitionAt: old(), + unblockDescriptor: { owner: "board", action: "Pick an owner" } }))).returning(); + const terminals = await db.insert(issues).values(Array.from({ length: 100 }, () => ({ companyId, + title: randomUUID(), status: "backlog", assigneeAgentId: agentId, updatedAt: old() }))).returning(); + await db.insert(issueRelations).values(roots.map((root, index) => ({ companyId, + issueId: terminals[index].id, relatedIssueId: root.id, type: "blocks" }))); + await db.execute(sql`update issues set updated_at = updated_at + interval '0.000321 seconds' where id = ${terminals[99].id}::uuid`); + const service = attentionRetentionService(db), scope = { companyId, sourceKind: "blocker_attention" as const }; + statements.length = 0; + const page = await service.load(scope); + const queries = [...statements]; + expect(page.rows).toHaveLength(200); expect(page.complete).toBe(false); + const retentionReads = queries.filter(query => query.includes('from "decision_retention"')); + expect(retentionReads).toHaveLength(200); + expect(queries.length - retentionReads.length).toBeLessThanOrEqual(30); + const terminal = page.rows!.find(entry => entry.rootId === roots[99].id && entry.variant === "terminal")!; + expect(terminal.sourceId).toBe(terminals[99].id); + const [expected] = await db.execute(sql`select updated_at::text as at, + ceil(extract(epoch from updated_at) * 1000 + ${delay}) as due from issues where id = ${terminals[99].id}::uuid`); + expect(terminal.activityAt).toBe(expected.at); expect(terminal.dueAt).toBe(Number(expected.due)); + await db.update(issues).set({ hiddenAt: new Date() }).where(eq(issues.id, terminals[99].id)); + expect(await service.check(terminal)).toBeNull(); + expect(await state("blocker_attention", terminals[99].id)).toBeNull(); + expect(await db.select().from(decisionArchiveNotificationOutbox)).toHaveLength(0); + expect((await service.load(scope)).complete).toBe(true); + const refreshed = await service.load(scope); + expect(refreshed.rows!.some(entry => entry.sourceId === terminals[99].id)).toBe(false); + }); + it("scopes ordinary execution-control writes to their company", async () => { const otherCompany = randomUUID(); await db.insert(companies).values({ id: otherCompany, name: "Other", issuePrefix: "OTHER" }); diff --git a/server/src/services/attention-retention.ts b/server/src/services/attention-retention.ts index 85e27a0d941..c72947e5349 100644 --- a/server/src/services/attention-retention.ts +++ b/server/src/services/attention-retention.ts @@ -1,4 +1,4 @@ -import { and, eq, inArray, isNull, sql, type SQL } from "drizzle-orm"; +import { and, eq, getTableColumns, inArray, isNull, sql, type SQL } from "drizzle-orm"; import { agents, decisionRetention, decisions, heartbeatRuns, issueApprovals, issueRecoveryActions, issueThreadInteractions, issues, approvals, type Db } from "@paperclipai/db"; import type { AttentionSourceKind } from "@paperclipai/shared"; @@ -23,6 +23,18 @@ type Candidate = { sourceKind: Kind; sourceId: string; companyId: string; activi type Entry = Candidate & { id: string; dueAt: number; rootId: string; variant?: "owner" | "terminal" }; type Cursor = { sourceKind: string; sourceId: string }; +type PageIssue = typeof issues.$inferSelect & { activityAt: string; deadlineAt: number; + blockedActivityAt: string | null; blockedDeadlineAt: number | null }; +type PageEvidence = { + decisions: Map; + recoveries: Map; + runs: Map; + agents: Set; + visibleIssues: Map; + newerRuns: Set; + blockers: Awaited["listBlockerAttention"]>>; +}; + const sourceTables: Record = { decision: sql`decisions`, issue_thread_interaction: sql`issue_thread_interactions`, recovery_action: sql`issue_recovery_actions`, blocker_attention: sql`issues`, review: sql`issues`, @@ -137,56 +149,106 @@ export function attentionRetentionService(db: Db) { return result; } - async function eligible(database: Db, candidate: Candidate, variant?: "owner" | "terminal", lock = false, reviewAttention?: Map, interactionAttention?: Map): Promise { + async function pageEvidence(database: Db, candidates: Candidate[]) { + const byCompany = new Map(); + for (const candidate of candidates) { + if (!["decision", "recovery_action", "failed_run", "blocker_attention"].includes(candidate.sourceKind)) continue; + const rows = byCompany.get(candidate.companyId) ?? []; + rows.push(candidate); + byCompany.set(candidate.companyId, rows); + } + const result = new Map(); + for (const [companyId, rows] of byCompany) { + const ids = (kind: Kind) => rows.filter(row => row.sourceKind === kind).map(row => row.sourceId); + const decisionIds = ids("decision"), recoveryIds = ids("recovery_action"), runIds = ids("failed_run"), blockerIds = ids("blocker_attention"); + const decisionRows = decisionIds.length ? await database.select({ id: decisions.id, originIssueId: decisions.originIssueId }) + .from(decisions).where(and(eq(decisions.companyId, companyId), inArray(decisions.id, decisionIds), eq(decisions.status, "open"))) : []; + const recoveryRows = recoveryIds.length ? await database.select().from(issueRecoveryActions).where(and( + eq(issueRecoveryActions.companyId, companyId), inArray(issueRecoveryActions.id, recoveryIds))) : []; + const runRows = runIds.length ? await database.select().from(heartbeatRuns).where(and( + eq(heartbeatRuns.companyId, companyId), inArray(heartbeatRuns.id, runIds))) : []; + const runAgents = runRows.length ? await database.select({ id: agents.id }).from(agents).where(and( + eq(agents.companyId, companyId), inArray(agents.id, [...new Set(runRows.map(row => row.agentId))]), + sql`${agents.status} <> 'terminated'`)) : []; + const readIssues = async (issueIds: string[]) => issueIds.length ? database.select({ ...getTableColumns(issues), + activityAt: sql`${issues.updatedAt}::text`, + deadlineAt: sql`ceil(extract(epoch from ${issues.updatedAt}) * 1000 + ${archiveDelay})`.mapWith(Number), + blockedActivityAt: sql`${issues.blockedTransitionAt}::text`, + blockedDeadlineAt: sql`ceil(extract(epoch from ${issues.blockedTransitionAt}) * 1000 + ${archiveDelay})`.mapWith(Number), + }).from(issues).where(and(eq(issues.companyId, companyId), inArray(issues.id, [...new Set(issueIds)]), + attentionIssueCondition(false))) : []; + const issueIds = [...blockerIds, ...decisionRows.map(row => row.originIssueId), ...recoveryRows.map(row => row.sourceIssueId), + ...runRows.map(row => readRunIssueId(row.contextSnapshot))].filter((id): id is string => Boolean(id)); + const visibleRows = await readIssues(issueIds); + const visible = new Map(visibleRows.map(row => [row.id, row])); + const blockerRoots = visibleRows.filter(row => blockerIds.includes(row.id)); + const blockers: PageEvidence["blockers"] = blockerRoots.length ? await issueService(database).listBlockerAttention(companyId, blockerRoots) : new Map(); + const terminalIds = [...blockers.values()].filter(terminalBlockerRequiresAttention) + .map(attention => attention.terminalBlockerIssueId).filter((id): id is string => id != null && !visible.has(id)); + for (const row of await readIssues(terminalIds)) visible.set(row.id, row); + result.set(companyId, { + decisions: new Map(decisionRows.map(row => [row.id, row])), recoveries: new Map(recoveryRows.map(row => [row.id, row])), + runs: new Map(runRows.map(row => [row.id, row])), agents: new Set(runAgents.map(row => row.id)), visibleIssues: visible, + newerRuns: await listFailedRunsWithNewerRun(database, companyId, + runRows.map(run => ({ ...run, issueId: readRunIssueId(run.contextSnapshot) }))), blockers, + }); + } + return result; + } + + async function eligible(database: Db, candidate: Candidate, variant?: "owner" | "terminal", lock = false, reviewAttention?: Map, interactionAttention?: Map, evidence?: PageEvidence): Promise { let sourceId = candidate.sourceId; let activityAt = candidate.activityAt; let dueAt = candidate.deadlineAt; if (candidate.sourceKind === "decision") { - const [row] = await database.select({ originIssueId: decisions.originIssueId }).from(decisions) - .where(and(eq(decisions.companyId, candidate.companyId), eq(decisions.id, sourceId), eq(decisions.status, "open"))); - if (!row || row.originIssueId && !await visibleIssue(database, candidate.companyId, row.originIssueId)) return null; + const row = evidence ? evidence.decisions.get(sourceId) : (await database.select({ originIssueId: decisions.originIssueId }).from(decisions) + .where(and(eq(decisions.companyId, candidate.companyId), eq(decisions.id, sourceId), eq(decisions.status, "open"))))[0]; + if (!row || row.originIssueId && !(evidence ? evidence.visibleIssues.has(row.originIssueId) + : await visibleIssue(database, candidate.companyId, row.originIssueId))) return null; } else if (candidate.sourceKind === "issue_thread_interaction") { const attention = interactionAttention ?? await interactionEligibility(database, [candidate]); if (!attention.get(sourceId)) return null; } else if (candidate.sourceKind === "recovery_action") { - const [row] = await database.select().from(issueRecoveryActions).where(and(eq(issueRecoveryActions.companyId, candidate.companyId), eq(issueRecoveryActions.id, sourceId))); - if (!row || !await visibleIssue(database, candidate.companyId, row.sourceIssueId)) return null; + const row = evidence ? evidence.recoveries.get(sourceId) : (await database.select().from(issueRecoveryActions).where(and(eq(issueRecoveryActions.companyId, candidate.companyId), eq(issueRecoveryActions.id, sourceId))))[0]; + if (!row || !(evidence ? evidence.visibleIssues.has(row.sourceIssueId) + : await visibleIssue(database, candidate.companyId, row.sourceIssueId))) return null; } else if (candidate.sourceKind === "review") { const attention = reviewAttention ?? await reviewEligibility(database, [candidate]); if (!attention.get(sourceId)) return null; } else if (candidate.sourceKind === "blocker_attention") { - const row = await visibleIssue(database, candidate.companyId, sourceId); + const row = evidence ? evidence.visibleIssues.get(sourceId) : await visibleIssue(database, candidate.companyId, sourceId); if (!row) return null; if (variant === "owner") { if (row.unblockDescriptor?.owner !== "board" || !isProspectiveBlockedTransition(row)) return null; - const [activity] = await database.execute(sql`select blocked_transition_at::text as at, + const activity = evidence ? { at: evidence.visibleIssues.get(sourceId)!.blockedActivityAt!, due: evidence.visibleIssues.get(sourceId)!.blockedDeadlineAt! } : (await database.execute(sql`select blocked_transition_at::text as at, ceil(extract(epoch from blocked_transition_at) * 1000 + ${archiveDelay}) as due from issues - where company_id = ${candidate.companyId}::uuid and id = ${row.id}::uuid`) as unknown as Array<{ at: string; due: string }>; + where company_id = ${candidate.companyId}::uuid and id = ${row.id}::uuid`) as unknown as Array<{ at: string; due: string }>)[0]; activityAt = activity.at; dueAt = Number(activity.due); } else { - const attention = (await issueService(database).listBlockerAttention(candidate.companyId, [row])).get(row.id); + const attention = evidence ? evidence.blockers.get(row.id) : (await issueService(database).listBlockerAttention(candidate.companyId, [row])).get(row.id); if (!terminalBlockerRequiresAttention(attention)) return null; sourceId = attention.terminalBlockerIssueId ?? row.id; if (lock && sourceId !== candidate.sourceId) await database.execute(sql`select id from issues where company_id = ${candidate.companyId}::uuid and id = ${sourceId}::uuid for update`); - const terminal = await visibleIssue(database, candidate.companyId, sourceId); + const terminal = evidence ? evidence.visibleIssues.get(sourceId) : await visibleIssue(database, candidate.companyId, sourceId); if (!terminal) return null; - const [activity] = await database.execute(sql`select updated_at::text as at, + const activity = evidence ? { at: evidence.visibleIssues.get(sourceId)!.activityAt, due: evidence.visibleIssues.get(sourceId)!.deadlineAt } : (await database.execute(sql`select updated_at::text as at, ceil(extract(epoch from updated_at) * 1000 + ${archiveDelay}) as due from issues - where company_id = ${candidate.companyId}::uuid and id = ${sourceId}::uuid`) as unknown as Array<{ at: string; due: string }>; + where company_id = ${candidate.companyId}::uuid and id = ${sourceId}::uuid`) as unknown as Array<{ at: string; due: string }>)[0]; activityAt = activity.at; dueAt = Number(activity.due); } const [retention] = await database.select().from(decisionRetention).where(and(eq(decisionRetention.companyId, candidate.companyId), eq(decisionRetention.sourceKind, candidate.sourceKind), eq(decisionRetention.sourceId, sourceId))); if (retention?.keep || retention?.archivedAt) return null; } else if (candidate.sourceKind === "failed_run") { - const [run] = await database.select().from(heartbeatRuns).where(and(eq(heartbeatRuns.companyId, candidate.companyId), eq(heartbeatRuns.id, sourceId))); + const run = evidence ? evidence.runs.get(sourceId) : (await database.select().from(heartbeatRuns).where(and(eq(heartbeatRuns.companyId, candidate.companyId), eq(heartbeatRuns.id, sourceId))))[0]; if (!run) return null; - const [agent] = await database.select({ id: agents.id }).from(agents).where(and(eq(agents.companyId, candidate.companyId), - eq(agents.id, run.agentId), sql`${agents.status} <> 'terminated'`)); + const agent = evidence ? evidence.agents.has(run.agentId) : (await database.select({ id: agents.id }).from(agents).where(and(eq(agents.companyId, candidate.companyId), + eq(agents.id, run.agentId), sql`${agents.status} <> 'terminated'`)))[0]; const issueId = readRunIssueId(run.contextSnapshot); - if (!agent || issueId && !await visibleIssue(database, candidate.companyId, issueId)) return null; - if ((await listFailedRunsWithNewerRun(database, candidate.companyId, [{ ...run, issueId }])).has(run.id)) return null; + if (!agent || issueId && !(evidence ? evidence.visibleIssues.has(issueId) + : await visibleIssue(database, candidate.companyId, issueId))) return null; + if ((evidence ? evidence.newerRuns : await listFailedRunsWithNewerRun(database, candidate.companyId, [{ ...run, issueId }])).has(run.id)) return null; } return { ...candidate, sourceId, activityAt, rootId: candidate.sourceId, variant, id: `${candidate.sourceKind}:${candidate.sourceId}:${variant ?? "source"}`, dueAt }; @@ -198,10 +260,11 @@ export function attentionRetentionService(db: Db) { const entries: Entry[] = []; const reviews = await reviewEligibility(db, rows); const interactions = await interactionEligibility(db, rows); + const evidence = await pageEvidence(db, rows); for (const row of rows) { if (signal?.aborted) return { rows: null, complete: true }; for (const variant of row.sourceKind === "blocker_attention" ? ["owner", "terminal"] as const : [undefined]) { - const entry = await eligible(db, row, variant, false, reviews, interactions); + const entry = await eligible(db, row, variant, false, reviews, interactions, evidence.get(row.companyId)); if (entry) entries.push(entry); } } From 4e97daafa06a54f02027335437c4f48e50b84236 Mon Sep 17 00:00:00 2001 From: Dotta Date: Fri, 30 Oct 2026 18:09:41 -0500 Subject: [PATCH 41/70] refactor: remove obsolete external object batch refresh entrypoints Exercise lease, selection, and recovery behavior through the registered scheduler owner. Co-Authored-By: Paperclip --- .../2026-10-30-external-objects-sleep.md | 8 ++- .../external-object-refresh-work.test.ts | 8 --- .../external-objects-service.test.ts | 66 +++++++++++-------- .../server-startup-feedback-export.test.ts | 4 -- server/src/services/external-objects.ts | 52 --------------- 5 files changed, 43 insertions(+), 95 deletions(-) diff --git a/doc/plans/2026-10-30-external-objects-sleep.md b/doc/plans/2026-10-30-external-objects-sleep.md index 61c7d8dded1..41bf59440d1 100644 --- a/doc/plans/2026-10-30-external-objects-sleep.md +++ b/doc/plans/2026-10-30-external-objects-sleep.md @@ -106,9 +106,11 @@ result; it releases only its own claim and cannot publish that late result. Startup does not await provider work. Shutdown closes the active GitHub HTTP request, and its saved due row can resume immediately after restart. -The two explicit `refreshDueObjects*` helpers remain manual diagnostic/test entry -points. They have no production scheduler caller. Their regression fixtures also -preserve the prior selection/lease semantics. The scheduled check retains its +The old `refreshDueObjects*` batch helpers are removed. Existing selection, +lease, concurrent manual refresh, terminal-history and not-found recovery tests +now enter through the registered scheduler owner. The removed baseline control +recorded six SQL reads over three empty aggregate ticks; the current quiet-work +acceptance measures zero recurring SQL after settlement. The scheduled check retains its post-refresh scoped read to revalidate current company, reference and resolver eligibility and recover an authoritative lease after a competing writer; all such reads target one object rather than rebuilding a company. diff --git a/server/src/__tests__/external-object-refresh-work.test.ts b/server/src/__tests__/external-object-refresh-work.test.ts index 754725c7d80..4c1817b9722 100644 --- a/server/src/__tests__/external-object-refresh-work.test.ts +++ b/server/src/__tests__/external-object-refresh-work.test.ts @@ -100,14 +100,6 @@ const active: ExternalObjectResolveResult = { ok: true, snapshot: { }); } - it("captures the old configured-empty aggregate SQL cost", async () => { - const svc = service(); await fixture(); - const selects = vi.spyOn(db, "select"); - for (let i = 0; i < 3; i++) await svc.refreshDueObjectsForActiveCompanies(); - expect(selects).toHaveBeenCalledTimes(6); - expect(provider).not.toHaveBeenCalled(); - }); - it("activates saved work when the ordinary feature and company settings enable it", async () => { clock(); const settings = instanceSettingsService(db); await settings.updateExperimental({ enableExternalObjects: true }); diff --git a/server/src/__tests__/external-objects-service.test.ts b/server/src/__tests__/external-objects-service.test.ts index bc268d0ba6a..e6de9780444 100644 --- a/server/src/__tests__/external-objects-service.test.ts +++ b/server/src/__tests__/external-objects-service.test.ts @@ -1,4 +1,5 @@ import { randomUUID } from "node:crypto"; +import { createDeliveryWorkCoordinator } from "../services/delivery-work-coordinator.js"; import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest"; import { eq } from "drizzle-orm"; import { @@ -733,6 +734,18 @@ describeEmbeddedPostgres("externalObjectService", () => { await tempDb?.cleanup(); }); + async function runRefreshWork(service: ReturnType) { + const errors: unknown[] = []; + const coordinator = createDeliveryWorkCoordinator({ owner: db, canRun: () => true, + onError: error => errors.push(error) }); + try { + await service.registerRefreshWork(coordinator, () => true).ready; + expect(errors).toEqual([]); + } finally { + await coordinator.stop(); + } + } + async function createIssue(companyId = randomUUID()) { const issueId = randomUUID(); await db.insert(companies).values({ @@ -882,9 +895,9 @@ describeEmbeddedPostgres("externalObjectService", () => { expect(object.nextRefreshAt).toBeInstanceOf(Date); - const refreshed = await svc.refreshDueObjects(companyId, 50, new Date(Date.now() + 1_000)); + await runRefreshWork(svc); - expect(refreshed).toHaveLength(1); + expect((await db.select().from(externalObjects))[0]?.lastResolvedAt).toBeInstanceOf(Date); expect(resolve).toHaveBeenCalledTimes(1); }); @@ -921,15 +934,13 @@ describeEmbeddedPostgres("externalObjectService", () => { await svc.syncIssue(issueId); const object = await db.select().from(externalObjects).then((rows) => rows[0]!); - const dueRefresh = svc.refreshDueObjects(companyId, 50, new Date(Date.now() + 1_000)); + const dueRefresh = runRefreshWork(svc); await resolverStarted; const manualRefresh = svc.refreshObject(object.id, { companyId, force: true }); finishResolver(); - const [dueResults, manualResult] = await Promise.all([dueRefresh, manualRefresh]); + const [, manualResult] = await Promise.all([dueRefresh, manualRefresh]); - expect(dueResults).toHaveLength(1); - expect(dueResults[0]?.refreshed).toBe(true); expect(manualResult.refreshed).toBe(true); expect(manualResult.object.statusLabel).toBe("Open"); expect(resolve).toHaveBeenCalledTimes(1); @@ -969,14 +980,12 @@ describeEmbeddedPostgres("externalObjectService", () => { await scheduledService.syncIssue(issueId); const object = await db.select().from(externalObjects).then((rows) => rows[0]!); - const dueRefresh = scheduledService.refreshDueObjects(companyId, 50, new Date(Date.now() + 1_000)); + const dueRefresh = runRefreshWork(scheduledService); await resolverStarted; const manualRefresh = await manualService.refreshObject(object.id, { companyId, force: true }); finishResolver(); - const dueResults = await dueRefresh; + await dueRefresh; - expect(dueResults).toHaveLength(1); - expect(dueResults[0]?.refreshed).toBe(true); expect(manualRefresh.refreshed).toBe(false); expect(manualRefresh.reason).toBe("refresh_in_progress"); expect(manualRefresh.object).not.toHaveProperty("refreshToken"); @@ -1032,7 +1041,7 @@ describeEmbeddedPostgres("externalObjectService", () => { const object = await db.select().from(externalObjects).then((rows) => rows[0]!); const claimedAt = new Date(Date.now() + 1_000); - const dueRefresh = scheduledService.refreshDueObjects(companyId, 50, claimedAt); + const dueRefresh = runRefreshWork(scheduledService); await resolverStarted; const replacementResult = await replacementService.refreshObject(object.id, { companyId, @@ -1040,11 +1049,10 @@ describeEmbeddedPostgres("externalObjectService", () => { now: new Date(claimedAt.getTime() + 301_000), }); finishResolver(); - const dueResults = await dueRefresh; + await dueRefresh; const finalObject = await db.select().from(externalObjects).then((rows) => rows[0]!); expect(replacementResult.refreshed).toBe(true); - expect(dueResults[0]).toMatchObject({ refreshed: false, reason: "refresh_superseded" }); expect(finalObject.statusLabel).toBe("Closed"); expect(finalObject.refreshStartedAt).toBeNull(); expect(finalObject.refreshToken).toBeNull(); @@ -1078,9 +1086,7 @@ describeEmbeddedPostgres("externalObjectService", () => { await svc.syncIssue(active.issueId); await svc.syncIssue(paused.issueId); - const result = await svc.refreshDueObjectsForActiveCompanies(50, new Date(Date.now() + 1_000)); - - expect(result).toEqual({ companies: 1, checked: 1, refreshed: 1 }); + await runRefreshWork(svc); expect(resolve).toHaveBeenCalledTimes(1); const rows = await db.select().from(externalObjects); const activeObject = rows.find((row) => row.companyId === active.companyId); @@ -1139,9 +1145,8 @@ describeEmbeddedPostgres("externalObjectService", () => { .set({ nextRefreshAt: new Date(0) }) .where(eq(externalObjects.id, object.id)); - const refreshed = await svc.refreshDueObjects(companyId); + await runRefreshWork(svc); - expect(refreshed).toEqual([]); expect(resolve).toHaveBeenCalledTimes(1); }); @@ -1184,7 +1189,7 @@ describeEmbeddedPostgres("externalObjectService", () => { }); it("re-checks previously frozen GitHub not-found objects so they can self-heal", async () => { - const { companyId } = await createIssue(); + const { companyId, issueId } = await createIssue(); const canonical = canonicalizeExternalObjectUrl("https://github.com/acme/app/pull/42"); if (!canonical) throw new Error("expected canonical url"); const inserted = await db.insert(externalObjects).values({ @@ -1225,9 +1230,10 @@ describeEmbeddedPostgres("externalObjectService", () => { const svc = externalObjectService(db, { resolvers: [resolver], github: false }); expect(object.isTerminal).toBe(true); - const refreshed = await svc.refreshDueObjects(companyId); - expect(refreshed).toHaveLength(1); - expect(refreshed[0]?.refreshed).toBe(true); + const [linked] = await db.select().from(externalObjects).where(eq(externalObjects.companyId, companyId)); + await db.insert(externalObjectMentions).values({ companyId, sourceIssueId: issueId, + sourceKind: "description", objectId: linked!.id }); + await runRefreshWork(svc); const updated = await db.select().from(externalObjects).then((rows) => rows[0]!); expect(updated.statusLabel).toBe("Open"); @@ -1236,7 +1242,7 @@ describeEmbeddedPostgres("externalObjectService", () => { }); it("does not re-check probe-verified GitHub not-found objects (deleted references are terminal)", async () => { - const { companyId } = await createIssue(); + const { companyId, issueId } = await createIssue(); const canonical = canonicalizeExternalObjectUrl("https://github.com/acme/app/pull/42"); if (!canonical) throw new Error("expected canonical url"); await db.insert(externalObjects).values({ @@ -1273,14 +1279,16 @@ describeEmbeddedPostgres("externalObjectService", () => { }; const svc = externalObjectService(db, { resolvers: [resolver], github: false }); - const refreshed = await svc.refreshDueObjects(companyId); + const [linked] = await db.select().from(externalObjects).where(eq(externalObjects.companyId, companyId)); + await db.insert(externalObjectMentions).values({ companyId, sourceIssueId: issueId, + sourceKind: "description", objectId: linked!.id }); + await runRefreshWork(svc); - expect(refreshed).toEqual([]); expect(resolver.resolve).not.toHaveBeenCalled(); }); it("does not re-check terminal objects that are not GitHub not-found verdicts", async () => { - const { companyId } = await createIssue(); + const { companyId, issueId } = await createIssue(); const canonical = canonicalizeExternalObjectUrl("https://github.com/acme/app/pull/42"); if (!canonical) throw new Error("expected canonical url"); await db.insert(externalObjects).values({ @@ -1316,9 +1324,11 @@ describeEmbeddedPostgres("externalObjectService", () => { }; const svc = externalObjectService(db, { resolvers: [resolver], github: false }); - const refreshed = await svc.refreshDueObjects(companyId); + const [linked] = await db.select().from(externalObjects).where(eq(externalObjects.companyId, companyId)); + await db.insert(externalObjectMentions).values({ companyId, sourceIssueId: issueId, + sourceKind: "description", objectId: linked!.id }); + await runRefreshWork(svc); - expect(refreshed).toEqual([]); expect(resolver.resolve).not.toHaveBeenCalled(); }); diff --git a/server/src/__tests__/server-startup-feedback-export.test.ts b/server/src/__tests__/server-startup-feedback-export.test.ts index 5002dc44a50..f380ef478dd 100644 --- a/server/src/__tests__/server-startup-feedback-export.test.ts +++ b/server/src/__tests__/server-startup-feedback-export.test.ts @@ -209,7 +209,6 @@ const { const externalObjectsServiceMock = { registerRefreshWork: vi.fn((coordinator: { register: typeof deliveryWorkRegisterMock }, _canRun: () => boolean) => coordinator.register("external-object-refresh", { run: async () => undefined, hasPending: async () => false })), - refreshDueObjectsForActiveCompanies: vi.fn(async () => ({ companies: 0, checked: 0, refreshed: 0 })), }; const externalObjectsServiceFactoryMock = vi.fn(() => externalObjectsServiceMock); const routineServiceMock = { @@ -1404,7 +1403,6 @@ describe("startServer feedback export wiring", () => { expect(externalObjectsServiceMock.registerRefreshWork).toHaveBeenCalledTimes(1); expect(externalObjectsServiceMock.registerRefreshWork).toHaveBeenCalledWith(expect.any(Object), expect.any(Function)); expect(deliveryWorkRegisterMock.mock.calls.some(([queue]) => queue === DELIVERY_QUEUES.externalObjectRefresh)).toBe(true); - expect(externalObjectsServiceMock.refreshDueObjectsForActiveCompanies).not.toHaveBeenCalled(); expect(issueThreadInteractionServiceMock.sweepMergedPullRequestConfirmations).toHaveBeenCalledTimes(1); expect(executionWorkspaceServiceMock.sweepTerminalWorkspaces).toHaveBeenCalledTimes(1); expect(routineServiceMock.tickScheduledTriggers).toHaveBeenCalledTimes(1); @@ -1434,7 +1432,6 @@ describe("startServer feedback export wiring", () => { }); await startServer(); expect(externalObjectsServiceMock.registerRefreshWork).toHaveBeenCalledOnce(); - expect(externalObjectsServiceMock.refreshDueObjectsForActiveCompanies).not.toHaveBeenCalled(); }); it("keeps external object refresh active when heartbeat scheduling is disabled", async () => { @@ -1467,7 +1464,6 @@ describe("startServer feedback export wiring", () => { expect(externalObjectsServiceMock.registerRefreshWork).toHaveBeenCalledTimes(1); expect(externalObjectsServiceMock.registerRefreshWork).toHaveBeenCalledWith(expect.any(Object), expect.any(Function)); expect(deliveryWorkRegisterMock.mock.calls.some(([queue]) => queue === DELIVERY_QUEUES.externalObjectRefresh)).toBe(true); - expect(externalObjectsServiceMock.refreshDueObjectsForActiveCompanies).not.toHaveBeenCalled(); expect(routineServiceMock.tickScheduledTriggers).not.toHaveBeenCalled(); expect(heartbeatServiceMock.reconcileCostAccounting).toHaveBeenCalledTimes(1); expect(environmentCustomImagesServiceMock.cleanupExpiredSetupSessions).not.toHaveBeenCalled(); diff --git a/server/src/services/external-objects.ts b/server/src/services/external-objects.ts index 0208cdab7b6..a4d26014d26 100644 --- a/server/src/services/external-objects.ts +++ b/server/src/services/external-objects.ts @@ -1109,56 +1109,6 @@ export function externalObjectService( return results; } - async function refreshDueObjectsUnchecked(companyId: string, limit = 50, now = new Date()) { - const staleRefreshStartedBefore = new Date(now.getTime() - DEFAULT_REFRESH_LEASE_SECONDS * 1000); - const refreshable = refreshableObjectWhere(); - const due = await db - .select({ id: externalObjects.id }) - .from(externalObjects) - .where( - and( - eq(externalObjects.companyId, companyId), - refreshable, - lte(externalObjects.nextRefreshAt, now), - or( - isNull(externalObjects.refreshStartedAt), - lte(externalObjects.refreshStartedAt, staleRefreshStartedBefore), - ), - ), - ) - .limit(limit); - const results = []; - for (const row of due) { - results.push(await refreshObject(row.id, { - companyId, - actor: { actorType: "system", actorId: "external-object-resolver", agentId: null, runId: null }, - now, - })); - } - return results; - } - - async function refreshDueObjects(companyId: string, limit = 50, now = new Date()) { - if (!(await isEnabled())) return []; - return refreshDueObjectsUnchecked(companyId, limit, now); - } - - async function refreshDueObjectsForActiveCompanies(limitPerCompany = 50, now = new Date()) { - if (!(await isEnabled())) return { companies: 0, checked: 0, refreshed: 0 }; - const activeCompanies = await db - .select({ id: companies.id }) - .from(companies) - .where(eq(companies.status, "active")); - let checked = 0; - let refreshed = 0; - for (const company of activeCompanies) { - const results = await refreshDueObjectsUnchecked(company.id, limitPerCompany, now); - checked += results.length; - refreshed += results.filter((result) => result.refreshed).length; - } - return { companies: activeCompanies.length, checked, refreshed }; - } - function registerRefreshWork( coordinator: ReturnType, canRun: () => boolean, @@ -1234,8 +1184,6 @@ export function externalObjectService( getProjectSummary, refreshObject, refreshIssueObjects, - refreshDueObjects, - refreshDueObjectsForActiveCompanies, registerRefreshWork, }; } From dc4456c76b3eaf261be7568c45940769b23c0236 Mon Sep 17 00:00:00 2001 From: Dotta Date: Fri, 30 Oct 2026 18:10:00 -0500 Subject: [PATCH 42/70] Drive issue recovery from scoped source writes and saved deadlines --- doc/plans/2026-10-30-issue-recovery-sleep.md | 18 ++ .../issue-recovery-deadlines.test.ts | 237 ++++++++++++++++++ .../issue-recovery-work-lifecycle.test.ts | 99 ++++++++ .../server-startup-feedback-export.test.ts | 6 +- server/src/index.ts | 76 +----- server/src/services/agent-conversations.ts | 2 + .../telegram/attachment-failure.ts | 2 + server/src/services/company-deletion.ts | 2 + server/src/services/company-portability.ts | 6 +- server/src/services/documents.ts | 11 + ...l-conversation-settlement-notifications.ts | 4 +- server/src/services/issue-recovery-actions.ts | 6 + server/src/services/issue-tree-control.ts | 8 +- server/src/services/issues.ts | 3 + .../native-runtime/chat-attachment-reuse.ts | 6 +- .../native-runner-file-handoff.ts | 6 +- ...ative-status-reassessment-notifications.ts | 4 +- server/src/services/system-recovery.ts | 2 + server/src/services/task-watchdog-work.ts | 2 +- server/src/services/task-watchdogs.ts | 6 +- 20 files changed, 419 insertions(+), 87 deletions(-) create mode 100644 doc/plans/2026-10-30-issue-recovery-sleep.md create mode 100644 server/src/__tests__/issue-recovery-deadlines.test.ts create mode 100644 server/src/__tests__/issue-recovery-work-lifecycle.test.ts diff --git a/doc/plans/2026-10-30-issue-recovery-sleep.md b/doc/plans/2026-10-30-issue-recovery-sleep.md new file mode 100644 index 00000000000..18670cc66b8 --- /dev/null +++ b/doc/plans/2026-10-30-issue-recovery-sleep.md @@ -0,0 +1,18 @@ +# Issue recovery without periodic scans + +Four existing owners reconstruct their current obligations once at startup. Committing source writes invalidate only their company. The coordinator keeps scopes, per-issue deadlines, and per-entry failures in memory. Its pending check does not query the database. + +| Owner | Durable truth and producer boundary | Deadline or event | Exit | +| --- | --- | --- | --- | +| Stranded assignment | Visible assigned todo/in-progress/review issues; issue create/update/checkout/release; run lifecycle and execution control; agent/company/budget/configuration; interaction, approval and subtree hold changes | A committed eligibility change. A transient continuation uses the latest finish plus the existing exponential delay. | An execution path, durable wait, escalation, or current ineligibility. | +| Resolved dependency | Blocked assigned issues and company-scoped blocker relations; blocker status or relation changes through issue service and import; run/control and pause gate changes | The committing change. A failed enqueue retains that issue with error backoff. | Existing idempotent wake, live path, unresolved dependency or deliberate self-block. | +| Task watchdog | Active issue watchdogs and their current source subtree; normal watchdog upsert/disable; issue/comment/document/work-product and wait changes | Exact source creation plus the existing 15-second first-run grace. An incomplete scan retains the watchdog with error backoff. | Live path, open/reviewed reusable review, disabled configuration, or ineligible source. | +| Stale issue lock | Existing checkout/execution run references; issue and run lifecycle changes | A committing ownership change, or the fresh owning controller's saved lease expiry. | No stale reference, a valid owner, or the existing guarded terminalization and lock clear. | + +The owner methods accept company and issue/watchdog identity. A narrowed lock check still reads other same-company references before using terminal-issue authority. A stale terminal alias cannot end an active issue's shared run. The controller lease owner supplies its fresh expiry; the existing write predicate revalidates the observed boot ID and database-clock expiry. + +Producer coverage uses the existing run-dispatch, execution-control, issue-review-state, native status reassessment and external conversation settlement boundaries. Direct gaps have explicit signals inside their owning transaction: document writes/import, attachment removal, blocker import, recovery-action adoption/mutation, conversation pause release, Telegram failure comment replacement, native artifact title/metadata updates and company deletion. Metadata-only comment classification that does not change timestamp, visibility or execution eligibility needs no additional watchdog activation. Approval links and decisions use their existing issue-review/continuation effects. Work-product create/update/delete/import uses its existing native status boundary. + +The new owners replace only these four startup and periodic calls. Deferred comments, silent active runs and residual orphan/resource recovery retain their existing backstop until the next slice. Every previously migrated native-owner skip remains unchanged. + +A controlled process test prints an execution proof but cannot apply a task disposition. The test operator completes that task through issue service. If the existing continuation policy already saved a disposition-repair retry, the test preserves it until its stored deadline. Only after all provider work has joined does it advance the clock; dispatch then revalidates the completed task and cancels the retry. The final quiet proof requires no registered deadlines, no tracked in-flight work and no owner SQL over twenty former polling minutes. It does not claim a saved future retry is already drained. diff --git a/server/src/__tests__/issue-recovery-deadlines.test.ts b/server/src/__tests__/issue-recovery-deadlines.test.ts new file mode 100644 index 00000000000..4faac4ab535 --- /dev/null +++ b/server/src/__tests__/issue-recovery-deadlines.test.ts @@ -0,0 +1,237 @@ +import { idleWorkSnapshot } from "../services/task-admission.js"; +import { randomUUID } from "node:crypto"; +import { and, eq, sql } from "drizzle-orm"; +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; +import { agents, authUsers, companies, companyMemberships, createDb, heartbeatRuns, issues, issueWatchdogs } from "@paperclipai/db"; +import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js"; +import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; +import { heartbeatService } from "../services/heartbeat.js"; +import { issueService } from "../services/issues.js"; +import { taskWatchdogService } from "../services/task-watchdogs.js"; +import { createDeliveryWorkCoordinator } from "../services/delivery-work-coordinator.js"; + +const external = resolveExternalTestDatabaseUrl("PAPERCLIP_ISSUE_RECOVERY_TEST_DATABASE_URL"); +const support = external ? { supported: true } : await getEmbeddedPostgresTestSupport(); +describe.skipIf(!support.supported)("issue recovery production producers and deadlines", () => { + let database: Awaited> | undefined; + let db: ReturnType; + let heartbeat: ReturnType; + let coordinator: ReturnType; + let worker: ReturnType["registerIssueStateRecoveryWork"]>; + let dispatch: ReturnType["registerRunDispatchWork"]>; + let cancellation: ReturnType["registerScheduledRetryCancellationWork"]>; + let draining = false; + const tasks = new Map number | null; hasPending: () => Promise }>(); + const errors: unknown[] = []; + const extraStops: Array<() => Promise> = []; + const extraDeadlines: Array<() => number | null> = []; + beforeAll(async () => { + if (!external) database = await startEmbeddedPostgresTestDatabase("issue-recovery-work-"); + db = createDb(external ?? database!.connectionString, { maxConnections: 6, idleTimeoutSeconds: 0, maxLifetimeSeconds: 0 }); + await Promise.all(Array.from({ length: 6 }, () => db.execute(sql`select pg_sleep(0.01)`))); + }, 90_000); + beforeEach(async () => { + await db.execute(sql`TRUNCATE companies CASCADE`); + heartbeat = heartbeatService(db); draining = false; errors.length = 0; + }); + afterEach(async () => { + for (const stop of extraStops.splice(0)) await stop(); + extraDeadlines.length = 0; + await worker?.stop(); await dispatch?.stop(); await cancellation?.stop(); await coordinator?.stop(); + await heartbeat.drainActiveRunExecutions(); + vi.useRealTimers(); vi.restoreAllMocks(); + expect(errors).toEqual([]); + }); + afterAll(async () => { await db?.$client.end({ timeout: 1 }); await database?.cleanup(); }); + async function start() { + coordinator = createDeliveryWorkCoordinator({ owner: db, canRun: () => !draining, canReconcile: () => true, + onError: error => errors.push(error) }); + tasks.clear(); + const register = coordinator.register.bind(coordinator); + coordinator.register = (queue, task) => { tasks.set(queue, task); return register(queue, task); }; + worker = heartbeat.registerIssueStateRecoveryWork(coordinator, () => !draining); + dispatch = heartbeat.registerRunDispatchWork(coordinator, () => !draining); + cancellation = heartbeat.registerScheduledRetryCancellationWork(coordinator, () => !draining); + await Promise.all([worker.ready, dispatch.ready, cancellation.ready]); + } + async function compete() { + const other = heartbeatService(db); + const competing = createDeliveryWorkCoordinator({ owner: db, canRun: () => true, onError: error => errors.push(error) }); + const work = other.registerIssueStateRecoveryWork(competing, () => true); + extraStops.push(() => competing.stop()); extraDeadlines.push(competing.nextWakeAt); await work.ready; + } + async function fixture() { + const companyId = randomUUID(); + await db.insert(authUsers).values({ id: "recovery-user", name: "Recovery operator", email: "recovery-user@example.test", createdAt: new Date(), updatedAt: new Date() }).onConflictDoNothing(); + await db.insert(companies).values({ id: companyId, name: "Recovery", issuePrefix: `R${companyId.slice(0, 7)}`, + requireBoardApprovalForNewAgents: false, defaultResponsibleUserId: "recovery-user" }); + await db.insert(companyMemberships).values({ companyId, principalId: "recovery-user", principalType: "user", status: "active", membershipRole: "owner" }); + const [agent] = await db.insert(agents).values({ companyId, name: "Recovery process", role: "engineer", status: "idle", + adapterType: "process", adapterConfig: { command: process.execPath, args: ["-e", "console.log('recovery-proof')"], cwd: process.cwd() }, + runtimeConfig: { heartbeat: { enabled: true, intervalSec: 0, wakeOnDemand: true } }, + }).returning(); + return { companyId, agentId: agent.id }; + } + async function settled() { + await vi.waitFor(async () => { + await heartbeat.drainActiveRunExecutions(); + const wake = coordinator.nextWakeAt(); + const detail = wake === null ? "" : JSON.stringify({ now: Date.now(), tasks: await Promise.all([...tasks].map(async ([queue, task]) => ({ queue, at: task.nextRunAt?.(), pending: await task.hasPending() }))), + runs: (await db.select().from(heartbeatRuns)).map(run => ({ id: run.id, status: run.status, error: run.errorCode, lease: run.controllerLeaseExpiresAt, reason: run.contextSnapshot?.wakeReason })) }); + expect(wake, detail).toBeNull(); + expect(extraDeadlines.map(next => next())).toEqual(extraDeadlines.map(() => null)); + expect(idleWorkSnapshot().active).toBe(0); + }, { timeout: 10_000 }); + } + async function quiet() { + await settled(); + const reads = vi.spyOn(db, "select"); const executes = vi.spyOn(db, "execute"); + vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"] }); + try { await vi.advanceTimersByTimeAsync(20 * 60_000); expect(reads).not.toHaveBeenCalled(); expect(executes).not.toHaveBeenCalled(); } + finally { vi.useRealTimers(); reads.mockRestore(); executes.mockRestore(); } + } + async function finish(issueId: string) { + await issueService(db).update(issueId, { status: "done" }); + await heartbeat.drainActiveRunExecutions(); + const saved = await db.select().from(heartbeatRuns).where(and(eq(heartbeatRuns.status, "scheduled_retry"), + sql`${heartbeatRuns.contextSnapshot}->>'issueId' = ${issueId}`)); + if (saved.length > 0) { + const due = Math.max(...saved.map(run => run.scheduledRetryAt!.getTime())); + expect(await db.select().from(heartbeatRuns).where(and( + sql`${heartbeatRuns.status} in ('queued', 'running')`, + sql`${heartbeatRuns.contextSnapshot}->>'issueId' = ${issueId}`))).toHaveLength(0); + vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"] }); + await vi.advanceTimersByTimeAsync(Math.max(0, due - Date.now()) + 1); + await vi.waitFor(async () => { + const remaining = await db.select().from(heartbeatRuns).where(and(eq(heartbeatRuns.status, "scheduled_retry"), + sql`${heartbeatRuns.contextSnapshot}->>'issueId' = ${issueId}`)); + expect(remaining).toHaveLength(0); + }); + for (const retry of saved) { + const [settledRetry] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, retry.id)); + expect(settledRetry.status).toBe("cancelled"); + } + } + } + async function completed(issueId: string) { + let runId: string | undefined; + await vi.waitFor(async () => { + const runs = await db.select().from(heartbeatRuns).where(sql`${heartbeatRuns.contextSnapshot}->>'issueId' = ${issueId}`); + const run = runs.find(row => row.status === "running" || row.status === "succeeded"); + expect(run).toBeDefined(); runId = run!.id; + }, { timeout: 15_000 }); + await heartbeat.drainActiveRunExecutions(); + const [run] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, runId!)); + expect(run.status, JSON.stringify({ error: run.error, code: run.errorCode, stderr: run.stderrExcerpt, result: run.resultJson })).toBe("succeeded"); expect(run.stdoutExcerpt).toContain("recovery-proof"); + return run; + } + it("keeps empty, configured idle, and terminal history quiet", async () => { + await fixture(); await start(); await quiet(); + }); + it("dispatches an ordinary new assignment after commit and returns to quiet after completion", async () => { + const f = await fixture(); await start(); await quiet(); + const issue = await issueService(db).create(f.companyId, { title: "Recover assignment", status: "todo", assigneeAgentId: f.agentId }); + await completed(issue.id); + await finish(issue.id); await quiet(); + }, 45_000); + it("reacts to blocker resolution once and dispatches the durable dependency wake", async () => { + const f = await fixture(); + const blocker = await issueService(db).create(f.companyId, { title: "Blocker", status: "todo" }); + const dependent = await issueService(db).create(f.companyId, { title: "Dependent", status: "blocked", assigneeAgentId: f.agentId, + blockedByIssueIds: [blocker.id] }); + await start(); await quiet(); + await compete(); + await issueService(db).update(blocker.id, { status: "done" }); + const run = await completed(dependent.id); + expect(run.contextSnapshot?.wakeReason).toBe("issue_blockers_resolved"); + await finish(dependent.id); await quiet(); + expect(await db.select().from(heartbeatRuns).where(and(eq(heartbeatRuns.companyId, f.companyId), + sql`${heartbeatRuns.contextSnapshot}->>'wakeReason' = 'issue_blockers_resolved'`))).toHaveLength(1); + }, 45_000); + it("uses ordinary watchdog configuration and subtree closure to dispatch a reusable review", async () => { + const f = await fixture(); + const source = await issueService(db).create(f.companyId, { title: "Finished subtree", status: "done" }); + await start(); await quiet(); + await compete(); + const watchdog = await taskWatchdogService(db).upsertForIssue(f.companyId, source.id, { agentId: f.agentId }); + let reviewId: string | undefined; + await vi.waitFor(async () => { + const [row] = await db.select().from(issueWatchdogs).where(eq(issueWatchdogs.id, watchdog.watchdog.id)); + expect(row.watchdogIssueId).toBeTruthy(); reviewId = row.watchdogIssueId!; + }, { timeout: 15_000 }); + await completed(reviewId!); + expect(await db.select().from(heartbeatRuns).where(and(eq(heartbeatRuns.companyId, f.companyId), + sql`${heartbeatRuns.contextSnapshot}->>'wakeReason' = 'task_watchdog_stopped_subtree'`))).toHaveLength(1); + await finish(reviewId!); + await taskWatchdogService(db).disableForIssue(f.companyId, source.id); await quiet(); + }, 45_000); + it("bounds ordinary writes to their company and reconstructs foreign stale debt after restart", async () => { + const selected = await fixture(); await start(); await quiet(); + const foreign = await fixture(); + const [run] = await db.insert(heartbeatRuns).values({ companyId: foreign.companyId, agentId: foreign.agentId, + status: "failed", finishedAt: new Date() }).returning(); + const [stale] = await db.insert(issues).values({ companyId: foreign.companyId, title: "Foreign stale lock", status: "done", + checkoutRunId: run.id }).returning(); + await issueService(db).create(selected.companyId, { title: "Selected change", status: "done" }); await quiet(); + expect((await db.select().from(issues).where(eq(issues.id, stale.id)))[0].checkoutRunId).toBe(run.id); + await worker.stop(); await dispatch.stop(); await cancellation.stop(); await coordinator.stop(); + heartbeat = heartbeatService(db); await start(); await quiet(); + expect((await db.select().from(issues).where(eq(issues.id, stale.id)))[0].checkoutRunId).toBeNull(); + }); + it("fences nested assignment writes until outer commit and ignores rollback", async () => { + const f = await fixture(); await start(); await quiet(); + let committedId = ""; + await db.transaction(async outer => { + await outer.transaction(async nested => { + const issue = await issueService(db).create(f.companyId, { title: "Nested commit", status: "todo", assigneeAgentId: f.agentId }, nested); + committedId = issue.id; + }); + await new Promise(resolve => setTimeout(resolve, 100)); + expect(await db.select().from(heartbeatRuns)).toHaveLength(0); + }); + await completed(committedId); await finish(committedId); await settled(); + await expect(db.transaction(async outer => { + await outer.transaction(async nested => { + await issueService(db).create(f.companyId, { title: "Nested rollback", status: "todo", assigneeAgentId: f.agentId }, nested); + }); + throw new Error("rollback control"); + })).rejects.toThrow("rollback control"); + await quiet(); + expect(await db.select().from(issues).where(eq(issues.title, "Nested rollback"))).toHaveLength(0); + }, 60_000); + it("retains the first committed assignment during drain and executes on resume", async () => { + const f = await fixture(); await start(); await quiet(); draining = true; + const issue = await issueService(db).create(f.companyId, { title: "Drained assignment", status: "todo", assigneeAgentId: f.agentId }); + await new Promise(resolve => setTimeout(resolve, 100)); + const reads = vi.spyOn(db, "select"); + vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"] }); + try { await vi.advanceTimersByTimeAsync(90_000); expect(reads).not.toHaveBeenCalled(); } + finally { vi.useRealTimers(); reads.mockRestore(); } + expect(await db.select().from(heartbeatRuns)).toHaveLength(0); + draining = false; worker.wake(); dispatch.wake(); + await completed(issue.id); await finish(issue.id); await quiet(); + }, 45_000); + it("retains a fresh subtree's exact first-run grace then disarms on disable", async () => { + const f = await fixture(); + const source = await issueService(db).create(f.companyId, { title: "Fresh unassigned subtree", status: "todo" }); + await start(); await quiet(); + await taskWatchdogService(db).upsertForIssue(f.companyId, source.id, { agentId: f.agentId }); + await vi.waitFor(() => expect(coordinator.nextWakeAt()).toBe(source.createdAt.getTime() + 15_000), { timeout: 5_000 }); + expect(await db.select().from(issues).where(eq(issues.originKind, "task_watchdog"))).toHaveLength(0); + await taskWatchdogService(db).disableForIssue(f.companyId, source.id); await quiet(); + }); + it("waits for a live foreign controller's exact stored lease before clearing a terminal lock", async () => { + const f = await fixture(); + const source = await issueService(db).create(f.companyId, { title: "Remote owner", status: "done" }); + const lease = new Date(Date.now() + 5_000); + const [run] = await db.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: f.agentId, status: "running", + runtimeMode: "legacy", controllerBootId: randomUUID(), controllerLeaseExpiresAt: lease, + contextSnapshot: { issueId: source.id }, processPid: 2_000_000_000, startedAt: new Date() }).returning(); + await db.update(issues).set({ executionRunId: run.id }).where(eq(issues.id, source.id)); + await start(); + await vi.waitFor(() => expect(coordinator.nextWakeAt()).toBe(lease.getTime()), { timeout: 3_000 }); + expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, run.id)))[0].status).toBe("running"); + await vi.waitFor(async () => expect((await db.select().from(issues).where(eq(issues.id, source.id)))[0].executionRunId).toBeNull(), { timeout: 12_000 }); + expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, run.id)))[0].status).toBe("succeeded"); await quiet(); + }, 30_000); +}); diff --git a/server/src/__tests__/issue-recovery-work-lifecycle.test.ts b/server/src/__tests__/issue-recovery-work-lifecycle.test.ts new file mode 100644 index 00000000000..4d0647f4406 --- /dev/null +++ b/server/src/__tests__/issue-recovery-work-lifecycle.test.ts @@ -0,0 +1,99 @@ +import { PgDialect } from "drizzle-orm/pg-core"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { issueWatchdogs } from "@paperclipai/db"; +import { installDatabaseWorkSignals } from "../../../packages/db/src/work-signals.js"; +import { createDeliveryWorkCoordinator } from "../services/delivery-work-coordinator.js"; +import { notifyIssueRecovery } from "../services/issue-recovery-notifications.js"; +import { registerIssueRecoveryWork } from "../services/recovery/work.js"; +import { idleWorkSnapshot } from "../services/task-admission.js"; + +const stops: Array<() => Promise> = []; +afterEach(async () => { for (const stop of stops.splice(0)) await stop(); vi.useRealTimers(); }); +function fixture() { + vi.useFakeTimers(); vi.setSystemTime(1_000_000); + const dialect = new PgDialect(); + const rows = { issues: [] as Array<{ id: string; companyId: string }>, watchdogs: [] as Array<{ id: string; companyId: string }> }; + const execute = vi.fn(async (query: any) => { + if (dialect.sqlToQuery(query).sql.includes("pg_current_xact_id")) return [{ xid: "42" }]; + throw new Error("unexpected SQL"); + }); + const load = vi.fn((table: unknown) => table === issueWatchdogs ? rows.watchdogs : rows.issues); + const db: any = installDatabaseWorkSignals({ execute, + select: () => ({ from: (table: unknown) => ({ where: async () => load(table) }) }), + transaction: async (callback: (tx: any) => Promise) => callback({ execute, select: db.select, transaction: db.transaction }), + }); + const state = { draining: false, standby: false, suppressed: false, deadline: null as number | null, fail: false, incomplete: false }; + const check = vi.fn(async (scope: { onDeferredDeadline(at: number): void }) => { + if (state.fail) throw new Error("database unavailable"); + if (state.deadline !== null && state.deadline > Date.now()) scope.onDeferredDeadline(state.deadline); + }); + const owner = { stranded: check, dependencies: async (scope: any) => { await check(scope); return { enqueueFailed: 0 }; }, + locks: check, watchdogs: async (scope: any) => { await check(scope); return { incomplete: state.incomplete ? 1 : 0 }; }, + schedulingSuppressed: async () => state.suppressed }; + const coordinator = createDeliveryWorkCoordinator({ owner: db, canRun: () => !state.standby, + canReconcile: () => !state.standby, onError: vi.fn() }); + const worker = registerIssueRecoveryWork(db, coordinator, owner, () => !state.draining && !state.standby); + stops.push(() => coordinator.stop()); + return { db, rows, state, check, load, coordinator, worker, + notify: () => db.transaction((tx: any) => notifyIssueRecovery(tx, { companyId: "selected" })), + settle: () => vi.advanceTimersByTimeAsync(1_000) }; +} +describe("issue recovery obligation lifecycle", () => { + it("has no timer, SQL, or owner probe after empty reconstruction", async () => { + const f = fixture(); await f.worker.ready; + expect(f.coordinator.nextWakeAt()).toBeNull(); const loaded = f.load.mock.calls.length; + await vi.advanceTimersByTimeAsync(20 * 60_000); + expect(f.load).toHaveBeenCalledTimes(loaded); expect(f.check).not.toHaveBeenCalled(); + }); + it("retains exact grace and lease deadlines without early owner probes", async () => { + const f = fixture(); f.rows.issues.push({ id: "issue", companyId: "selected" }); + f.rows.watchdogs.push({ id: "watchdog", companyId: "selected" }); f.state.deadline = 1_015_000; + await f.worker.ready; expect(f.coordinator.nextWakeAt()).toBe(1_015_000); + const checked = f.check.mock.calls.length; + await vi.advanceTimersByTimeAsync(14_999); expect(f.check).toHaveBeenCalledTimes(checked); + await vi.advanceTimersByTimeAsync(1); expect(f.check.mock.calls.length).toBe(checked * 2); + expect(f.coordinator.nextWakeAt()).toBeNull(); + }); + it("releases settled producer holds during domain drain while coordinator remains open", async () => { + const f = fixture(); await f.worker.ready; f.state.draining = true; + f.rows.issues.push({ id: "issue", companyId: "selected" }); + await f.notify(); await f.settle(); const loaded = f.load.mock.calls.length; + await vi.advanceTimersByTimeAsync(90_000); + expect(f.load).toHaveBeenCalledTimes(loaded); expect(f.check).not.toHaveBeenCalled(); + expect(idleWorkSnapshot().active).toBe(0); + f.state.draining = false; f.worker.wake(); await f.settle(); + expect(f.check).toHaveBeenCalled(); expect(f.coordinator.nextWakeAt()).toBeNull(); + }); + it("retries concrete failed entries and then disarms without reloading the company", async () => { + const f = fixture(); f.rows.issues.push({ id: "issue", companyId: "selected" }); f.state.fail = true; + await f.worker.ready; const loaded = f.load.mock.calls.length; + expect(f.coordinator.nextWakeAt()).toBe(1_030_000); + await vi.advanceTimersByTimeAsync(29_999); const checked = f.check.mock.calls.length; + f.state.fail = false; await vi.advanceTimersByTimeAsync(1); + expect(f.check.mock.calls.length).toBe(checked * 2); expect(f.load).toHaveBeenCalledTimes(loaded); + expect(f.coordinator.nextWakeAt()).toBeNull(); + }); + it("keeps an incomplete watchdog subtree as concrete retry debt", async () => { + const f = fixture(); f.rows.watchdogs.push({ id: "watchdog", companyId: "selected" }); f.state.incomplete = true; + await f.worker.ready; expect(f.coordinator.nextWakeAt()).toBe(1_030_000); + const loaded = f.load.mock.calls.length; f.state.incomplete = false; + await vi.advanceTimersByTimeAsync(30_000); + expect(f.check).toHaveBeenCalledTimes(2); expect(f.load).toHaveBeenCalledTimes(loaded); + expect(f.coordinator.nextWakeAt()).toBeNull(); + }); + it("keeps a concurrent producer dirty until the in-flight owner has completed", async () => { + const f = fixture(); f.rows.issues.push({ id: "issue", companyId: "selected" }); await f.worker.ready; + let release!: () => void; const held = new Promise(resolve => { release = resolve; }); + f.check.mockImplementationOnce(() => held); + await f.notify(); await f.settle(); await f.notify(); release(); await f.settle(); await f.settle(); + expect(f.coordinator.nextWakeAt()).toBeNull(); + expect(f.check.mock.calls.length).toBeGreaterThan(3); + }); + it("retains startup work on standby and cancels future deadlines on shutdown", async () => { + const f = fixture(); f.state.standby = true; f.rows.watchdogs.push({ id: "watchdog", companyId: "selected" }); + await f.worker.ready; await vi.advanceTimersByTimeAsync(90_000); expect(f.load).not.toHaveBeenCalled(); + f.state.standby = false; f.state.deadline = Date.now() + 60_000; f.worker.wake(); await f.settle(); + expect(f.check).toHaveBeenCalledOnce(); await f.worker.stop(); + await vi.advanceTimersByTimeAsync(120_000); expect(f.check).toHaveBeenCalledOnce(); + }); +}); diff --git a/server/src/__tests__/server-startup-feedback-export.test.ts b/server/src/__tests__/server-startup-feedback-export.test.ts index e00c8519df7..a26d94f50d4 100644 --- a/server/src/__tests__/server-startup-feedback-export.test.ts +++ b/server/src/__tests__/server-startup-feedback-export.test.ts @@ -134,6 +134,7 @@ const { })), reconcileHotRestartAdoption: vi.fn(async () => ({ mode: "none" })), reapOrphanedRuns: vi.fn(async () => ({ reaped: 0, runIds: [] })), + registerIssueStateRecoveryWork: vi.fn((_coordinator: unknown, _canRun: () => boolean) => ({ ready: Promise.resolve() })), registerRunDispatchWork: vi.fn((_coordinator: unknown, _canRun: () => boolean) => ({ ready: Promise.resolve() })), registerScheduledRetryCancellationWork: vi.fn((_coordinator: unknown, _canRun: () => boolean) => ({ ready: Promise.resolve() })), registerStoppedInstructionCollectionWork: vi.fn((_coordinator: unknown, _canRun: () => boolean) => ({ ready: Promise.resolve() })), @@ -927,6 +928,7 @@ describe("startServer feedback export wiring", () => { it.each([false, true])("registers retry and queued dispatch only when maintenance is enabled (%s)", async enabled => { loadConfigMock.mockReturnValue(buildTestConfig({ heartbeatSchedulerEnabled: enabled })); await startServer(); await settleStartupHousekeeping(); + expect(heartbeatServiceMock.registerIssueStateRecoveryWork).toHaveBeenCalledTimes(enabled ? 1 : 0); expect(heartbeatServiceMock.registerRunDispatchWork).toHaveBeenCalledTimes(enabled ? 1 : 0); if (enabled) { const [coordinator, admitted] = heartbeatServiceMock.registerRunDispatchWork.mock.calls[0]; @@ -1334,11 +1336,11 @@ describe("startServer feedback export wiring", () => { await startServer(); await settleStartupHousekeeping(); expect(heartbeatServiceMock.reconcileCostAccounting).toHaveBeenCalledTimes(1); - expect(heartbeatServiceMock.sweepStaleIssueLocks).toHaveBeenCalledTimes(1); + expect(heartbeatServiceMock.sweepStaleIssueLocks).not.toHaveBeenCalled(); expect(intervalCallback).not.toBeNull(); intervalCallback?.(); await new Promise((resolve) => setImmediate(resolve)); - expect(heartbeatServiceMock.sweepStaleIssueLocks).toHaveBeenCalledTimes(2); + expect(heartbeatServiceMock.sweepStaleIssueLocks).not.toHaveBeenCalled(); expect(heartbeatServiceMock.reapOrphanedRuns).toHaveBeenLastCalledWith({ staleThresholdMs: 300_000, skipScheduledRetryCancellation: true, skipCapturedInstructionCommit: true, skipInstructionDirectoryCleanup: true, skipStoppedInstructionCollection: true, skipCancelledWorkspacePreservation: true, skipObsoletePolicyReviews: true, skipAutomaticCompletionReviews: true, skipUnsafeWorkspaceExports: true, skipNativePendingEffects: true, skipNativeResponsePresentation: true, skipNativeReviewResponse: true, skipNativeStatusReassessment: true, skipRetainedNativeSessionCleanup: true, skipNativeWorkspaceFinalization: true, skipNativeResultCommit: true, skipNativeRunProjection: true, skipNativeRecoveryNotices: true, skipExternalConversationSettlement: true, skipExternalAgentSettlement: true, skipNativeEnvironmentSettlement: true }); expect(heartbeatServiceMock.reconcileCostAccounting).toHaveBeenCalledTimes(1); expect(heartbeatServiceMock.promoteDueScheduledRetries).not.toHaveBeenCalled(); diff --git a/server/src/index.ts b/server/src/index.ts index cd8bb951946..0ebe5c3332c 100644 --- a/server/src/index.ts +++ b/server/src/index.ts @@ -109,7 +109,6 @@ import { createApp } from "./app.js"; import { initializeCloudRuntimeConfig } from "./services/cloud-runtime-config.js"; import { loadCloudBootstrapConfig, loadConfig, type Config } from "./config.js"; import { logger } from "./middleware/logger.js"; -import { createIncompleteTaskWatchdogScanReporter } from "./services/task-watchdog-diagnostics.js"; import { setStartupRecoveryPhase } from "./startup-recovery-state.js"; import { StartupRefusalError, @@ -973,9 +972,6 @@ async function startServerWithDatabaseTeardown( } }; const pluginWorkerManager = createPluginWorkerManager(); - const reportIncompleteWatchdogScans = createIncompleteTaskWatchdogScanReporter( - (details, message) => logger.warn(details, message), - ); const heartbeat = config.heartbeatSchedulerEnabled ? heartbeatService(db as any, { pluginWorkerManager }) : null; @@ -1796,47 +1792,10 @@ async function startServerWithDatabaseTeardown( "startup session-goal recovery resumed durable agent goals", ); } - const reconciled = await heartbeat.reconcileStrandedAssignedIssues(); - if ( - reconciled.assignmentDispatched > 0 || - reconciled.dispatchRequeued > 0 || - reconciled.continuationRequeued > 0 || - reconciled.successfulRunHandoffEscalated > 0 || - reconciled.successfulRunHandoffRetried > 0 || - reconciled.escalated > 0 - ) { - logger.warn( - { ...reconciled }, - "startup heartbeat recovery changed assigned issue state", - ); - } - - const dependencyWakesReconciled = await heartbeat.reconcileResolvedDependencyWakes(); - if (dependencyWakesReconciled.healed > 0) { - logger.warn( - { ...dependencyWakesReconciled }, - "startup dependency-wake reconciliation restored task execution paths", - ); - } - - const taskWatchdogsReconciled = await heartbeat.reconcileTaskWatchdogs(); - reportIncompleteWatchdogScans(taskWatchdogsReconciled, "startup"); - if (taskWatchdogsReconciled.triggered > 0) { - logger.warn( - { ...taskWatchdogsReconciled }, - "startup task-watchdog reconciliation triggered watchdog work", - ); - } - const scanned = await heartbeat.scanSilentActiveRuns(); if (scanned.created > 0 || scanned.escalated > 0) { logger.warn({ ...scanned }, "startup active-run output watchdog created review work"); } - - const swept = await heartbeat.sweepStaleIssueLocks(); - if (swept.cleared > 0) { - logger.warn({ ...swept }, "startup stale-lock sweeper cleared issue locks"); - } })().catch((err) => { logger.error({ err }, "startup heartbeat recovery failed"); throw err; @@ -1845,6 +1804,8 @@ async function startServerWithDatabaseTeardown( await startupHeartbeatRecovery; } + if (config.heartbeatSchedulerEnabled) await heartbeat.registerIssueStateRecoveryWork(deliveryWork, + () => !isIdleTaskDrainActive() && !isWarmStandby() && !heartbeatSchedulerStopped).ready; if (config.heartbeatSchedulerEnabled) await heartbeat.registerRunDispatchWork(deliveryWork, () => !isIdleTaskDrainActive() && !isWarmStandby() && !heartbeatSchedulerStopped).ready; if (config.heartbeatSchedulerEnabled) await heartbeat.registerScheduledRetryCancellationWork(deliveryWork, @@ -2051,33 +2012,6 @@ async function startServerWithDatabaseTeardown( .reapOrphanedRuns({ staleThresholdMs: 5 * 60 * 1000, skipScheduledRetryCancellation: true, skipCapturedInstructionCommit: true, skipInstructionDirectoryCleanup: true, skipStoppedInstructionCollection: true, skipCancelledWorkspacePreservation: true, skipObsoletePolicyReviews: true, skipAutomaticCompletionReviews: true, skipUnsafeWorkspaceExports: true, skipNativePendingEffects: true, skipNativeResponsePresentation: true, skipNativeReviewResponse: true, skipNativeStatusReassessment: true, skipRetainedNativeSessionCleanup: true, skipNativeWorkspaceFinalization: true, skipNativeResultCommit: true, skipNativeRunProjection: true, skipNativeRecoveryNotices: true, skipExternalConversationSettlement: true, skipExternalAgentSettlement: true, skipNativeEnvironmentSettlement: true }) .then(async () => { await heartbeat.resumeQueuedRuns(); - const reconciled = await heartbeat.reconcileStrandedAssignedIssues({ skipExternalConversationSettlement: true }); - if ( - reconciled.assignmentDispatched > 0 || - reconciled.dispatchRequeued > 0 || - reconciled.continuationRequeued > 0 || - reconciled.successfulRunHandoffEscalated > 0 || - reconciled.successfulRunHandoffRetried > 0 || - reconciled.escalated > 0 - ) { - logger.warn( - { ...reconciled }, - "periodic heartbeat recovery changed assigned issue state", - ); - } - }) - .then(async () => { - const reconciled = await heartbeat.reconcileResolvedDependencyWakes(); - if (reconciled.healed > 0) { - logger.warn({ ...reconciled }, "periodic dependency-wake reconciliation restored task execution paths"); - } - }) - .then(async () => { - const reconciled = await heartbeat.reconcileTaskWatchdogs(); - reportIncompleteWatchdogScans(reconciled, "periodic"); - if (reconciled.triggered > 0) { - logger.warn({ ...reconciled }, "periodic task-watchdog reconciliation triggered watchdog work"); - } }) .then(async () => { const scanned = await heartbeat.scanSilentActiveRuns(); @@ -2085,12 +2019,6 @@ async function startServerWithDatabaseTeardown( logger.warn({ ...scanned }, "periodic active-run output watchdog created review work"); } }) - .then(async () => { - const swept = await heartbeat.sweepStaleIssueLocks(); - if (swept.cleared > 0) { - logger.warn({ ...swept }, "periodic stale-lock sweeper cleared issue locks"); - } - }) .catch((err) => { logger.error({ err }, "periodic heartbeat recovery failed"); })); diff --git a/server/src/services/agent-conversations.ts b/server/src/services/agent-conversations.ts index af44a0355d8..a10c6ee5eb6 100644 --- a/server/src/services/agent-conversations.ts +++ b/server/src/services/agent-conversations.ts @@ -1,3 +1,4 @@ +import { notifyIssueRecovery } from "./issue-recovery-notifications.js"; import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js"; import { notifyUnsafeWorkspaceExportRecovery } from "./unsafe-workspace-export-notifications.js"; import { FAST_RESPONSE_AGENT_GUIDANCE, fastResponseHistoryBody } from "@paperclipai/shared"; @@ -544,6 +545,7 @@ export async function resumeConversationForReset(db: Db, comment: typeof issueCo eq(issues.id, comment.issueId), eq(issues.companyId, comment.companyId), )).for("update"); if (!isConversation(issue) || comment.conversationSessionGeneration != null) return; + await notifyIssueRecovery(tx, { companyId: issue.companyId }); const released = await tx.update(issueTreeHolds).set({ status: "released", releasedAt: new Date(), updatedAt: new Date(), releasedByActorType: "user", releasedByUserId: comment.authorUserId, diff --git a/server/src/services/chat-channels/telegram/attachment-failure.ts b/server/src/services/chat-channels/telegram/attachment-failure.ts index 58d5d4a2677..785b3116baf 100644 --- a/server/src/services/chat-channels/telegram/attachment-failure.ts +++ b/server/src/services/chat-channels/telegram/attachment-failure.ts @@ -1,3 +1,4 @@ +import { notifyIssueRecovery } from "../../issue-recovery-notifications.js"; import { createProviderEffects, type ProviderEffectTarget } from "../provider-effects.js"; import type { EndpointRow, RuntimeContext, InboundRuntimeContext } from "../runtime-types.js"; @@ -78,6 +79,7 @@ export function createTelegramAttachmentFailure(db: Db, options: TelegramAttachm })(), ); const effect = await db.transaction(async (tx) => { + await notifyIssueRecovery(tx, { companyId: input.endpoint.companyId }); // Replace the optimistic attachment placeholder with an explicit safe // Paperclip-visible failure. The message link stays authoritative for // exact provider redelivery, while no agent wakeup is enqueued for an diff --git a/server/src/services/company-deletion.ts b/server/src/services/company-deletion.ts index 7ab88aadfa9..26bc4e081a1 100644 --- a/server/src/services/company-deletion.ts +++ b/server/src/services/company-deletion.ts @@ -1,3 +1,4 @@ +import { notifyIssueRecovery } from "./issue-recovery-notifications.js"; import { notifyUnsafeWorkspaceExportRecovery } from "./unsafe-workspace-export-notifications.js"; import { notifyCancelledWorkspacePreservation } from "./cancelled-workspace-preservation-notifications.js"; import { notifyInstructionCollection } from "./instruction-collection-notifications.js"; @@ -71,6 +72,7 @@ export async function deleteCompany(db: Db, id: string) { const [existing] = await tx.select({ id: companies.id }).from(companies) .where(eq(companies.id, id)).for("no key update"); if (!existing) return null; + await notifyIssueRecovery(tx, { companyId: id }); await notifyChatCompanyConfiguration(tx, id); await chatCompanyDeletion.deleteCompanyData(tx, id); await notifyDeliveryWork(tx, DELIVERY_QUEUES.skillSourceSync); diff --git a/server/src/services/company-portability.ts b/server/src/services/company-portability.ts index dc7dca311d6..585b8fcf5f4 100644 --- a/server/src/services/company-portability.ts +++ b/server/src/services/company-portability.ts @@ -1,3 +1,4 @@ +import { notifyIssueRecovery } from "./issue-recovery-notifications.js"; import { createAgentLifecycle } from "./agent-lifecycle.js"; import { agentAppearanceSchema, agentHarnessType, paperclipRunnerProfileForHarness, agentRunner, type AgentRunnerChoice } from "@paperclipai/shared"; import { resolveNewAgentRunnerForCompany } from "./agent-runner-selection.js"; @@ -6465,7 +6466,9 @@ export function companyPortabilityService(db: Db, storage?: StorageService) { } if (relationRows.length > 0) { const relationCompanyId = targetCompany.id; - await db + await db.transaction(async tx => { + await notifyIssueRecovery(tx, { companyId: relationCompanyId }); + await tx .insert(issueRelations) .values(relationRows.map((row) => ({ companyId: relationCompanyId, @@ -6476,6 +6479,7 @@ export function companyPortabilityService(db: Db, storage?: StorageService) { createdByUserId: actorUserId ?? null, }))) .onConflictDoNothing(); + }); } } diff --git a/server/src/services/documents.ts b/server/src/services/documents.ts index 493c495e007..814c68eeea7 100644 --- a/server/src/services/documents.ts +++ b/server/src/services/documents.ts @@ -1,3 +1,4 @@ +import { notifyIssueRecovery } from "./issue-recovery-notifications.js"; import { randomUUID } from "node:crypto"; import { and, asc, desc, eq, sql } from "drizzle-orm"; import type { Db } from "@paperclipai/db"; @@ -275,6 +276,7 @@ export function documentService(db: Db) { .where(eq(issueDocuments.issueId, issue.id)); const fallbackKey = nextAvailableDocumentKey(key, issueDocumentKeys.map((row) => row.key)); + await notifyIssueRecovery(tx, { companyId: issue.companyId }); const [document] = await tx .insert(documents) .values({ @@ -376,6 +378,7 @@ export function documentService(db: Db) { }); } + await notifyIssueRecovery(tx, { companyId: issue.companyId }); const nextRevisionNumber = existing.latestRevisionNumber + 1; const [revision] = await tx .insert(documentRevisions) @@ -438,6 +441,7 @@ export function documentService(db: Db) { throw conflict("Document does not exist yet", { key }); } + await notifyIssueRecovery(tx, { companyId: issue.companyId }); const [document] = await tx .insert(documents) .values({ @@ -592,6 +596,9 @@ export function documentService(db: Db) { }); } await db.transaction(async (tx) => { + for (const companyId of new Set(rows.map(row => row.companyId))) { + await notifyIssueRecovery(tx, { companyId }); + } await insertRowsInChunks(tx, documents, documentRows); await insertRowsInChunks(tx, documentRevisions, revisionRows); await insertRowsInChunks(tx, issueDocuments, linkRows); @@ -644,6 +651,7 @@ export function documentService(db: Db) { }); } + await notifyIssueRecovery(tx, { companyId: existing.companyId }); const now = new Date(); const nextRevisionNumber = existing.latestRevisionNumber + 1; const [restoredRevision] = await tx @@ -722,6 +730,7 @@ export function documentService(db: Db) { }; } + await notifyIssueRecovery(tx, { companyId: existing.companyId }); const now = new Date(); await tx .update(documents) @@ -769,6 +778,7 @@ export function documentService(db: Db) { }; } + await notifyIssueRecovery(tx, { companyId: existing.companyId }); const now = new Date(); await tx .update(documents) @@ -817,6 +827,7 @@ export function documentService(db: Db) { }); } + await notifyIssueRecovery(tx, { companyId: existing.companyId }); await tx.delete(issueDocuments).where(eq(issueDocuments.documentId, existing.id)); await tx.delete(documents).where(eq(documents.id, existing.id)); diff --git a/server/src/services/external-conversation-settlement-notifications.ts b/server/src/services/external-conversation-settlement-notifications.ts index 46f4493ff2a..d5e4cdd506c 100644 --- a/server/src/services/external-conversation-settlement-notifications.ts +++ b/server/src/services/external-conversation-settlement-notifications.ts @@ -1,7 +1,9 @@ +import { notifyIssueRecovery } from "./issue-recovery-notifications.js"; import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js"; import type { ExternalConversationSettlementScope } from "./slack-conversation-lifecycle.js"; -export function notifyExternalConversationSettlement(tx: object, scope?: ExternalConversationSettlementScope) { +export async function notifyExternalConversationSettlement(tx: object, scope?: ExternalConversationSettlementScope) { + await notifyIssueRecovery(tx, scope); return notifyDeliveryWork(tx, DELIVERY_QUEUES.externalConversationSettlement, scope ? JSON.stringify({ companyId: scope.companyId, issueId: scope.issueId }) : "*"); } diff --git a/server/src/services/issue-recovery-actions.ts b/server/src/services/issue-recovery-actions.ts index 48fb001c0b4..316eb413afe 100644 --- a/server/src/services/issue-recovery-actions.ts +++ b/server/src/services/issue-recovery-actions.ts @@ -1,3 +1,4 @@ +import { notifyIssueRecovery } from "./issue-recovery-notifications.js"; import { notifyNativeRecoveryNotices } from "./native-recovery-notice-notifications.js"; import { notifyUnsafeWorkspaceExportRecovery } from "./unsafe-workspace-export-notifications.js"; import { writeExecutionControl } from "./execution-control-notifications.js"; @@ -322,6 +323,7 @@ export function issueRecoveryActionService(db: DbOrTransaction) { ): Promise { try { const created = await db.transaction(async (tx) => { + await notifyIssueRecovery(tx, { companyId: input.companyId }); await notifyNativeRecoveryNotices(tx, { companyId: input.companyId, issueId: input.sourceIssueId }); await notifyDeliveryWork(tx, DELIVERY_QUEUES.executionDisposition); await notifyDeliveryWork(tx, DELIVERY_QUEUES.executionContinuation); @@ -413,6 +415,7 @@ export function issueRecoveryActionService(db: DbOrTransaction) { Math.min(nextAttemptCount, effectiveMaxAttempts), ); const [exhausted] = await writeExecutionControl(db, [DELIVERY_QUEUES.executionDisposition, DELIVERY_QUEUES.executionContinuation], async tx => { + await notifyIssueRecovery(tx, { companyId: input.companyId }); await notifyNativeRecoveryNotices(tx, { companyId: input.companyId, issueId: input.sourceIssueId }); if ((!input.preserveExistingOwner && existing.cause !== input.cause) || (input.evidence?.runId !== undefined && input.evidence.runId !== existing.evidence?.runId)) { @@ -474,6 +477,7 @@ export function issueRecoveryActionService(db: DbOrTransaction) { return toReadModel(exhausted); } const [updated] = await writeExecutionControl(db, [DELIVERY_QUEUES.executionDisposition, DELIVERY_QUEUES.executionContinuation], async tx => { + await notifyIssueRecovery(tx, { companyId: input.companyId }); await notifyNativeRecoveryNotices(tx, { companyId: input.companyId, issueId: input.sourceIssueId }); if ((!input.preserveExistingOwner && existing.cause !== input.cause) || (input.evidence?.runId !== undefined && input.evidence.runId !== existing.evidence?.runId)) { @@ -548,6 +552,7 @@ export function issueRecoveryActionService(db: DbOrTransaction) { try { const [created] = await writeExecutionControl(db, [DELIVERY_QUEUES.executionDisposition, DELIVERY_QUEUES.executionContinuation], async tx => { + await notifyIssueRecovery(tx, { companyId: input.companyId }); await notifyNativeRecoveryNotices(tx, { companyId: input.companyId, issueId: input.sourceIssueId }); await notifyUnsafeWorkspaceExportRecovery(tx, { companyId: input.companyId, issueId: input.sourceIssueId }); return tx @@ -595,6 +600,7 @@ export function issueRecoveryActionService(db: DbOrTransaction) { const pending = await tx.select({ id: issueRecoveryActions.id }).from(issueRecoveryActions) .where(and(...predicates)).limit(1); if (!pending.length) return []; + await notifyIssueRecovery(tx, { companyId: input.companyId }); await notifyNativeRecoveryNotices(tx, { companyId: input.companyId, issueId: input.sourceIssueId }); await notifyUnsafeWorkspaceExportRecovery(tx, { companyId: input.companyId, issueId: input.sourceIssueId }); return tx diff --git a/server/src/services/issue-tree-control.ts b/server/src/services/issue-tree-control.ts index 0d7633f846a..ff558b81ef0 100644 --- a/server/src/services/issue-tree-control.ts +++ b/server/src/services/issue-tree-control.ts @@ -1,3 +1,4 @@ +import { notifyIssueRecovery } from "./issue-recovery-notifications.js"; import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js"; import { notifyNativeStatusReassessment } from "./native-status-reassessment-notifications.js"; import { notifyUnsafeWorkspaceExportRecovery } from "./unsafe-workspace-export-notifications.js"; @@ -758,6 +759,7 @@ export function issueTreeControlService(db: Db) { const releaseReason = input.reason ?? "Subtree resume applied."; const { hold: resumeHold } = await db.transaction(async (tx) => { + await notifyIssueRecovery(tx, { companyId }); const [createdHold] = await tx .insert(issueTreeHolds) .values({ @@ -836,6 +838,7 @@ export function issueTreeControlService(db: Db) { } const { hold, members } = await db.transaction(async (tx) => { + await notifyIssueRecovery(tx, { companyId }); const [createdHold] = await tx .insert(issueTreeHolds) .values({ @@ -1177,7 +1180,9 @@ export function issueTreeControlService(db: Db) { throw conflict("Issue tree hold is already released"); } - const [updated] = await db + const [updated] = await db.transaction(async tx => { + await notifyIssueRecovery(tx, { companyId }); + return tx .update(issueTreeHolds) .set({ status: "released", @@ -1195,6 +1200,7 @@ export function issueTreeControlService(db: Db) { }) .where(and(eq(issueTreeHolds.id, holdId), eq(issueTreeHolds.companyId, companyId))) .returning(); + }); const members = await db .select() diff --git a/server/src/services/issues.ts b/server/src/services/issues.ts index 575648ccc37..5a569f4ce1e 100644 --- a/server/src/services/issues.ts +++ b/server/src/services/issues.ts @@ -1,3 +1,4 @@ +import { notifyIssueRecovery } from "./issue-recovery-notifications.js"; import { notifyRunDispatch } from "./run-dispatch-notifications.js"; import { notifyExternalConversationSettlement } from "./external-conversation-settlement-notifications.js"; import { notifyNativeRecoveryNotices } from "./native-recovery-notice-notifications.js"; @@ -7627,6 +7628,7 @@ export function issueService(db: Db) { await assertNoBlockingCycles(companyId, issueId, deduped, dbOrTx); } + await notifyIssueRecovery(dbOrTx, { companyId }); await dbOrTx .delete(issueRelations) .where( @@ -13799,6 +13801,7 @@ export function issueService(db: Db) { .then((rows) => rows[0] ?? null); if (!existing) return null; + await notifyIssueRecovery(tx, { companyId: existing.companyId }); await tx.delete(issueAttachments).where(eq(issueAttachments.id, id)); await tx.delete(assets).where(eq(assets.id, existing.assetId)); return existing; diff --git a/server/src/services/native-runtime/chat-attachment-reuse.ts b/server/src/services/native-runtime/chat-attachment-reuse.ts index cc7990e9b7a..6cc44299ee7 100644 --- a/server/src/services/native-runtime/chat-attachment-reuse.ts +++ b/server/src/services/native-runtime/chat-attachment-reuse.ts @@ -1,3 +1,4 @@ +import { notifyIssueRecovery } from "../issue-recovery-notifications.js"; import { createHash } from "node:crypto"; import { and, desc, eq, inArray, isNull, sql } from "drizzle-orm"; @@ -1582,7 +1583,9 @@ export async function prepareReusedChatAttachment(input: { if (!workProduct) { throw new Error("paperclip_runner_chat_attachment_origin_not_persisted"); } - await input.db + await input.db.transaction(async tx => { + await notifyIssueRecovery(tx, { companyId: input.binding.companyId }); + await tx .update(issueWorkProducts) .set({ title: input.title, @@ -1602,6 +1605,7 @@ export async function prepareReusedChatAttachment(input: { eq(issueWorkProducts.createdByRunId, input.binding.runId), ), ); + }); const comment = await issueService(input.db).addComment( input.binding.issueId, `Prepared ${input.title} for this response.`, diff --git a/server/src/services/native-runtime/native-runner-file-handoff.ts b/server/src/services/native-runtime/native-runner-file-handoff.ts index 6f2c32beec8..420380500ea 100644 --- a/server/src/services/native-runtime/native-runner-file-handoff.ts +++ b/server/src/services/native-runtime/native-runner-file-handoff.ts @@ -1,3 +1,4 @@ +import { notifyIssueRecovery } from "../issue-recovery-notifications.js"; import { createHash, randomUUID } from "node:crypto"; import { execFile } from "node:child_process"; import { constants } from "node:fs"; @@ -1265,7 +1266,9 @@ export async function prepareNativeRunnerFileHandoff(input: { ) { throw new Error("paperclip_runner_file_handoff_origin_not_persisted"); } - await input.db + await input.db.transaction(async tx => { + await notifyIssueRecovery(tx, { companyId: input.binding.companyId }); + await tx .update(issueWorkProducts) .set({ title: verified.title, updatedAt: new Date() }) .where( @@ -1276,6 +1279,7 @@ export async function prepareNativeRunnerFileHandoff(input: { eq(issueWorkProducts.createdByRunId, input.binding.runId), ), ); + }); const comment = await issueService(input.db).addComment( input.binding.issueId, `Prepared ${verified.title} for this response.`, diff --git a/server/src/services/native-status-reassessment-notifications.ts b/server/src/services/native-status-reassessment-notifications.ts index fb845523bb4..4b5eac73b2f 100644 --- a/server/src/services/native-status-reassessment-notifications.ts +++ b/server/src/services/native-status-reassessment-notifications.ts @@ -1,8 +1,10 @@ +import { notifyIssueRecovery } from "./issue-recovery-notifications.js"; import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js"; export type NativeStatusReassessmentScope = { companyId: string; issueId?: string; runId?: string; agentId?: string }; -export function notifyNativeStatusReassessment(tx: object, scope: NativeStatusReassessmentScope) { +export async function notifyNativeStatusReassessment(tx: object, scope: NativeStatusReassessmentScope) { + await notifyIssueRecovery(tx, scope); return notifyDeliveryWork(tx, DELIVERY_QUEUES.nativeStatusReassessment, JSON.stringify({ companyId: scope.companyId, issueId: scope.issueId, runId: scope.runId, agentId: scope.agentId })); } diff --git a/server/src/services/system-recovery.ts b/server/src/services/system-recovery.ts index a554a765c4c..347593dafed 100644 --- a/server/src/services/system-recovery.ts +++ b/server/src/services/system-recovery.ts @@ -1,3 +1,4 @@ +import { notifyIssueRecovery } from "./issue-recovery-notifications.js"; import { and, eq, sql } from "drizzle-orm"; import { heartbeatRuns, issueComments, issueRecoveryActions, type Db } from "@paperclipai/db"; import { writeExecutionControl } from "./execution-control-notifications.js"; @@ -11,6 +12,7 @@ export async function adoptSystemRecovery(db: Db, source: WorkspaceRecoverySourc .where(and(eq(issueRecoveryActions.companyId, source.companyId), eq(issueRecoveryActions.id, source.actionId), eq(issueRecoveryActions.sourceIssueId, source.issueId))).for("update"); if (!existing || existing.evidence.systemRecovery) return; + await notifyIssueRecovery(tx, { companyId: source.companyId }); await notifyDeliveryWork(tx, DELIVERY_QUEUES.workspaceRecovery); const state = createSystemRecovery(source.runId); // Atomic, once per source. An exhausted previous execution budget is history; diff --git a/server/src/services/task-watchdog-work.ts b/server/src/services/task-watchdog-work.ts index 091c8c6595f..cde0a7eca43 100644 --- a/server/src/services/task-watchdog-work.ts +++ b/server/src/services/task-watchdog-work.ts @@ -29,7 +29,7 @@ export function registerTaskWatchdogWork( let next: number | null = null; const result = await owner.reconcile({ companyId: entry.companyId, watchdogId: entry.id, onDeferredDeadline: at => { next = next === null ? at : Math.min(next, at); } }); - if (result.incomplete > 0) throw new Error("Task watchdog subtree remains incomplete"); + if (result.incomplete > 0) throw new Error(`Task watchdog ${entry.id} in company ${entry.companyId} has an incomplete subtree`); return next === null ? null : { ...entry, dueAt: next }; }, }, canRun); diff --git a/server/src/services/task-watchdogs.ts b/server/src/services/task-watchdogs.ts index 6f94e5520a7..dd1aa980d3c 100644 --- a/server/src/services/task-watchdogs.ts +++ b/server/src/services/task-watchdogs.ts @@ -848,6 +848,7 @@ async function upsertIssueWatchdogInTransaction( .where(and(eq(issueWatchdogs.companyId, companyId), eq(issueWatchdogs.issueId, issueId))) .then((rows: IssueWatchdogRow[]) => rows[0] ?? null); + await notifyIssueRecovery(dbOrTx, { companyId }); if (existing) { const updated = await updateIssueWatchdogRow(dbOrTx, existing, input, now); return { watchdog: toIssueWatchdog(updated), created: false }; @@ -892,10 +893,7 @@ export function upsertIssueWatchdogForIssue( issueId: string, input: IssueWatchdogUpsertInput, ): Promise<{ watchdog: IssueWatchdog; created: boolean }> { - return dbOrTx.transaction(async tx => { - await notifyIssueRecovery(tx, { companyId }); - return upsertIssueWatchdogInTransaction(tx, companyId, issueId, input); - }); + return dbOrTx.transaction(tx => upsertIssueWatchdogInTransaction(tx, companyId, issueId, input)); } export function taskWatchdogService(db: Db, deps: TaskWatchdogServiceDeps = {}) { From ab5108611da3a5841604e706fc78a95e716e692b Mon Sep 17 00:00:00 2001 From: Dotta Date: Fri, 30 Oct 2026 18:15:23 -0500 Subject: [PATCH 43/70] Run recovery producer acceptance on isolated PostgreSQL fixtures --- server/src/__tests__/issue-tree-control-service.test.ts | 2 +- .../services/native-runtime/chat-attachment-reuse.test.ts | 2 +- .../native-runtime/native-runner-file-handoff.test.ts | 8 ++++---- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/server/src/__tests__/issue-tree-control-service.test.ts b/server/src/__tests__/issue-tree-control-service.test.ts index 923f399d3bd..e98efb18309 100644 --- a/server/src/__tests__/issue-tree-control-service.test.ts +++ b/server/src/__tests__/issue-tree-control-service.test.ts @@ -41,7 +41,7 @@ describeEmbeddedPostgres("issueTreeControlService", () => { beforeAll(async () => { if (externalUrl) { await applyPendingMigrations(externalUrl); - db = createDb(externalUrl); + db = createDb(externalUrl, { maxConnections: 6 }); return; } tempDb = await startEmbeddedPostgresTestDatabase("paperclip-issue-tree-control-"); diff --git a/server/src/services/native-runtime/chat-attachment-reuse.test.ts b/server/src/services/native-runtime/chat-attachment-reuse.test.ts index ccd4bbe30d6..d0de5ab04c9 100644 --- a/server/src/services/native-runtime/chat-attachment-reuse.test.ts +++ b/server/src/services/native-runtime/chat-attachment-reuse.test.ts @@ -73,7 +73,7 @@ describe("native same-conversation chat attachment reuse", () => { beforeAll(async () => { const external = resolveExternalTestDatabaseUrl("PAPERCLIP_CHAT_REUSE_TEST_DATABASE_URL"); if (!external) temporary = await startEmbeddedPostgresTestDatabase("native-chat-reuse-"); - db = createDb(external ?? temporary!.connectionString); + db = createDb(external ?? temporary!.connectionString, { maxConnections: 6 }); const storageRoot = await mkdtemp( path.join(tmpdir(), "paperclip-chat-reuse-"), ); diff --git a/server/src/services/native-runtime/native-runner-file-handoff.test.ts b/server/src/services/native-runtime/native-runner-file-handoff.test.ts index 0abae30539f..e302c432190 100644 --- a/server/src/services/native-runtime/native-runner-file-handoff.test.ts +++ b/server/src/services/native-runtime/native-runner-file-handoff.test.ts @@ -1,3 +1,4 @@ +import { resolveExternalTestDatabaseUrl } from "../../__tests__/helpers/external-test-database.js"; import type { PrpStructuredRunResult } from "../../vendor/paperclip-runner/index.js"; import { nativeCompletionFeedback } from "./native-completion-feedback.js"; import { createHash, randomUUID } from "node:crypto"; @@ -61,10 +62,9 @@ describe("native runner file handoff", () => { const runId = "00000000-0000-4000-8000-000000009104"; beforeAll(async () => { - temporary = await startEmbeddedPostgresTestDatabase( - "native-runner-file-handoff-", - ); - db = createDb(temporary.connectionString); + const external = resolveExternalTestDatabaseUrl("PAPERCLIP_NATIVE_HANDOFF_TEST_DATABASE_URL"); + if (!external) temporary = await startEmbeddedPostgresTestDatabase("native-runner-file-handoff-"); + db = createDb(external ?? temporary!.connectionString, { maxConnections: 6 }); temporaryRoot = await mkdtemp( path.join(tmpdir(), "paperclip-native-file-handoff-"), ); From 4b8d9a64ca3873100e0814ea7d89e3e8a11472cf Mon Sep 17 00:00:00 2001 From: Dotta Date: Fri, 30 Oct 2026 18:16:05 -0500 Subject: [PATCH 44/70] fix(server): notify routine status after subtree restore Co-Authored-By: Paperclip --- doc/plans/2026-10-30-routine-status-sleep.md | 4 +++- .../issue-tree-control-service.test.ts | 2 +- .../routine-execution-status-work.test.ts | 22 +++++++++++++++++++ server/src/services/issue-tree-control.ts | 2 ++ 4 files changed, 28 insertions(+), 2 deletions(-) diff --git a/doc/plans/2026-10-30-routine-status-sleep.md b/doc/plans/2026-10-30-routine-status-sleep.md index e852bf94861..0afe809bb8a 100644 --- a/doc/plans/2026-10-30-routine-status-sleep.md +++ b/doc/plans/2026-10-30-routine-status-sleep.md @@ -9,7 +9,9 @@ scanned every five minutes, even with no routines or only completed history. `routine-run-status.ts` owns the projection, custody checks and repair transaction. The worker loads actual mismatches in keyset pages of 50 at startup. It accepts issue hints from issue creation, status/origin changes, checkout/release, routine issue association, -execution control writes and review state writes. It uses the existing +execution control writes, review state writes and subtree status restoration. +Bulk restoration registers each issue hint in the same transaction before its +grouped write. Restored history does not depend on an optional agent wake. It uses the existing transaction-aware work notification protocol. An outer transaction owns the hint; a savepoint release does not wake the repair worker before commit. diff --git a/server/src/__tests__/issue-tree-control-service.test.ts b/server/src/__tests__/issue-tree-control-service.test.ts index 923f399d3bd..e98efb18309 100644 --- a/server/src/__tests__/issue-tree-control-service.test.ts +++ b/server/src/__tests__/issue-tree-control-service.test.ts @@ -41,7 +41,7 @@ describeEmbeddedPostgres("issueTreeControlService", () => { beforeAll(async () => { if (externalUrl) { await applyPendingMigrations(externalUrl); - db = createDb(externalUrl); + db = createDb(externalUrl, { maxConnections: 6 }); return; } tempDb = await startEmbeddedPostgresTestDatabase("paperclip-issue-tree-control-"); diff --git a/server/src/__tests__/routine-execution-status-work.test.ts b/server/src/__tests__/routine-execution-status-work.test.ts index e6bd818852c..0a6ff9339cb 100644 --- a/server/src/__tests__/routine-execution-status-work.test.ts +++ b/server/src/__tests__/routine-execution-status-work.test.ts @@ -1,3 +1,4 @@ +import { issueTreeControlService } from "../services/issue-tree-control.js"; import { writeExecutionControl } from "../services/execution-control-notifications.js"; import { DELIVERY_QUEUES } from "../services/delivery-work-notifications.js"; import { randomUUID } from "node:crypto"; @@ -174,6 +175,27 @@ const support = external ? { supported: true } : await getEmbeddedPostgresTestSu await db.transaction(async tx => { await notifyRoutineExecutionStatus(tx, otherRun.linkedIssueId!); }); await projected(otherRun.id, "completed"); await quiet(); expect(errors).toEqual([]); }); + it("projects subtree cancel and restore without waking agents, then becomes quiet", async () => { + const f = await fixture(), root = await dispatch(f), child = await dispatch(f); + await issueService(db).update(child.linkedIssueId!, { parentId: root.linkedIssueId }); + const tree = issueTreeControlService(db), actor = { actorType: "system" as const, actorId: "routine-status-test" }; + await start(); await quiet(); + const cancel = await tree.createHold(f.companyId, root.linkedIssueId!, { mode: "cancel", actor }); + expect((await tree.cancelIssueStatusesForHold(f.companyId, root.linkedIssueId!, cancel.hold.id)).updatedIssueIds.sort()) + .toEqual([root.linkedIssueId, child.linkedIssueId].sort()); + await projected(root.id, "failed"); await projected(child.id, "failed"); await quiet(); + const restore = await tree.createHold(f.companyId, root.linkedIssueId!, { mode: "restore", actor }); + const restored = await tree.restoreIssueStatusesForHold(f.companyId, root.linkedIssueId!, restore.hold.id, { actor }); + expect(restored.updatedIssues).toEqual(expect.arrayContaining([ + expect.objectContaining({ id: root.linkedIssueId, status: "todo" }), + expect.objectContaining({ id: child.linkedIssueId, status: "todo" }), + ])); + await projected(root.id, "issue_created"); await projected(child.id, "issue_created"); + expect(await saved(root.id)).toMatchObject({ completedAt: null, failureReason: null }); + expect(await saved(child.id)).toMatchObject({ completedAt: null, failureReason: null }); + expect(await db.select().from(heartbeatRuns)).toEqual([]); + await quiet(); expect(errors).toEqual([]); + }); it("serializes a direct status hook with later task changes before projecting the routine", async () => { const f = await fixture(), run = await dispatch(f); await start(); await quiet(); await db.update(issues).set({ status: "done" }).where(eq(issues.id, run.linkedIssueId!)); diff --git a/server/src/services/issue-tree-control.ts b/server/src/services/issue-tree-control.ts index 0d7633f846a..68ba89f527e 100644 --- a/server/src/services/issue-tree-control.ts +++ b/server/src/services/issue-tree-control.ts @@ -1,3 +1,4 @@ +import { notifyRoutineExecutionStatus } from "./routine-execution-status-notifications.js"; import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js"; import { notifyNativeStatusReassessment } from "./native-status-reassessment-notifications.js"; import { notifyUnsafeWorkspaceExportRecovery } from "./unsafe-workspace-export-notifications.js"; @@ -1006,6 +1007,7 @@ export function issueTreeControlService(db: Db) { for (const [status, issueIdsForStatus] of issueIdsByStatus) { if (issueIdsForStatus.length === 0) continue; for (const issueId of issueIdsForStatus) { + await notifyRoutineExecutionStatus(tx, issueId, companyId); await notifyNativeStatusReassessment(tx, { companyId, issueId }); await notifyTerminalWorkspaceWork(tx, `issue:${issueId}`); await notifyChatResponseAvailability(tx, { companyId, issueId }); From 972fbc0760f2d8ec6779abb49e7294465377c1fc Mon Sep 17 00:00:00 2001 From: Dotta Date: Fri, 30 Oct 2026 18:24:45 -0500 Subject: [PATCH 45/70] fix: normalize run dispatch hints to company identity Wider interaction and execution objects must coalesce into one company dispatch scope. Co-Authored-By: Paperclip --- .../src/__tests__/heartbeat-run-dispatch-lifecycle.test.ts | 6 ++++-- server/src/services/run-dispatch-notifications.ts | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/server/src/__tests__/heartbeat-run-dispatch-lifecycle.test.ts b/server/src/__tests__/heartbeat-run-dispatch-lifecycle.test.ts index b6fbb7353de..f3398d828f8 100644 --- a/server/src/__tests__/heartbeat-run-dispatch-lifecycle.test.ts +++ b/server/src/__tests__/heartbeat-run-dispatch-lifecycle.test.ts @@ -71,14 +71,16 @@ describe("run dispatch obligations", () => { it("batches repeated tenant writes and passes only their company to queued dispatch", async () => { const f = fixture(); await f.worker.ready; for (let index = 0; index < 100; index++) { - await f.notify({ companyId: "company-a" }); + const source = { companyId: "company-a", issueId: `issue-${index % 2}`, + runId: `run-${index}`, agentId: `agent-${index}`, interactionBody: "not a dispatch key" }; + await f.notify(source); await vi.advanceTimersByTimeAsync(10); } expect(f.promote.mock.calls.length).toBeLessThanOrEqual(2); expect(f.dispatch.mock.calls.length).toBeLessThanOrEqual(2); await f.settle(); expect(f.dispatch).toHaveBeenLastCalledWith({ companyId: "company-a" }); - expect(f.dispatch.mock.calls.slice(1).every(([scope]) => scope?.companyId === "company-a")).toBe(true); + for (const [scope] of f.dispatch.mock.calls.slice(1)) expect(scope).toEqual({ companyId: "company-a" }); expect(f.coordinator.nextWakeAt()).toBeNull(); const count = f.dispatch.mock.calls.length; await vi.advanceTimersByTimeAsync(20 * 60_000); expect(f.dispatch).toHaveBeenCalledTimes(count); diff --git a/server/src/services/run-dispatch-notifications.ts b/server/src/services/run-dispatch-notifications.ts index 2e04e6e7531..e62b8c62105 100644 --- a/server/src/services/run-dispatch-notifications.ts +++ b/server/src/services/run-dispatch-notifications.ts @@ -3,7 +3,7 @@ import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notificatio export type RunDispatchScope = { companyId: string }; export async function notifyRunDispatch(transaction: object, scope?: RunDispatchScope) { - const hint = scope ? JSON.stringify(scope) : "*"; + const hint = scope ? JSON.stringify({ companyId: scope.companyId }) : "*"; await notifyDeliveryWork(transaction, DELIVERY_QUEUES.scheduledRetryDispatch, hint); await notifyDeliveryWork(transaction, DELIVERY_QUEUES.queuedRunDispatch, hint); } From efbe3fa67d8dee2fb2037d9b097ed6ae1876a982 Mon Sep 17 00:00:00 2001 From: Dotta Date: Fri, 30 Oct 2026 18:25:01 -0500 Subject: [PATCH 46/70] fix(server): isolate blocker traversal budgets per root Share queried graph facts across roots while retaining independent node/depth limits, expanded reachability and truncation for classification. Unrelated limited graphs must not change terminal subjects or delay archive deadlines. Prove depth/node overflow and overlapping-root liveness red/green, preserve the100-root SQL bound, and qualify existing owner assertions on isolated PostgreSQL. Co-Authored-By: Paperclip --- .../2026-10-30-attention-retention-sleep.md | 2 +- .../attention-retention-work.test.ts | 57 ++++++++++++++++++- .../__tests__/issue-blocker-attention.test.ts | 12 +++- server/src/services/issues.ts | 56 +++++++++++------- 4 files changed, 101 insertions(+), 26 deletions(-) diff --git a/doc/plans/2026-10-30-attention-retention-sleep.md b/doc/plans/2026-10-30-attention-retention-sleep.md index 2e0373fc975..463fbe94188 100644 --- a/doc/plans/2026-10-30-attention-retention-sleep.md +++ b/doc/plans/2026-10-30-attention-retention-sleep.md @@ -25,7 +25,7 @@ The feed and worker share confirmation collapse, newer-run probes, issue visibil ## Existing mutation boundaries reactivate work -Committed hints identify a company, source kind, and source ID where those values are known. Retention's own writes use exact source hints; blocker retention changes invalidate the company/source because multiple blocked roots can project one terminal subject. Reconstruction pages and due batches remain bounded. Review attention and interaction invokability are computed in company batches per candidate page; a full 100-candidate review page uses at most 20 statements and 100 confirmations use at most six. Decisions, recovery source links, runs, eligible agents, issue visibility, newer-run probes and blocker graphs also load in company batches per page. A 100-source decision or recovery page uses at most three statements; 100 failed runs use at most six; 100 blocker roots use at most 30 graph/source statements plus retention-state reads for their emitted entries. These evidence maps exist only for one load. Archive checks still lock and freshly re-read sources, terminal subjects and related eligibility. Ordinary execution, decision and budget mutations pass their affected company; exact decision sources are included. +Committed hints identify a company, source kind, and source ID where those values are known. Retention's own writes use exact source hints; blocker retention changes invalidate the company/source because multiple blocked roots can project one terminal subject. Reconstruction pages and due batches remain bounded. Review attention and interaction invokability are computed in company batches per candidate page; a full 100-candidate review page uses at most 20 statements and 100 confirmations use at most six. Decisions, recovery source links, runs, eligible agents, issue visibility, newer-run probes and blocker graphs also load in company batches per page. A 100-source decision or recovery page uses at most three statements; 100 failed runs use at most six; 100 blocker roots use at most 30 graph/source statements plus retention-state reads for their emitted entries. These evidence maps exist only for one load. The blocker owner shares fetched node and edge facts while tracking depth, node count, expanded nodes and live-work reachability independently for each root. An unrelated graph reaching its limit cannot change a shallow root’s terminal subject or deadline. Archive checks still lock and freshly re-read sources, terminal subjects and related eligibility. Ordinary execution, decision and budget mutations pass their affected company; exact decision sources are included. Producer coverage includes decision creation and changes; interaction creation, resolution, expiry and supersession; issue creation, updates, blocker relations, checkout, adoption and release; agent hire, configuration, lifecycle, error clearing and deletion; budget incidents and company pause/resume; exhausted-run events and retry-run creation; native status, review, finalization and reconstruction; external conversation transitions; and imported blocker relations. Existing service transactions retain outer commit ownership. Savepoint release does not publish work before outer commit. diff --git a/server/src/__tests__/attention-retention-work.test.ts b/server/src/__tests__/attention-retention-work.test.ts index fd1a01e8582..29a82f866a5 100644 --- a/server/src/__tests__/attention-retention-work.test.ts +++ b/server/src/__tests__/attention-retention-work.test.ts @@ -15,7 +15,7 @@ import { decisionRetentionService, DEFAULT_DECISION_ARCHIVE_DAYS } from "../serv import { appendHeartbeatRunEvent } from "../services/heartbeat-run-events.js"; import { issueThreadInteractionService } from "../services/issue-thread-interactions.js"; import { createAgentLifecycle } from "../services/agent-lifecycle.js"; -import { issueService } from "../services/issues.js"; +import { BLOCKER_ATTENTION_MAX_DEPTH, BLOCKER_ATTENTION_MAX_NODES, issueService } from "../services/issues.js"; import { notifyAttentionRetentionWork } from "../services/retention-work-notifications.js"; import { idleWorkSnapshot } from "../services/task-admission.js"; import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; @@ -256,6 +256,61 @@ const delay = DEFAULT_DECISION_ARCHIVE_DAYS * 86_400_000; expect(refreshed.rows!.some(entry => entry.sourceId === terminals[99].id)).toBe(false); }); + it.each(["depth", "nodes"] as const)("preserves shallow terminal selection when an unrelated graph hits its %s limit", async limit => { + const root = await insertIssue("blocked", { updatedAt: new Date() }); + const middle = await insertIssue("todo", { updatedAt: new Date() }); + const terminal = await insertIssue("backlog", { assigneeAgentId: agentId }); + const heavy = await insertIssue("blocked", { updatedAt: new Date() }); + const count = limit === "depth" ? BLOCKER_ATTENTION_MAX_DEPTH + 1 : BLOCKER_ATTENTION_MAX_NODES; + const heavyNodes = await db.insert(issues).values(Array.from({ length: count }, () => ({ companyId, + title: randomUUID(), status: "todo", updatedAt: new Date() }))).returning(); + await db.insert(issueRelations).values([ + { companyId, issueId: middle.id, relatedIssueId: root.id, type: "blocks" }, + { companyId, issueId: terminal.id, relatedIssueId: middle.id, type: "blocks" }, + ...heavyNodes.map((node, index) => ({ companyId, issueId: node.id, + relatedIssueId: limit === "nodes" || index === 0 ? heavy.id : heavyNodes[index - 1].id, type: "blocks" })), + ]); + const owner = issueService(db); + const expected = (await owner.listBlockerAttention(companyId, [root])).get(root.id)!; + const limited = (await owner.listBlockerAttention(companyId, [heavy])).get(heavy.id)!; + expect(expected.terminalBlockerIssueId).toBe(terminal.id); + expect(limited.terminalBlockerIssueId).toBe(limited.directBlockerIssueId); + for (const input of [[root, heavy], [heavy, root]]) { + const batch = await owner.listBlockerAttention(companyId, input); + expect(batch.get(root.id)).toEqual(expected); expect(batch.get(heavy.id)).toEqual(limited); + } + const page = await attentionRetentionService(db).load({ companyId, sourceKind: "blocker_attention" }); + const entry = page.rows!.find(item => item.rootId === root.id && item.variant === "terminal")!; + expect(entry.sourceId).toBe(terminal.id); expect(entry.dueAt).toBeLessThan(Date.now()); + await attentionRetentionService(db).check(entry); + expect((await state("blocker_attention", terminal.id))?.archivedAt).toBeInstanceOf(Date); + expect(await db.select().from(decisionArchiveNotificationOutbox).where(eq(decisionArchiveNotificationOutbox.sourceId, terminal.id))).toHaveLength(1); + }); + + it("keeps root-specific reachability and cycle budgets when another root explores farther", async () => { + const root = await insertIssue("blocked"); + const chain = await db.insert(issues).values(Array.from({ length: BLOCKER_ATTENTION_MAX_DEPTH + 1 }, (_, index) => ({ + companyId, title: randomUUID(), status: index === BLOCKER_ATTENTION_MAX_DEPTH ? "in_progress" + : index === BLOCKER_ATTENTION_MAX_DEPTH - 2 ? "blocked" : "todo", updatedAt: old() }))).returning(); + const overlapping = chain[BLOCKER_ATTENTION_MAX_DEPTH - 2]; + const cycle = await insertIssue("blocked"); + const leaf = await insertIssue("backlog", { assigneeAgentId: agentId }); + await db.insert(issueRelations).values([ + ...chain.map((node, index) => ({ companyId, issueId: node.id, relatedIssueId: index ? chain[index - 1].id : root.id, type: "blocks" })), + { companyId, issueId: leaf.id, relatedIssueId: cycle.id, type: "blocks" }, + { companyId, issueId: cycle.id, relatedIssueId: leaf.id, type: "blocks" }, + ]); + const owner = issueService(db), roots = [root, overlapping, cycle]; + const singles = new Map(); + for (const item of roots) singles.set(item.id, (await owner.listBlockerAttention(companyId, [item])).get(item.id)); + expect(singles.get(root.id).blockingTreeLive).toBe(false); + expect(singles.get(overlapping.id).blockingTreeLive).toBe(true); + for (const input of [roots, [...roots].reverse()]) { + const batch = await owner.listBlockerAttention(companyId, input); + for (const item of roots) expect(batch.get(item.id)).toEqual(singles.get(item.id)); + } + }); + it("scopes ordinary execution-control writes to their company", async () => { const otherCompany = randomUUID(); await db.insert(companies).values({ id: otherCompany, name: "Other", issuePrefix: "OTHER" }); diff --git a/server/src/__tests__/issue-blocker-attention.test.ts b/server/src/__tests__/issue-blocker-attention.test.ts index 1e6c00631e7..4f1a0e71573 100644 --- a/server/src/__tests__/issue-blocker-attention.test.ts +++ b/server/src/__tests__/issue-blocker-attention.test.ts @@ -8,6 +8,7 @@ import { approvals, companies, createDb, + applyPendingMigrations, heartbeatRuns, issueApprovals, issueRelations, @@ -21,7 +22,10 @@ import { import { issueService } from "../services/issues.js"; import { buildIssueGraphLivenessIncidentKey } from "../services/recovery/origins.js"; -const embeddedPostgresSupport = await getEmbeddedPostgresTestSupport(); +import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js"; + +const externalUrl = resolveExternalTestDatabaseUrl("PAPERCLIP_BLOCKER_ATTENTION_TEST_DATABASE_URL"); +const embeddedPostgresSupport = externalUrl ? { supported: true } : await getEmbeddedPostgresTestSupport(); const describeEmbeddedPostgres = embeddedPostgresSupport.supported ? describe : describe.skip; if (!embeddedPostgresSupport.supported) { @@ -36,8 +40,9 @@ describeEmbeddedPostgres("issue blocker attention", () => { let tempDb: Awaited> | null = null; beforeAll(async () => { - tempDb = await startEmbeddedPostgresTestDatabase("paperclip-issue-blocker-attention-"); - db = createDb(tempDb.connectionString); + if (externalUrl) await applyPendingMigrations(externalUrl); + else tempDb = await startEmbeddedPostgresTestDatabase("paperclip-issue-blocker-attention-"); + db = createDb(externalUrl ?? tempDb!.connectionString, { maxConnections: 6 }); svc = issueService(db); }, 20_000); @@ -55,6 +60,7 @@ describeEmbeddedPostgres("issue blocker attention", () => { }); afterAll(async () => { + await db?.$client.end({ timeout: 1 }); await tempDb?.cleanup(); }); diff --git a/server/src/services/issues.ts b/server/src/services/issues.ts index 7239075de99..fd1fd9ba819 100644 --- a/server/src/services/issues.ts +++ b/server/src/services/issues.ts @@ -3840,18 +3840,19 @@ async function listIssueBlockerAttentionMap( const edgesByIssueId = new Map(); for (const root of roots) nodesById.set(root.id, { ...root }); - let frontier = roots.map((root) => root.id); - let truncated = false; + type Traversal = { nodes: Set; expanded: Set; frontier: string[]; truncated: boolean }; + const traversals = new Map(roots.map(root => [root.id, { + nodes: new Set([root.id]), expanded: new Set(), frontier: [root.id], truncated: false, + }])); + const expanded = new Set(); const pendingFinalizeBlockerIssueIds = new Set(); - for ( - let depth = 0; - frontier.length > 0 && depth < BLOCKER_ATTENTION_MAX_DEPTH; - depth += 1 - ) { - const nextFrontier = new Set(); + for (let depth = 0; depth < BLOCKER_ATTENTION_MAX_DEPTH; depth += 1) { + const active = [...traversals.values()].filter(traversal => !traversal.truncated && traversal.frontier.length > 0); + if (active.length === 0) break; + const frontier = [...new Set(active.flatMap(traversal => traversal.frontier))].filter(id => !expanded.has(id)); for (const chunk of chunkList( - [...new Set(frontier)], + frontier, ISSUE_LIST_RELATED_QUERY_CHUNK_SIZE, )) { const readinessByIssueId = await listIssueDependencyReadinessMap( @@ -3959,17 +3960,27 @@ async function listIssueBlockerAttentionMap( assigneeAgentId: row.assigneeAgentId, assigneeUserId: row.assigneeUserId, }); - nextFrontier.add(row.blockerIssueId); } + for (const id of chunk) expanded.add(id); } - if (nodesById.size > BLOCKER_ATTENTION_MAX_NODES) { - truncated = true; - break; + for (const traversal of active) { + const nextFrontier = new Set(); + for (const id of traversal.frontier) { + traversal.expanded.add(id); + for (const edge of edgesByIssueId.get(id) ?? []) { + if (traversal.nodes.has(edge.blockerIssueId) || !nodesById.has(edge.blockerIssueId)) continue; + traversal.nodes.add(edge.blockerIssueId); + nextFrontier.add(edge.blockerIssueId); + } + } + traversal.truncated = traversal.nodes.size > BLOCKER_ATTENTION_MAX_NODES; + traversal.frontier = traversal.truncated ? [] : [...nextFrontier]; } - frontier = [...nextFrontier]; } - if (frontier.length > 0) truncated = true; + for (const traversal of traversals.values()) { + if (traversal.frontier.length > 0) traversal.truncated = true; + } const nodeIds = [...nodesById.keys()]; const activeIssueIds = new Set(); @@ -4215,6 +4226,7 @@ async function listIssueBlockerAttentionMap( const classifyPath = ( nodeId: string, seen: Set, + truncated: boolean, ): PathClassification => { const sample = blockerSampleIdentifier(nodesById.get(nodeId)); if (truncated || seen.has(nodeId)) { @@ -4319,7 +4331,7 @@ async function listIssueBlockerAttentionMap( const nextSeen = new Set(seen); nextSeen.add(nodeId); const classified = downstream.map((edge) => - classifyPath(edge.blockerIssueId, nextSeen), + classifyPath(edge.blockerIssueId, nextSeen, truncated), ); const stalledChild = classified.find( (result) => result.stalled || result.sampleStalledBlockerIdentifier, @@ -4385,8 +4397,8 @@ async function listIssueBlockerAttentionMap( }; }; - const pathHasLiveWork = (nodeId: string, seen: Set): boolean => { - if (seen.has(nodeId)) return false; + const pathHasLiveWork = (nodeId: string, seen: Set, traversal: Traversal): boolean => { + if (seen.has(nodeId) || !traversal.nodes.has(nodeId)) return false; const node = nodesById.get(nodeId); if (!node || node.companyId !== companyId) return false; if ( @@ -4397,6 +4409,7 @@ async function listIssueBlockerAttentionMap( ) return true; + if (!traversal.expanded.has(nodeId)) return false; const nextSeen = new Set(seen); nextSeen.add(nodeId); return (edgesByIssueId.get(node.id) ?? []).some((edge) => { @@ -4406,7 +4419,7 @@ async function listIssueBlockerAttentionMap( !pendingFinalizeBlockerIssueIds.has(edge.blockerIssueId) ) return false; - return pathHasLiveWork(edge.blockerIssueId, nextSeen); + return pathHasLiveWork(edge.blockerIssueId, nextSeen, traversal); }); }; @@ -4419,6 +4432,7 @@ async function listIssueBlockerAttentionMap( }; for (const root of roots) { + const traversal = traversals.get(root.id)!; const topLevelEdges = (edgesByIssueId.get(root.id) ?? []).filter((edge) => { const blocker = nodesById.get(edge.blockerIssueId); return ( @@ -4440,7 +4454,7 @@ async function listIssueBlockerAttentionMap( const classified = topLevelEdges.map((edge) => ({ edge, - result: classifyPath(edge.blockerIssueId, new Set([root.id])), + result: classifyPath(edge.blockerIssueId, new Set([root.id]), traversal.truncated), })); const coveredBlockerCount = classified.filter( (entry) => entry.result.covered, @@ -4523,7 +4537,7 @@ async function listIssueBlockerAttentionMap( sampleStalledFromChain ?? null, blockingTreeLive: topLevelEdges.some((edge) => - pathHasLiveWork(edge.blockerIssueId, new Set([root.id])), + pathHasLiveWork(edge.blockerIssueId, new Set([root.id]), traversal), ), directBlockerIssueId: sampleEntry?.edge.blockerIssueId ?? null, terminalBlockerIssueId, From 2ee5c7d6dc4a6cbf4ee9211c71dec3b75496509b Mon Sep 17 00:00:00 2001 From: Dotta Date: Fri, 30 Oct 2026 18:28:40 -0500 Subject: [PATCH 47/70] fix: preserve status-card deadline ownership through restart Use a dedicated advisory-lock connection so a one-slot write pool can finish evaluation and result writeback. Record internal deadline provenance so startup recovers lost membership hints without advancing saved policy or retry debt. Co-Authored-By: Paperclip --- doc/plans/2026-10-30-status-cards-sleep.md | 23 + .../src/migrations/0419_moaning_vulture.sql | 1 + ...{0414_snapshot.json => 0419_snapshot.json} | 516 +++++++++++++++++- packages/db/src/migrations/meta/_journal.json | 7 + packages/db/src/schema/status_cards.ts | 3 + .../status-card-evaluation-work.test.ts | 88 ++- server/src/__tests__/status-cards.test.ts | 2 + server/src/routes/status-cards.ts | 26 +- .../src/services/status-card-finalization.ts | 1 + server/src/services/status-cards.ts | 47 +- 10 files changed, 684 insertions(+), 30 deletions(-) create mode 100644 packages/db/src/migrations/0419_moaning_vulture.sql rename packages/db/src/migrations/meta/{0414_snapshot.json => 0419_snapshot.json} (99%) diff --git a/doc/plans/2026-10-30-status-cards-sleep.md b/doc/plans/2026-10-30-status-cards-sleep.md index 2af51ff65cf..dbf229cdace 100644 --- a/doc/plans/2026-10-30-status-cards-sleep.md +++ b/doc/plans/2026-10-30-status-cards-sleep.md @@ -100,3 +100,26 @@ An expired saved deadline cannot schedule an immediate skipped-attempt loop. Existing startup, status-card, company-search, execution and monitor suites cover the production wiring and unchanged consumers. Exact commands and results belong in the PR verification record; this document does not claim deployed savings. + +### Review corrections: pool capacity and restart deadline ownership + +Scheduled evaluation holds its advisory transaction on `withDedicatedDbConnection`, +not on the ordinary write pool. The helper ends that physical connection in +`finally`, including contention and failed evaluation. Its lock still spans the +actual refresh settlement. Summary writeback runs watched-query reads on its +existing transaction, so a one-connection main pool completes the full flow. + +Migration0419 adds nullable internal `next_eval_reason` to the existing card row. +Membership deadlines may be reconstructed at startup after a committed source +hint was lost. Policy eligibility and the atomically recorded five-minute retry +lease retain their saved deadline. Explicit refresh deadlines remain due work. +The HTTP card response removes this internal provenance; the public schema stays +unchanged. + +For legacy rows, a pending change hash preserves policy eligibility. An untagged +active deadline farther than five minutes away cannot be an outstanding +five-minute claim, so startup rechecks it. An ambiguous untagged deadline within +five minutes keeps its remaining time without extension. This bounds migration +compatibility delay while preserving prior crash/failure recovery. After the +first successful evaluation all newly saved deadlines have explicit ownership. +No recurring compatibility scan is added. diff --git a/packages/db/src/migrations/0419_moaning_vulture.sql b/packages/db/src/migrations/0419_moaning_vulture.sql new file mode 100644 index 00000000000..03ea349aec1 --- /dev/null +++ b/packages/db/src/migrations/0419_moaning_vulture.sql @@ -0,0 +1 @@ +ALTER TABLE "status_cards" ADD COLUMN "next_eval_reason" text; \ No newline at end of file diff --git a/packages/db/src/migrations/meta/0414_snapshot.json b/packages/db/src/migrations/meta/0419_snapshot.json similarity index 99% rename from packages/db/src/migrations/meta/0414_snapshot.json rename to packages/db/src/migrations/meta/0419_snapshot.json index 2679e7d98bf..d03afbb279d 100644 --- a/packages/db/src/migrations/meta/0414_snapshot.json +++ b/packages/db/src/migrations/meta/0419_snapshot.json @@ -1,6 +1,6 @@ { - "id": "27c9008c-60e3-449b-9630-9e4977fad444", - "prevId": "5690a14a-f9cc-41a6-af51-c27f688f4ef6", + "id": "cce00822-b7a4-4acd-9c59-0f6f5822c0cf", + "prevId": "c777fa84-85a6-418f-bea1-0d60b56cf071", "version": "7", "dialect": "postgresql", "tables": { @@ -19052,6 +19052,98 @@ "checkConstraints": {}, "isRLSEnabled": false }, + "public.company_skill_mutation_receipts": { + "name": "company_skill_mutation_receipts", + "schema": "", + "columns": { + "company_id": { + "name": "company_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "key": { + "name": "key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "fingerprint": { + "name": "fingerprint", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "skill_id": { + "name": "skill_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "version_id": { + "name": "version_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "response": { + "name": "response", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "company_skill_mutation_receipts_key_idx": { + "name": "company_skill_mutation_receipts_key_idx", + "columns": [ + { + "expression": "company_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "company_skill_mutation_receipts_company_id_companies_id_fk": { + "name": "company_skill_mutation_receipts_company_id_companies_id_fk", + "tableFrom": "company_skill_mutation_receipts", + "tableTo": "companies", + "columnsFrom": [ + "company_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, "public.company_skill_repository_snapshots": { "name": "company_skill_repository_snapshots", "schema": "", @@ -19079,7 +19171,46 @@ "name": "file_inventory", "type": "jsonb", "primaryKey": false, - "notNull": true + "notNull": true, + "default": "'[]'::jsonb" + }, + "storage": { + "name": "storage", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "upload": { + "name": "upload", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "restore": { + "name": "restore", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "migration_at": { + "name": "migration_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false, + "default": "now()" + }, + "maintenance_at": { + "name": "maintenance_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false, + "default": "now()" + }, + "restore_retry_at": { + "name": "restore_retry_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false }, "created_at": { "name": "created_at", @@ -19110,6 +19241,38 @@ "concurrently": false, "method": "btree", "with": {} + }, + "company_skill_snapshots_migration_idx": { + "name": "company_skill_snapshots_migration_idx", + "columns": [ + { + "expression": "migration_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"company_skill_repository_snapshots\".\"migration_at\" is not null", + "concurrently": false, + "method": "btree", + "with": {} + }, + "company_skill_snapshots_maintenance_idx": { + "name": "company_skill_snapshots_maintenance_idx", + "columns": [ + { + "expression": "maintenance_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"company_skill_repository_snapshots\".\"maintenance_at\" is not null", + "concurrently": false, + "method": "btree", + "with": {} } }, "foreignKeys": { @@ -19999,6 +20162,25 @@ "primaryKey": false, "notNull": false }, + "package_snapshot_id": { + "name": "package_snapshot_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "package_file_map": { + "name": "package_file_map", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "migration_at": { + "name": "migration_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false, + "default": "now()" + }, "revision_number": { "name": "revision_number", "type": "integer", @@ -20057,6 +20239,52 @@ } }, "indexes": { + "company_skill_versions_migration_idx": { + "name": "company_skill_versions_migration_idx", + "columns": [ + { + "expression": "migration_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"company_skill_versions\".\"migration_at\" is not null", + "concurrently": false, + "method": "btree", + "with": {} + }, + "company_skill_versions_package_snapshot_idx": { + "name": "company_skill_versions_package_snapshot_idx", + "columns": [ + { + "expression": "package_snapshot_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "company_skill_versions_repository_snapshot_idx": { + "name": "company_skill_versions_repository_snapshot_idx", + "columns": [ + { + "expression": "repository_snapshot_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, "company_skill_versions_skill_revision_idx": { "name": "company_skill_versions_skill_revision_idx", "columns": [ @@ -20168,6 +20396,19 @@ "onDelete": "restrict", "onUpdate": "no action" }, + "company_skill_versions_package_snapshot_id_company_skill_repository_snapshots_id_fk": { + "name": "company_skill_versions_package_snapshot_id_company_skill_repository_snapshots_id_fk", + "tableFrom": "company_skill_versions", + "tableTo": "company_skill_repository_snapshots", + "columnsFrom": [ + "package_snapshot_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "restrict", + "onUpdate": "no action" + }, "company_skill_versions_author_agent_id_agents_id_fk": { "name": "company_skill_versions_author_agent_id_agents_id_fk", "tableFrom": "company_skill_versions", @@ -20613,6 +20854,78 @@ "checkConstraints": {}, "isRLSEnabled": false }, + "public.skill_package_deletions": { + "name": "skill_package_deletions", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "company_id": { + "name": "company_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "provider": { + "name": "provider", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "archive_key": { + "name": "archive_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "manifest_key": { + "name": "manifest_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "delete_after": { + "name": "delete_after", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "skill_package_deletions_due_idx": { + "name": "skill_package_deletions_due_idx", + "columns": [ + { + "expression": "delete_after", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, "public.company_transfer_runs": { "name": "company_transfer_runs", "schema": "", @@ -33849,7 +34162,7 @@ } ], "isUnique": false, - "where": "\"heartbeat_runs\".\"runtime_mode\" = 'native' AND \"heartbeat_runs\".\"status\" = 'cancelled'\n AND \"heartbeat_runs\".\"runner_profile_json\"->'nativeWorkspaceSync'->>'state' = 'prepared'\n AND \"heartbeat_runs\".\"result_json\"->'nativeCancellation'->>'dispatchState' = 'acknowledged'\n AND (\"heartbeat_runs\".\"result_json\"->'workspaceRestoreRecovery'->>'schema' = 'paperclip.workspace-restore-recovery.v1'\n AND \"heartbeat_runs\".\"result_json\"->>'workspaceRestoreFailure' IN ('restore_permission_denied', 'restore_lock_timeout', 'restore_unsafe_archive', 'restore_failed')) IS NOT TRUE", + "where": "\"heartbeat_runs\".\"runtime_mode\" = 'native' AND \"heartbeat_runs\".\"status\" = 'cancelled'\n AND \"heartbeat_runs\".\"runner_profile_json\"->'nativeWorkspaceSync'->>'state' = 'prepared'\n AND \"heartbeat_runs\".\"result_json\"->'nativeCancellation'->>'dispatchState' = 'acknowledged'\n AND (\"heartbeat_runs\".\"result_json\"->'workspaceRestoreRecovery'->>'schema' = 'paperclip.workspace-restore-recovery.v1'\n AND \"heartbeat_runs\".\"result_json\"->>'workspaceRestoreFailure' IN ('restore_permission_denied', 'restore_lock_timeout', 'restore_unsafe_archive', 'restore_invalid_metadata', 'restore_failed')) IS NOT TRUE", "concurrently": false, "method": "btree", "with": {} @@ -51261,6 +51574,195 @@ "checkConstraints": {}, "isRLSEnabled": false }, + "public.sandbox_creation_operations": { + "name": "sandbox_creation_operations", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true + }, + "company_id": { + "name": "company_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "environment_id": { + "name": "environment_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "original_environment_id": { + "name": "original_environment_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "run_id": { + "name": "run_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "request_key": { + "name": "request_key", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'run'" + }, + "issue_id": { + "name": "issue_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "execution_workspace_id": { + "name": "execution_workspace_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "provider": { + "name": "provider", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "plugin_id": { + "name": "plugin_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "driver_key": { + "name": "driver_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "state": { + "name": "state", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'pending'" + }, + "config": { + "name": "config", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "metadata": { + "name": "metadata", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "lease_id": { + "name": "lease_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "next_attempt_at": { + "name": "next_attempt_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "sandbox_creation_operations_pending_deadline_idx": { + "name": "sandbox_creation_operations_pending_deadline_idx", + "columns": [ + { + "expression": "next_attempt_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"sandbox_creation_operations\".\"state\" = 'pending'", + "concurrently": false, + "method": "btree", + "with": {} + }, + "sandbox_creation_operations_owner_idx": { + "name": "sandbox_creation_operations_owner_idx", + "columns": [ + { + "expression": "company_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "original_environment_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "run_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "request_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "sandbox_creation_operations_environment_id_environments_id_fk": { + "name": "sandbox_creation_operations_environment_id_environments_id_fk", + "tableFrom": "sandbox_creation_operations", + "tableTo": "environments", + "columnsFrom": [ + "environment_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, "public.secret_access_events": { "name": "secret_access_events", "schema": "", @@ -52268,6 +52770,12 @@ "primaryKey": false, "notNull": false }, + "next_eval_reason": { + "name": "next_eval_reason", + "type": "text", + "primaryKey": false, + "notNull": false + }, "next_eval_at": { "name": "next_eval_at", "type": "timestamp with time zone", diff --git a/packages/db/src/migrations/meta/_journal.json b/packages/db/src/migrations/meta/_journal.json index 47d9d449f1a..3cc42512d8c 100644 --- a/packages/db/src/migrations/meta/_journal.json +++ b/packages/db/src/migrations/meta/_journal.json @@ -2920,6 +2920,13 @@ "when": 1793395364937, "tag": "0418_curved_lady_mastermind", "breakpoints": true + }, + { + "idx": 419, + "version": "7", + "when": 1793402408013, + "tag": "0419_moaning_vulture", + "breakpoints": true } ] } \ No newline at end of file diff --git a/packages/db/src/schema/status_cards.ts b/packages/db/src/schema/status_cards.ts index 06efaa0368b..fd18d22fb8b 100644 --- a/packages/db/src/schema/status_cards.ts +++ b/packages/db/src/schema/status_cards.ts @@ -56,6 +56,9 @@ export const statusCards = pgTable( lastModel: text("last_model"), generatingIssueId: uuid("generating_issue_id").references(() => issues.id, { onDelete: "set null" }), failureReason: text("failure_reason"), + // Internal deadline ownership; null denotes a row written before provenance + // was recorded. Membership may be rechecked at startup, retry/policy may not. + nextEvalReason: text("next_eval_reason").$type<"membership" | "policy" | "retry" | "refresh">(), nextEvalAt: timestamp("next_eval_at", { withTimezone: true }), archivedAt: timestamp("archived_at", { withTimezone: true }), archivedByUserId: text("archived_by_user_id"), diff --git a/server/src/__tests__/status-card-evaluation-work.test.ts b/server/src/__tests__/status-card-evaluation-work.test.ts index 4f369cd76e9..4e00e326855 100644 --- a/server/src/__tests__/status-card-evaluation-work.test.ts +++ b/server/src/__tests__/status-card-evaluation-work.test.ts @@ -3,7 +3,7 @@ import { documentService } from "../services/documents.js"; import { randomUUID } from "node:crypto"; import { eq, sql } from "drizzle-orm"; import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; -import { activityLog, agentWakeupRequests, agents, approvals, budgetIncidents, budgetPolicies, costEvents, companies, companyMemberships, createDb, documents, +import { activityLog, agentWakeupRequests, agents, approvals, budgetIncidents, budgetPolicies, costEvents, companies, companyMemberships, createDb, documents, withDedicatedDbConnection, heartbeatRuns, instanceSettings, issueComments, issues, statusCards, statusCardUpdates, documentRevisions } from "@paperclipai/db"; import { companySearchQuerySchema, statusCardRefreshPolicySchema } from "@paperclipai/shared"; import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js"; @@ -188,6 +188,86 @@ const support = external ? { supported: true } : await getEmbeddedPostgresTestSu .toHaveLength(1); }); + it("finishes generation and writeback with a one-connection pool and releases dedicated locks", async () => { + vi.useRealTimers(); + const previousDb = db; + db = createDb(external ?? temporary!.connectionString, { maxConnections: 1, idleTimeoutSeconds: 0, maxLifetimeSeconds: 0 }); + try { + const created = await card("reactive", { ...policy(), debounceSeconds: 1 }); + const countLocks = async () => Number((await db.execute<{ count: string }>(sql` + select count(*) from pg_stat_activity where datname = current_database() + and application_name = 'paperclip-workspace-finalization-lock'`))[0]!.count); + const baseline = await countLocks(); + await issueService(db).create(companyId, { title: "Pool-one blocker", status: "blocked", priority: "medium" }); + const service = statusCardEvaluationService(db, { wakeup }); + const acquired = Promise.withResolvers(), release = Promise.withResolvers(); + const holder = withDedicatedDbConnection(db, connection => connection.transaction(async tx => { + await tx.execute(sql`select pg_advisory_xact_lock(hashtextextended(${created.id}, 0))`); + acquired.resolve(); await release.promise; + })); + await acquired.promise; + try { + await service.run(new AbortController().signal); + expect(wakeup).not.toHaveBeenCalled(); + expect(await service.next()).toBeGreaterThan(Date.now()); + } finally { release.resolve(); await holder; } + await service.run(new AbortController().signal); + expect((await statusCardService(db).getById(created.id))!.pendingChangeCount).toBeGreaterThan(0); + await vi.waitFor(async () => { + await service.run(new AbortController().signal); + expect(wakeup).toHaveBeenCalledOnce(); + }, { timeout: 5000, interval: 100 }); + await writeback(created.id, (await statusCardService(db).getById(created.id))!.generatingIssueId!); + await service.run(new AbortController().signal); + expect(await service.next()).toBeNull(); + expect(idleWorkSnapshot().active).toBe(0); + await vi.waitFor(async () => expect(await countLocks()).toBe(baseline)); + // Failure exits also release the physical connection and advisory lock. + await db.update(statusCards).set({ queries: [{ scope: "invalid" } as never], nextEvalAt: new Date() }).where(eq(statusCards.id, created.id)); + await service.run(new AbortController().signal); + expect((await statusCardService(db).getById(created.id))!.nextEvalReason).toBe("retry"); + await vi.waitFor(async () => expect(await countLocks()).toBe(baseline)); + await withDedicatedDbConnection(db, connection => connection.transaction(async tx => { + expect((await tx.execute<{ acquired: boolean }>(sql`select pg_try_advisory_xact_lock(hashtextextended(${created.id}, 0)) as acquired`))[0]!.acquired).toBe(true); + })); + } finally { await db.$client.end({ timeout: 1 }); db = previousDb; } + }, 15_000); + + it.each([false, true])("recovers a lost committed hint before a future membership deadline (legacy=%s)", async legacy => { + await issueService(db).create(companyId, { title: "Original blocker", status: "blocked", priority: "medium" }); + compiledQuery = companySearchQuerySchema.parse({ scope: "issues", status: ["blocked"], updatedWithin: "1d" }); + const created = await card(); + expect(created.nextEvalReason).toBe("membership"); + const oldExpiry = created.nextEvalAt!.getTime(); + // No coordinator exists to receive this committed source hint. A fresh + // service has no in-memory hint and must reconstruct from durable truth. + await issueService(db).create(companyId, { title: "Committed before crash", status: "blocked", priority: "medium" }); + if (legacy) await db.update(statusCards).set({ nextEvalReason: null }).where(eq(statusCards.id, created.id)); + const restarted = statusCardEvaluationService(db, { wakeup }); + await restarted.run(new AbortController().signal); + const pending = (await statusCardService(db).getById(created.id))!; + expect(pending.pendingChangeCount).toBe(1); + expect(pending.nextEvalReason).toBe("policy"); + expect(pending.nextEvalAt!.getTime()).toBeLessThan(oldExpiry); + expect(wakeup).not.toHaveBeenCalled(); + vi.setSystemTime(pending.nextEvalAt!); await restarted.run(new AbortController().signal); + expect(wakeup).toHaveBeenCalledOnce(); + await writeback(created.id, (await statusCardService(db).getById(created.id))!.generatingIssueId!); + expect((await statusCardService(db).getById(created.id))!.nextEvalReason).toBe("membership"); + }); + + it("preserves a legacy ambiguous five-minute claim without extending it on restart", async () => { + const created = await card(); + const retry = new Date(Date.now() + 240_000); + await db.update(statusCards).set({ nextEvalAt: retry, nextEvalReason: null }).where(eq(statusCards.id, created.id)); + await issueService(db).create(companyId, { title: "Unconsumed source", status: "blocked", priority: "medium" }); + const restarted = statusCardEvaluationService(db, { wakeup }); + await restarted.run(new AbortController().signal); + expect(await restarted.next()).toBe(retry.getTime()); + vi.setSystemTime(retry); await restarted.run(new AbortController().signal); + expect((await statusCardService(db).getById(created.id))!.pendingChangeCount).toBeGreaterThan(0); + }); + it.each(["debounce", "hours", "rate", "budget"] as const)("restores pending %s eligibility after restart without advancing it or acting early", async gate => { vi.setSystemTime(new Date("2026-11-02T18:00:00Z")); const refreshPolicy = { ...policy(), ...(gate === "hours" ? { activeHours: { start: "09:00", end: "17:00", timezone: "UTC" } } : {}), @@ -335,8 +415,12 @@ const support = external ? { supported: true } : await getEmbeddedPostgresTestSu await issueService(db).create(companyId, { title: "Assignment blocker", status: "blocked", priority: "medium" }); await vi.waitFor(async () => expect((await statusCardService(db).getById(created.id))?.pendingChangeCount).toBeGreaterThan(0)); await vi.advanceTimersByTimeAsync(60_000); await settled(); + await vi.waitFor(async () => { + await vi.advanceTimersByTimeAsync(1000); + expect((await statusCardService(db).getById(created.id))!.state).toBe("error"); + }); const failed = (await statusCardService(db).getById(created.id))!; - expect(failed).toMatchObject({ state: "error", generatingIssueId: null, nextEvalAt: null }); + expect(failed).toMatchObject({ state: "error", generatingIssueId: null, nextEvalAt: null, nextEvalReason: null }); expect(errors.length).toBeGreaterThan(0); const history = await db.select().from(statusCardUpdates).where(eq(statusCardUpdates.cardId, created.id)); const ledger = history.find(row => row.status === "failed")!; diff --git a/server/src/__tests__/status-cards.test.ts b/server/src/__tests__/status-cards.test.ts index 0c24b05dd70..05e1bba48e5 100644 --- a/server/src/__tests__/status-cards.test.ts +++ b/server/src/__tests__/status-cards.test.ts @@ -192,6 +192,7 @@ describeEmbeddedPostgres("status card routes", () => { .send({ title: "Launch health", titlePinned: true }); expect(patched.status).toBe(200); expect(patched.body).toMatchObject({ title: "Launch health", titlePinned: true }); + expect(patched.body).not.toHaveProperty("nextEvalReason"); const scheduled = await request(app) .patch(`/api/status-cards/${created.body.id}`) @@ -1152,6 +1153,7 @@ describeEmbeddedPostgres("status card routes", () => { }); expect(summaryWrite.status).toBe(200); expect(summaryWrite.body.card).toMatchObject({ state: "active", queryVersion: 1, generatingIssueId: null }); + expect(summaryWrite.body.card).not.toHaveProperty("nextEvalReason"); expect(summaryWrite.body.document.latestBody).toContain("**Decide:**"); expect(await db.select().from(statusCardUpdates).then((rows) => rows.find((row) => row.kind === "full"))).toMatchObject({ status: "ok", finishedAt: expect.any(Date), inputTokens: 5200, outputTokens: 980 }); diff --git a/server/src/routes/status-cards.ts b/server/src/routes/status-cards.ts index da4f91680bb..d21616b7cf2 100644 --- a/server/src/routes/status-cards.ts +++ b/server/src/routes/status-cards.ts @@ -16,6 +16,12 @@ import { accessService, heartbeatService, instanceSettingsService, issueService, import { queueIssueAssignmentWakeup, type IssueAssignmentWakeupDeps } from "../services/issue-assignment-wakeup.js"; import { assertCompanyAccess, getAccessibleResource, getActorInfo, hasCompanyAccess } from "./authz.js"; +// Deadline provenance belongs to scheduling, not the public card contract. +function publicCard(card: T) { + const { nextEvalReason: _reason, ...result } = card; + return result; +} + export function statusCardRoutes(db: Db, opts: { heartbeat?: IssueAssignmentWakeupDeps } = {}) { const router = Router(); const access = accessService(db); @@ -144,9 +150,9 @@ export function statusCardRoutes(db: Db, opts: { heartbeat?: IssueAssignmentWake assertCompanyAccess(req, companyId); await assertStatusCardsEnabled(); const query = listStatusCardsQuerySchema.parse(req.query); - res.json(await service.list(companyId, query.archived, { + res.json((await service.list(companyId, query.archived, { issueReadCondition: await issueReadSqlCondition(db, req.actor), - })); + })).map(publicCard)); }); router.post("/companies/:companyId/status-cards", validate(createStatusCardSchema), async (req, res) => { @@ -162,7 +168,7 @@ export function statusCardRoutes(db: Db, opts: { heartbeat?: IssueAssignmentWake try { const compile = await enqueueCompile(req, card.id); await logMutation(req, companyId, "status_card.created", card.id, { state: card.state }); - res.status(201).json(compile.card); + res.status(201).json(publicCard(compile.card)); } catch (error) { await service.remove(card.id); throw error; @@ -173,9 +179,9 @@ export function statusCardRoutes(db: Db, opts: { heartbeat?: IssueAssignmentWake await assertStatusCardsEnabled(); const card = await getAccessibleResource(req, res, service.getById(req.params.id as string), "Status card not found"); if (!card) return; - res.json(await service.hydrate(card, { + res.json(publicCard(await service.hydrate(card, { issueReadCondition: await issueReadSqlCondition(db, req.actor), - })); + }))); }); router.patch("/status-cards/:id", validate(patchStatusCardSchema), async (req, res) => { @@ -197,7 +203,7 @@ export function statusCardRoutes(db: Db, opts: { heartbeat?: IssueAssignmentWake fields: Object.keys(req.body), archived: Boolean(updated.archivedAt), }); - res.json(compile?.card ?? restore?.card ?? updated); + res.json(publicCard(compile?.card ?? restore?.card ?? updated)); }); router.delete("/status-cards/:id", async (req, res) => { @@ -234,7 +240,7 @@ export function statusCardRoutes(db: Db, opts: { heartbeat?: IssueAssignmentWake generatingIssueId: result.generatingIssue.id, alreadyGenerating: result.alreadyGenerating, }); - res.status(result.alreadyGenerating ? 200 : 202).json(result); + res.status(result.alreadyGenerating ? 200 : 202).json({ ...result, card: publicCard(result.card) }); }); router.post("/status-cards/:id/refresh", validate(refreshStatusCardSchema), async (req, res) => { @@ -249,7 +255,7 @@ export function statusCardRoutes(db: Db, opts: { heartbeat?: IssueAssignmentWake alreadyGenerating: result.alreadyGenerating, enqueued: result.enqueued, }); - res.status(result.enqueued && !result.alreadyGenerating ? 202 : 200).json(result); + res.status(result.enqueued && !result.alreadyGenerating ? 202 : 200).json({ ...result, card: publicCard(result.card) }); }); router.get("/status-cards/:id/dry-run", async (req, res) => { @@ -289,7 +295,7 @@ export function statusCardRoutes(db: Db, opts: { heartbeat?: IssueAssignmentWake generationIssueId: req.body.generationIssueId, changeSummary: req.body.changeSummary, }); - res.json(updated); + res.json(publicCard(updated)); }); router.put("/status-cards/:id/summary", validate(writeStatusCardSummarySchema), async (req, res) => { @@ -309,7 +315,7 @@ export function statusCardRoutes(db: Db, opts: { heartbeat?: IssueAssignmentWake documentId: result.document.id, changeSummary: req.body.changeSummary, }); - res.json(result); + res.json({ ...result, card: publicCard(result.card) }); }); return router; diff --git a/server/src/services/status-card-finalization.ts b/server/src/services/status-card-finalization.ts index 5d7ee62b523..702509be8b4 100644 --- a/server/src/services/status-card-finalization.ts +++ b/server/src/services/status-card-finalization.ts @@ -47,6 +47,7 @@ export async function finalizeStatusCardsForStalledGeneration( failureReason, generatingIssueId: null, nextEvalAt: null, + nextEvalReason: null, updatedAt: now, }) .where( diff --git a/server/src/services/status-cards.ts b/server/src/services/status-cards.ts index 19e7ac75140..f9e6372ea2e 100644 --- a/server/src/services/status-cards.ts +++ b/server/src/services/status-cards.ts @@ -1,6 +1,6 @@ import { notifyStatusCardEvaluation } from "./status-card-notifications.js"; import { createHash } from "node:crypto"; -import { and, desc, eq, gte, inArray, isNotNull, isNull, lte, ne, or, sql, type SQL } from "drizzle-orm"; +import { and, desc, eq, gt, gte, inArray, isNotNull, isNull, lte, ne, or, sql, type SQL } from "drizzle-orm"; import { agents, companies, @@ -11,6 +11,7 @@ import { issueComments, statusCards, statusCardUpdates, + withDedicatedDbConnection, type Db, } from "@paperclipai/db"; import type { @@ -301,11 +302,12 @@ export function statusCardService( ...(input.refreshPolicy !== undefined ? { refreshPolicy: input.refreshPolicy, + nextEvalReason: "refresh", nextEvalAt: card.archivedAt ? null : input.refreshPolicy.mode === "reactive" && card.queries.length > 0 ? now : nextStatusCardEvaluationAt(input.refreshPolicy, now), } : {}), ...(archiveChanged && input.archived - ? { archivedAt: now, archivedByAgentId: actor.agentId, archivedByUserId: actor.userId, nextEvalAt: null } + ? { archivedAt: now, archivedByAgentId: actor.agentId, archivedByUserId: actor.userId, nextEvalAt: null, nextEvalReason: null } : {}), ...(archiveChanged && !input.archived ? { @@ -315,6 +317,7 @@ export function statusCardService( lastChangeAt: now, lastUpdateRunKind: null, nextEvalAt: card.queries.length > 0 ? now : null, + nextEvalReason: "refresh", } : {}), }; @@ -537,9 +540,10 @@ export function statusCardService( companyId: string, issueIds: string[], options: StatusCardReadOptions = {}, + queryDb: Db = db, ): Promise { if (issueIds.length === 0) return []; - const rows = await db + const rows = await queryDb .select({ id: issues.id, identifier: issues.identifier, @@ -589,11 +593,11 @@ export function statusCardService( return loadIssueSummaries(card.companyId, card.mentionedIssueIds ?? [], options); } - async function executeQueries(card: StatusCardRow, options: StatusCardReadOptions = {}, onNextMembershipChange?: (at: Date | null) => void) { + async function executeQueries(card: StatusCardRow, options: StatusCardReadOptions = {}, onNextMembershipChange?: (at: Date | null) => void, queryDb: Db = db) { const issueMap = new Map(); for (const storedQuery of card.queries) { const query = companySearchQuerySchema.parse(storedQuery); - const response = await searchSvc.search(card.companyId, query, { + const response = await companySearchService(queryDb).search(card.companyId, query, { issueReadCondition: options.issueReadCondition, onNextMembershipChange, }); for (const result of response.results) { @@ -605,12 +609,12 @@ export function statusCardService( const mentioned = await loadIssueSummaries( card.companyId, (card.mentionedIssueIds ?? []).filter((issueId) => !issueMap.has(issueId)), - options, + options, queryDb, ); for (const issue of mentioned) issueMap.set(issue.id, issue); const snapshot = [...issueMap.values()]; if (snapshot.length === 0) return snapshot; - const latestHumanComments = await db + const latestHumanComments = await queryDb .select({ issueId: issueComments.issueId, // The postgres-js driver returns the `max()` aggregate over a timestamp @@ -672,6 +676,7 @@ export function statusCardService( lastChangeAt: null, state: "active", nextEvalAt, + nextEvalReason: nextEvalAt ? card.refreshPolicy.mode === "reactive" ? "membership" : "refresh" : null, }).where(eq(statusCards.id, card.id)).returning()); return { card: next!, generatingIssue: null, alreadyGenerating: false, enqueued: false }; } @@ -699,6 +704,7 @@ export function statusCardService( pendingChangeHash, lastChangeAt, state: decision.action === "pause_budget" ? "paused_budget" : decision.action === "pause_hours" ? "paused_hours" : "active", + nextEvalReason: "policy", nextEvalAt: nextStatusCardPolicyEligibility({ policy: card.refreshPolicy, now, lastChangeAt, completedUpdateTimes: recent.filter(row => row.kind !== "compile" && row.finishedAt).map(row => row.startedAt), @@ -755,6 +761,7 @@ export function statusCardService( lastChangeAt, state: "active", nextEvalAt, + nextEvalReason: nextEvalAt ? card.refreshPolicy.mode === "reactive" ? "membership" : "refresh" : null, failureReason: null, }).where(and(eq(statusCards.id, card.id), isNull(statusCards.archivedAt), or(isNull(statusCards.generatingIssueId), priorGenerationPredicate))).returning()); if (!next) { @@ -788,9 +795,16 @@ export function statusCardService( async function tickDueStatusCards(now = new Date(), reactiveCompanies?: ReadonlySet | "changed", reconstruct = false) { const changedScope = reactiveCompanies === "changed" ? sql`true` : reactiveCompanies?.size ? inArray(statusCards.companyId, [...reactiveCompanies]) : sql`false`; - // Startup restores every disarmed reactive scope. A concurrent committed - // source hint can additionally preempt its company's saved future deadline. - const reactiveScope = or(reconstruct ? isNull(statusCards.nextEvalAt) : undefined, changedScope); + // Startup recovers lost source hints for membership deadlines as well as + // disarmed cards. Saved policy gates and retry leases keep their owner. + // Legacy untagged leases are at most five minutes from their claim time: + // preserve that bounded ambiguity, never a days-away membership deadline. + const startupScope = reconstruct ? or( + isNull(statusCards.nextEvalAt), eq(statusCards.nextEvalReason, "membership"), + and(isNull(statusCards.nextEvalReason), isNull(statusCards.pendingChangeHash), + eq(statusCards.state, "active"), gt(statusCards.nextEvalAt, new Date(now.getTime() + 5 * 60_000))), + ) : undefined; + const reactiveScope = or(startupScope, changedScope); const due = await db.select({ card: statusCards }).from(statusCards) .innerJoin(companies, eq(companies.id, statusCards.companyId)) .where(and(eq(companies.status, "active"), isNull(statusCards.archivedAt), isNull(statusCards.generatingIssueId), @@ -805,13 +819,13 @@ export function statusCardService( // event may preempt a genuine future deadline, so serialize scheduler // evaluation across connections without holding a row lock while the // existing issue/card services commit through their own transactions. - await db.transaction(async lockTx => { + await withDedicatedDbConnection(db, connection => connection.transaction(async lockTx => { const [lock] = await lockTx.execute<{ acquired: boolean }>( sql`select pg_try_advisory_xact_lock(hashtextextended(${candidate.id}, 0)) as acquired`, ); if (!lock?.acquired) { deferredCompanyIds.add(candidate.companyId); return; } const claimUntil = new Date(now.getTime() + 5 * 60 * 1000); - const [claimed] = await mutate(candidate.companyId, tx => tx.update(statusCards).set({ nextEvalAt: claimUntil }) + const [claimed] = await mutate(candidate.companyId, tx => tx.update(statusCards).set({ nextEvalAt: claimUntil, nextEvalReason: "retry" }) .where(and(eq(statusCards.id, candidate.id), isNull(statusCards.archivedAt), isNull(statusCards.generatingIssueId), candidate.nextEvalAt ? eq(statusCards.nextEvalAt, candidate.nextEvalAt) : isNull(statusCards.nextEvalAt))) .returning()); @@ -826,7 +840,7 @@ export function statusCardService( "status card scheduled refresh failed", ); } - }); + })); } return { evaluated, enqueued, deferredCompanyIds }; } @@ -859,7 +873,9 @@ export function statusCardService( // Current watched membership: compiled-query matches plus the issues this // summary mentions. let membershipAt: Date | null = null; - const watchedSnapshot = await executeQueries({ ...current, mentionedIssueIds }, {}, at => { if (at && (!membershipAt || at < membershipAt)) membershipAt = at; }); + // All watched-source reads use this transaction. Borrowing the root + // pool here would deadlock a one-connection installation. + const watchedSnapshot = await executeQueries({ ...current, mentionedIssueIds }, {}, at => { if (at && (!membershipAt || at < membershipAt)) membershipAt = at; }, tx as unknown as Db); const watchedNow = buildStatusCardFingerprint(watchedSnapshot); let snapshot: StatusCardFingerprint; if (payloadFingerprint) { @@ -931,6 +947,9 @@ export function statusCardService( pendingChangeCount: 0, pendingChangeHash: null, lastChangeAt: null, + nextEvalReason: current.refreshPolicy.mode === "reactive" && + filterStatusCardChanges(diffStatusCardFingerprint(snapshot, watchedNow), current.refreshPolicy).length > 0 + ? "refresh" : membershipAt ? "membership" : null, nextEvalAt: current.refreshPolicy.mode === "reactive" && filterStatusCardChanges(diffStatusCardFingerprint(snapshot, watchedNow), current.refreshPolicy).length > 0 ? now : nextEvaluationAt(current, membershipAt, now), From 91261d5a038d4d009c01ba102da4a00921caa175 Mon Sep 17 00:00:00 2001 From: Dotta <34892728+cryppadotta@users.noreply.github.com> Date: Fri, 30 Oct 2026 18:36:36 -0500 Subject: [PATCH 48/70] fix(server): preserve budget history for deleted resources (#18389) ## Thinking Path > - Paperclip is the open source app people use to manage AI agents for work. > - Budget views show saved policies, incidents and spend for each company. > - A policy can remain after its agent or project has been deleted. > - The overview currently stops when it cannot resolve that old resource. > - A current agent's budget then becomes unreadable during normal setup. > - This pull request preserves the historical row in display summaries. > - Admission and policy changes still require a real company-owned resource. ## Linked Issues or Issue Description **What happened?** The budget overview throws `Agent not found` when a saved policy references a deleted agent. One old resource prevents the company from reading its current budgets. The same problem affects historical project policies and incident summaries. **Expected behavior** The overview should retain historical policy and incident IDs and spend. It should display `Deleted agent` or `Deleted project` for a missing resource. Read access must not grant permission to execute work or change a budget for that resource. **Steps to reproduce** 1. Create an agent and a budget policy for it. 2. Permanently delete the agent after its other dependencies have been cleared. 3. Read the company's budget overview. The existing service throws instead of returning its remaining current and historical rows. **Paperclip version or commit** The isolated repair starts from merged commit `d9354bbfc35ef74333e90e596ce21dc8e59149b9` and integrates the exact merged baseline test fix `8595efc20bda3ce20a9fbbf34a5d156a6fcf6d57`. It also integrates the actual merged Antigravity auth fix `18b1637b6371a6a6adade857e2a73654ee626fef` (#18254). The diff against the selected merged parent remains two budget files. **Related work** Earlier reports and implementations: #11058 and #4333. This extracts the product prerequisite from #18351 so it can qualify independently of that journey's other dependencies. The change contains only the budget service repair and four database regression cases. ## What Changed - Resolve deleted agent and project scopes only when building display summaries. Preserve their original IDs, policies, incidents and accounting records. - Reject a saved policy or incident that resolves to another company's resource. Propagate other read failures. - Keep missing-company resolution, work admission and policy changes strict. Validate company, policy and incident scope bindings before either incident action. A budget raise requires a live company-owned resource. Historical dismissal may proceed only for a genuinely deleted owned agent/project, and must reject live foreign or mismatched scopes. - Add four canonical PostgreSQL regression cases: deleted agent history, deleted project history, missing-scope admission and budget raises with safe historical dismissal, and foreign/mismatched references. The history cases also verify current ledger spend and unchanged stored rows. ## Verification - Real-service RED/GREEN against official PGlite with every canonical migration: the original overview throws for deleted history; the repair returns the historical IDs, incidents and spend while retaining current budgets. - Mutation regression RED/GREEN: missing-resource budget raises must reject before writes; owned deleted-scope dismissal must reject its linked pending approval, retain policy/agent/ledger data and incident history, and remove the incident from the active list. Live foreign and mismatched scopes reject both actions without mutations. - Current integrated head `b3434f5ad05fd3067379352700c36bfe826179c4`: all four canonical PostgreSQL regressions pass in [server 2/12](https://github.com/paperclipai/paperclip/actions/runs/41142588583/job/124361194077), and all five existing budget-service tests pass in [server 7/12](https://github.com/paperclipai/paperclip/actions/runs/41142588583/job/124361194080). Both use the CI merge against selected parent `18b1637`. - Clean-build declaration typing passes in the current CI typecheck gate. The earlier local source-alias no-emit check also passed, including the canonical actor argument. The local worktree borrows compiled dependencies from a sibling checkout; a later real-service import was blocked by its stale `sandboxCreationOperations` export, so clean-build CI is authoritative for the final integration. No production workaround was made. - `git diff --check` passes. Broad repository typecheck, tests and build run in current-head CI rather than repeated locally. Native local PostgreSQL could not start because this host exhausted its semaphore pool; no global IPC settings or unrelated processes were changed. - One unchanged Runner lane retry was authorized after a probable intermittent close-settlement fixture failure. The [original failure](https://github.com/paperclipai/paperclip/actions/runs/41142588583/job/124360234111) occurred in `omitted-context` cleanup. A [fresh passing Linux lane](https://github.com/paperclipai/paperclip/actions/runs/41142552195/job/124360110386) passed the same case and all 1,998 tests with byte-identical entire Runner and adapter-utils trees, lockfile, root package, scripts and workflows, against the same merged base. The fixture has a five-second close allowance with a 2.5-second suspension reserve; a host stall is not proven. No budget path or changed close input was found. The [single targeted retry](https://github.com/paperclipai/paperclip/actions/runs/41142588583/job/124365812609) passed the same case in 1,034 ms and all 1,998 lane tests; dependent verification also passed. All 57 current PR checks are terminal and green. - The product helper requires all current CI and clean review before merge. Actual QA budget-overview proof will run on the combined final app before hiring qualification. No deployed success is claimed. The older `b4863dd` artifact was never deployed; selected parent `18b1637` adds sandbox/lifecycle changes and migration 0418, so whole-app identity is not claimed. The budget module and relevant budget contracts remain unchanged, and final hiring requires the combined final app with explicit source identities. ## Risks Budget summaries now return historical rows whose resources have been deleted. These rows use explicit display names and preserve their accounting data. They do not create a resource or relax work admission, authorization or budget grants. Historical incident dismissal changes only that incident and its linked approval; it does not resume work or raise a budget. Cross-company references still fail closed. There is no schema migration and no new background work. ## Model Used OpenAI Codex, GPT-6 family. The session does not expose a more specific backend model ID or fixed context window. Used reasoning, repository tools, TypeScript and database test execution. ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above - [x] I have either (a) linked existing issues with `Fixes: #` / `Closes #` / `Refs #` OR (b) described the issue in-PR following the relevant issue template - [x] I have not referenced internal/instance-local Paperclip issues or links (only public GitHub `#NNN` / `github.com/paperclipai/paperclip` URLs) - [x] My branch name describes the change (e.g. `docs/...`, `fix/...`) and contains no internal Paperclip ticket id or instance-derived details - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [x] I have updated relevant documentation to reflect my changes (the PR description documents the display-only history and strict-mutation contract) - [x] I have considered and documented any risks above - [x] All Paperclip CI gates are green - [x] Greptile is 5/5 with no open P2s, recommendations, or follow-ups - [x] I will address all Greptile and reviewer comments before requesting merge --------- Co-authored-by: Paperclip --- server/src/__tests__/budgets-history.test.ts | 109 +++++++++++++++++++ server/src/services/budgets.ts | 24 +++- 2 files changed, 131 insertions(+), 2 deletions(-) create mode 100644 server/src/__tests__/budgets-history.test.ts diff --git a/server/src/__tests__/budgets-history.test.ts b/server/src/__tests__/budgets-history.test.ts new file mode 100644 index 00000000000..5b106329d72 --- /dev/null +++ b/server/src/__tests__/budgets-history.test.ts @@ -0,0 +1,109 @@ +import { randomUUID } from "node:crypto"; +import { eq } from "drizzle-orm"; +import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest"; +import { agents, approvals, budgetIncidents, budgetPolicies, companies, costEvents, createDb, projects, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS } from "@paperclipai/db"; +import { budgetService } from "../services/budgets.js"; +import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; + +vi.mock("../services/activity-log.js", () => ({ logActivity: vi.fn() })); +const support = await getEmbeddedPostgresTestSupport(); + +describe.skipIf(!support.supported)("budget history for deleted scopes", () => { + let db: ReturnType; + let database: Awaited>; + beforeAll(async () => { + database = await startEmbeddedPostgresTestDatabase("paperclip-budget-history-"); + db = createDb(database.connectionString); + }, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS); + afterEach(async () => { + await db.delete(budgetIncidents); await db.delete(approvals); await db.delete(budgetPolicies); await db.delete(costEvents); + await db.delete(projects); await db.delete(agents); await db.delete(companies); + }); + afterAll(async () => { await db?.$client.end({ timeout: 1 }); await database?.cleanup(); }); + + async function fixture() { + const companyId = randomUUID(), agentId = randomUUID(); + await db.insert(companies).values({ id: companyId, name: "History company", issuePrefix: `B${companyId.replace(/-/g, "").slice(0, 6).toUpperCase()}` }); + await db.insert(agents).values({ id: agentId, companyId, name: "Current agent", role: "engineer" }); + const [policy] = await db.insert(budgetPolicies).values({ companyId, scopeType: "agent", scopeId: agentId, + metric: "billed_cents", windowKind: "calendar_month_utc", amount: 300 }).returning(); + await db.insert(costEvents).values({ companyId, agentId, provider: "openai", model: "history-fixture", inputTokens: 1, outputTokens: 1, costCents: 17, occurredAt: new Date() }); + return { companyId, agentId, policy: policy! }; + } + + it.each(["agent", "project"] as const)("keeps current budgets and deleted %s policy/incident history readable without changing stored rows", async scopeType => { + const { companyId, policy } = await fixture(), deletedId = randomUUID(); + if (scopeType === "agent") await db.insert(agents).values({ id: deletedId, companyId, name: "Former agent", role: "engineer" }); + else await db.insert(projects).values({ id: deletedId, companyId, name: "Former project" }); + const [historical] = await db.insert(budgetPolicies).values({ companyId, scopeType, scopeId: deletedId, + metric: "billed_cents", windowKind: "calendar_month_utc", amount: 200 }).returning(); + await db.insert(budgetIncidents).values({ companyId, policyId: historical!.id, scopeType, scopeId: deletedId, + metric: "billed_cents", windowKind: "calendar_month_utc", windowStart: new Date("2020-01-01"), windowEnd: new Date("2020-02-01"), + thresholdType: "hard", amountLimit: 200, amountObserved: 201 }); + if (scopeType === "agent") await db.delete(agents).where(eq(agents.id, deletedId)); + else await db.delete(projects).where(eq(projects.id, deletedId)); + const policiesBefore = await db.select().from(budgetPolicies), incidentsBefore = await db.select().from(budgetIncidents), costsBefore = await db.select().from(costEvents); + + const overview = await budgetService(db).overview(companyId); + expect(overview.policies).toHaveLength(2); + expect(overview.policies.find(row => row.policyId === policy.id)).toMatchObject({ companyId, scopeName: "Current agent", amount: 300, observedAmount: 17 }); + expect(overview.policies.find(row => row.policyId === historical!.id)).toMatchObject({ companyId, scopeType, scopeId: deletedId, scopeName: `Deleted ${scopeType}`, amount: 200 }); + expect(overview.activeIncidents).toMatchObject([{ companyId, policyId: historical!.id, scopeId: deletedId, scopeName: `Deleted ${scopeType}`, status: "open", amountObserved: 201 }]); + expect(await db.select().from(budgetPolicies)).toEqual(policiesBefore); + expect(await db.select().from(budgetIncidents)).toEqual(incidentsBefore); + expect(await db.select().from(costEvents)).toEqual(costsBefore); + }); + it("rejects missing-scope budget grants but can dismiss their historical incidents", async () => { + const { companyId, agentId } = await fixture(), missingId = randomUUID(), service = budgetService(db); + await expect(service.evaluateAdmission(companyId, missingId)).rejects.toThrow("Agent not found"); + await expect(service.evaluateAdmission(companyId, agentId, { projectId: missingId })).rejects.toThrow("Project not found"); + for (const scopeType of ["agent", "project"] as const) { + await expect(service.upsertPolicy(companyId, { scopeType, scopeId: missingId, amount: 100 }, "board")).rejects.toThrow(`${scopeType === "agent" ? "Agent" : "Project"} not found`); + const [policy] = await db.insert(budgetPolicies).values({ companyId, scopeType, scopeId: missingId, + windowKind: "calendar_month_utc", amount: 100 }).returning(); + const [approval] = await db.insert(approvals).values({ companyId, type: "budget_override_required", payload: {} }).returning(); + const [incident] = await db.insert(budgetIncidents).values({ companyId, policyId: policy!.id, approvalId: approval!.id, scopeType, scopeId: missingId, + metric: "billed_cents", windowKind: "calendar_month_utc", windowStart: new Date("2020-01-01"), windowEnd: new Date("2020-02-01"), + thresholdType: "hard", amountLimit: 100, amountObserved: 101 }).returning(); + const before = { policies: await db.select().from(budgetPolicies), incidents: await db.select().from(budgetIncidents), + agents: await db.select().from(agents), costs: await db.select().from(costEvents), approvals: await db.select().from(approvals) }; + await expect(service.resolveIncident(companyId, incident!.id, { action: "raise_budget_and_resume", amount: 200 }, "board")) + .rejects.toThrow(`${scopeType === "agent" ? "Agent" : "Project"} not found`); + expect({ policies: await db.select().from(budgetPolicies), incidents: await db.select().from(budgetIncidents), + agents: await db.select().from(agents), costs: await db.select().from(costEvents), approvals: await db.select().from(approvals) }).toEqual(before); + const dismissed = await service.resolveIncident(companyId, incident!.id, { action: "keep_paused" }, "board"); + expect(dismissed).toMatchObject({ companyId, policyId: policy!.id, scopeId: missingId, scopeName: `Deleted ${scopeType}`, status: "dismissed" }); + const { status: _status, resolvedAt: _resolvedAt, updatedAt: _updatedAt, ...preserved } = incident!; + const [saved] = await db.select().from(budgetIncidents).where(eq(budgetIncidents.id, incident!.id)); + expect(saved).toMatchObject({ ...preserved, status: "dismissed", resolvedAt: expect.any(Date) }); + expect((await db.select().from(approvals).where(eq(approvals.id, approval!.id)))[0]).toMatchObject({ status: "rejected" }); + expect(await db.select().from(budgetPolicies)).toEqual(before.policies); + expect(await db.select().from(agents)).toEqual(before.agents); + expect(await db.select().from(costEvents)).toEqual(before.costs); + expect((await service.overview(companyId)).activeIncidents.some(row => row.id === incident!.id)).toBe(false); + } + }); + + it("rejects a corrupted policy pointing to another company's scope instead of exposing its name", async () => { + const { companyId } = await fixture(), foreignCompanyId = randomUUID(), foreignAgentId = randomUUID(); + await db.insert(companies).values({ id: foreignCompanyId, name: "Other company", issuePrefix: `F${foreignCompanyId.replace(/-/g, "").slice(0, 6).toUpperCase()}` }); + await db.insert(agents).values({ id: foreignAgentId, companyId: foreignCompanyId, name: "Private other-company agent", role: "engineer" }); + const [policy] = await db.insert(budgetPolicies).values({ companyId, scopeType: "agent", scopeId: foreignAgentId, windowKind: "calendar_month_utc", amount: 200 }).returning(); + const [incident] = await db.insert(budgetIncidents).values({ companyId, policyId: policy!.id, scopeType: "agent", scopeId: foreignAgentId, + metric: "billed_cents", windowKind: "calendar_month_utc", windowStart: new Date("2020-01-01"), windowEnd: new Date("2020-02-01"), + thresholdType: "hard", amountLimit: 200, amountObserved: 201 }).returning(); + const service = budgetService(db); + await expect(service.overview(companyId)).rejects.toThrow("Budget scope not found"); + for (const scopeId of [foreignAgentId, randomUUID()]) { + await db.update(budgetIncidents).set({ scopeId }).where(eq(budgetIncidents.id, incident!.id)); + const before = { policies: await db.select().from(budgetPolicies), incidents: await db.select().from(budgetIncidents), + agents: await db.select().from(agents), costs: await db.select().from(costEvents) }; + for (const action of ["raise_budget_and_resume", "keep_paused"] as const) { + await expect(service.resolveIncident(companyId, incident!.id, { action, amount: 300 }, "board")).rejects.toThrow("Budget scope not found"); + expect({ policies: await db.select().from(budgetPolicies), incidents: await db.select().from(budgetIncidents), + agents: await db.select().from(agents), costs: await db.select().from(costEvents) }).toEqual(before); + } + } + }); + +}); diff --git a/server/src/services/budgets.ts b/server/src/services/budgets.ts index 6777124ee90..3a15d489f0a 100644 --- a/server/src/services/budgets.ts +++ b/server/src/services/budgets.ts @@ -163,6 +163,19 @@ async function resolveScopeRecord(db: Db, scopeType: BudgetScopeType, scopeId: s }; } +async function resolveSummaryScopeRecord(db: Db, companyId: string, scopeType: BudgetScopeType, scopeId: string): Promise { + let scope: ScopeRecord; + try { + scope = await resolveScopeRecord(db, scopeType, scopeId); + } catch (error) { + if (!(error instanceof HttpError) || error.status !== 404 || (scopeType !== "agent" && scopeType !== "project")) throw error; + return { companyId, name: scopeType === "agent" ? "Deleted agent" : "Deleted project", + paused: false, pauseReason: null, budgetHeld: false }; + } + if (scope.companyId !== companyId) throw notFound("Budget scope not found"); + return scope; +} + // Keep a large dashboard from filling the database pool with queued reads. async function mapBudgetReads(rows: T[], read: (row: T) => Promise): Promise { const result: R[] = []; @@ -395,7 +408,7 @@ export function budgetServiceInTransaction(db: Db, publications: ActivityPublica } async function buildPolicySummary(policy: PolicyRow): Promise { - const scope = await resolveScopeRecord(db, policy.scopeType as BudgetScopeType, policy.scopeId); + const scope = await resolveSummaryScopeRecord(db, policy.companyId, policy.scopeType as BudgetScopeType, policy.scopeId); const observed = await computeObservedSpend(db, policy); const { total: observedAmount, totalExact: observedAmountExact, unpricedEventCount, pendingRunCount } = observed; const { windowStart: start, windowEnd: end } = observed; @@ -560,7 +573,7 @@ export function budgetServiceInTransaction(db: Db, publications: ActivityPublica const approvalStatusById = new Map(approvalRows.map((row) => [row.id, row.status])); return mapBudgetReads(rows, async (row) => { - const scope = await resolveScopeRecord(db, row.scopeType as BudgetScopeType, row.scopeId); + const scope = await resolveSummaryScopeRecord(db, row.companyId, row.scopeType as BudgetScopeType, row.scopeId); return { id: row.id, companyId: row.companyId, @@ -925,6 +938,13 @@ export function budgetServiceInTransaction(db: Db, publications: ActivityPublica if (incident.companyId !== companyId) throw notFound("Budget incident not found"); const policy = await getPolicyRow(incident.policyId); + if (policy.companyId !== companyId || policy.scopeType !== incident.scopeType || policy.scopeId !== incident.scopeId) { + throw notFound("Budget scope not found"); + } + const scope = input.action === "keep_paused" + ? await resolveSummaryScopeRecord(db, companyId, policy.scopeType as BudgetScopeType, policy.scopeId) + : await resolveScopeRecord(db, policy.scopeType as BudgetScopeType, policy.scopeId); + if (scope.companyId !== companyId) throw notFound("Budget scope not found"); if (input.action === "raise_budget_and_resume") { const nextAmount = Math.max(0, Math.floor(input.amount ?? 0)); const observed = await computeObservedSpend(db, policy); From 1353a232f2cd9e5fb58933689e485a9ff1f1480f Mon Sep 17 00:00:00 2001 From: Dotta <34892728+cryppadotta@users.noreply.github.com> Date: Fri, 30 Oct 2026 18:37:17 -0500 Subject: [PATCH 49/70] test(e2e): add a portable active follow-up journey (#18345) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Thinking Path > - Paperclip lets people direct agents while their work is active. > - Product E2E checks must prove that saved user input reaches the intended work. > - The existing active chat story depends on a file in a local agent workspace. > - That setup cannot qualify an attached deployment with remote agent execution. > - This change adds one shared active follow-up journey through the normal task UI and API. > - Independent native operation, saved input, final reply, and cleanup checks make the result inspectable across targets. ## Linked Issues or Issue Description **What existing behavior does this improve?** Product E2E coverage for a user follow-up sent while a native agent is working. **Current behavior** The classic chat story holds work open with a local file. The shared deployment-targeted harness has no active follow-up journey. A running heartbeat alone cannot prove that work was active when input arrived. **Proposed behavior** Run one bounded command through the real native provider. Observe its active execution before and after one follow-up is saved. Require one final reply with the late requirement, correct task and agent identity, an empty input queue, and visible readback after reload. Use the same story on managed and attached targets. ## What Changed - Add a shared `followup-while-running` journey with normal task creation and reply submission. - Require a bound native turn and pending shell execution at the input boundary. Retain a closed receipt with source identity and a row hash. - Accept one steered work run or one normal successor (1–2 native runs). Preserve every other journey's declared run contract. - Reuse the saved native final reply observer, run binding checks, usage collection, target lifecycle and private evidence path. - Accept canonical builtin/process execution events independent of provider tool display names; calibrate against real OpenCode, Codex and ACP mappers. Reject Paperclip semantic tool names across native wrappers/prefixes and unnamed or ambiguous builtin calls. - Recognize only the observed legacy PRP v1 builtin/bash/unknown/readOnly=false command shape and fail promptly when owned work finishes before a live boundary can be observed. - Ready the task page/composer before the observed command boundary, use a bounded 90-second command, and preserve active-after-persistence proof. - Add adversarial oracle and retained-result calibration. Register the journey, include its sources in the definition hash, and document its coverage. - Keep six real browser document-readiness checks and verify shared-deadline behavior with a deterministic clock/state test that rejects timeout-reset mutations. ## Verification Exact current head `e72e20ad111bd94728d38606015d1fdac3c92929` passed [full CI run 41142552195](https://github.com/paperclipai/paperclip/actions/runs/41142552195): all 49 workflow jobs reached accepted terminal results, including typecheck, build, every test lane and aggregate verify. All 59 reported checks are terminal: 55 success, three expected skips and one neutral 5/5 Paperclip review. Snyk passed. [Fresh Greptile review](https://github.com/paperclipai/paperclip/pull/18345#issuecomment-6528913235) explicitly names this exact head and gives 5/5. No review threads remain open. The PR is mergeable. These source gates do not establish a live QA pass at this head. Fresh protected QA at this exact source and the separately recorded deployed application/runtime remains required before primary merge and daily admission. Historical live QA staging **PASS** on qualified commit `af9e091b15b5e24b6b2cc1b424523430cc2ab024`: 6/6 checks, all five phases including cleanup, two native work runs, and the configured model/environment check passed. The [sanitized qualification report](https://pages.paperclip.ing/evals-qa-smoke-41122620146-1/) records that exact measurement. Final integration head `e72e20ad111bd94728d38606015d1fdac3c92929` merges exact actual squash `18b1637b6371a6a6adade857e2a73654ee626fef` from [#18254](https://github.com/paperclipai/paperclip/pull/18254) into previous head `e54d64743964b1a19d165537bebbe62c50389a23`. The merge had no conflicts. All 18 files in this PR's change set, the complete 45-file shared journey directory, and six shared native/evidence helpers retain their exact previous-head Git blobs: 59 scoped files in total. The production clarification helper remains blob `800a747284db19b8484650b6759e8bd1f356839d`. Follow-up 1–2 runs, question 1–3 runs, older exact counts, owned native event checks, active-before/after persistence proof, cleanup/accounting and feature mappings remain unchanged. Source integration head `e55d74491f5bcd99c1455e9ace02bd7ad7319fae` integrates exact actual squash `8595efc20bda3ce20a9fbbf34a5d156a6fcf6d57` from [#18385](https://github.com/paperclipai/paperclip/pull/18385). The merge had no conflicts. All 17 files in this PR's change set and six shared native/evidence helpers retain their exact Git blobs from previous head `162e341cda5b627639b486b4019a1675bb018ea2`. This preserves the follow-up and question readiness checks, feature mappings, six closed follow-up check IDs, immutable before/after active-operation proof, follow-up 1–2 runs, question 1–3 runs and older exact counts. Previously reviewed head `e54d64743964b1a19d165537bebbe62c50389a23` adds a separate test-only reliability repair on top of that source integration. It replaces one scheduler-sensitive browser case with a deterministic Vitest clock and locator-state model that calls the actual production helper. The six other real Chromium checks remain byte-for-byte unchanged and pass **6/6**. Focused unit suites pass **51/51**, including the deterministic deadline case. This is six browser checks plus focused unit coverage, not seven browser checks. The model makes the panel ready at 100 ms, the supplied answer ready at 600 ms, and full valid content ready at 1,300 ms. It requires rejection at the shared 1,000 ms deadline and records immutable rejection state before simulating a 10-second scheduler stall. Both restarted-timeout mutations fail: resetting all waits wrongly resolves at 1,300 ms; resetting matcher timeouts while keeping the final real deadline guard rejects only at 1,300 ms with valid content present. The genuine helper rejects at 1,000 ms with the answer still incomplete. The production helper is unchanged, Git blob `800a747284db19b8484650b6759e8bd1f356839d`. That two-file test-only patch changed no journey, application, run, accounting, model or runtime behavior. Its six-browser/51-unit results are prior-head evidence; the complete patch is unchanged in the new integration. The final source also retains the actual merged question-primary squash `d9354bbfc35ef74333e90e596ce21dc8e59149b9` from [#18346](https://github.com/paperclipai/paperclip/pull/18346). Later commits `409117451da260a03fc28a3107c7808ecbe68f8f` and `ea3e7b1d1bed562351fcf4722c7175001b987b59` bound question document readiness to one campaign deadline and test that waits cannot restart it. Commit `162e341cda5b627639b486b4019a1675bb018ea2` adds question/follow-up report feature mappings and coverage. These files are unchanged by the final merge. The imported upstream source has real application changes: [#18334](https://github.com/paperclipai/paperclip/pull/18334) makes managed skill content durable and extends native skill-test contracts; [#18350](https://github.com/paperclipai/paperclip/pull/18350) requires managed Cloud shared storage and bootstrap prerequisites. #18385 updates TestOps job metadata and repairs their combined maintenance fixture. The new merge additionally imports [#18383](https://github.com/paperclipai/paperclip/pull/18383) at `81ecaf0654fe79a2a2dbda6588f033e39ee610e4`, which bounds failed lifecycle verification; [#18357](https://github.com/paperclipai/paperclip/pull/18357) at `45a3a92e6dc52d20923b9aa34a22fee2ec5443f9`, which persists sandbox-creation ownership and updates cleanup/adapter/plugin contracts; and #18254 at `18b1637b6371a6a6adade857e2a73654ee626fef`, which accepts a valid late Antigravity authorization URL while native authentication drains. This PR's diff against exact upstream `18b1637b6371a6a6adade857e2a73654ee626fef` contains only journey tests, report coverage and documentation. Harness source and deployed application/runtime identity are separate inputs. This source integration does not deploy the imported application changes or claim the QA target runs this application revision. This final SHA has not been measured by the historical run above: **fresh QA at this exact final head is required before primary merge**, followed by normal TestOps verification/admission. Fresh exact-head CI and review have passed as recorded above. The live measurement must record the actual separately deployed QA application/runtime identity. Seven whole-file blobs remain identical to the qualified SHA. All paths are relative to `tests/runner-e2e/`: | Journey/helper | Unchanged Git blob | | --- | --- | | `product-journeys/active-followup-evidence.ts` | `83a5d9fab52abb9849a40e83eca6b6a9c29bf587` | | `product-journeys/active-followup.test.ts` | `728160c610c18292d2cf06d4a87d25a8719f1734` | | `native-task-reply.ts` | `59d11f1a178dad9db3da8fa2342ff93643cb9922` | | `native-event-envelope.ts` | `305b0b456c59c511ee328e529ad574c0583075a0` | | `run-observations.ts` | `08c8ad184a821a7cd536926b3d7e3d1156a1e3da` | | `user-actions.ts` | `e9611348dc41e17899367dab2e3b36d3d1be9480` | | `stop-redirect-flow.ts` | `4580bd605a425bc72511e02d988492acbec8d60e` | The follow-up preservation comparison against historical `af9` has three explicit exceptions; no whole-file identity is claimed for them. The separately documented question readiness and report additions above also require final-source verification: 1. `product-journeys/agent-flows.ts` adds `question-answer-resume` to the `AgentJourney` type exclusion, alongside follow-up. Its remaining contents and runtime helper bodies are byte-for-byte unchanged. Qualified blob `a0afaa968c1d854476159fef21de7b6dd50a4a98`; final blob `0cc797c0ae594e7769338e63a80c4f67e29e4a54`. 2. The common run-count helper is centralized in `success-contract.ts` and re-exported by the catalog. It accepts follow-up **1–2**, clarification **1–3**, and each older paid journey's exact count. The follow-up catalog entry, maximum of two runs, six required checks, native dispatch, collection and retained-run contract remain byte-for-byte unchanged. The helper's text differs; its preserved count semantics are verified explicitly. 3. `product-journeys/active-followup-flow.ts` changes exactly one composer readiness assertion from Playwright's implicit five-second default to an explicit 30-second timeout, matching `submitTaskReply` for attached targets. Prompts, actions, native evidence, before/after persistence proof and grading otherwise compare byte-for-byte unchanged. Qualified blob `64392329198a1a4d9828e32834d4507e175216e6`; final blob `2746b5ba89e2af26b4097ba7897913186786ac5d`. This correction is why a fresh final-head QA run is required. - **310 focused tests passed at current final head `e72e20ad111bd94728d38606015d1fdac3c92929`**: 286 journey/evidence/catalog/report/shared-deadline tests in 14 files and 24 native authentication/real-child stream-order tests in two files. Both invocations used Node24 and one Vitest worker. The earlier `e55` integration also passed 365 focused tests, including 40 Cloud configuration and 40 TestOps metadata cases; those results remain prior-source evidence. - **157 explicit count assertions passed again at the current final head**, including both interactive ranges, older exact counts, invalid/fractional/non-number values and catalog/contract export identity. The maximum bounds and six closed follow-up check IDs also matched. - Maps and diff checks passed. The official QA runtime-import preflight passed on Node24 at the current final head without network access. It uses a dummy adapter and no credentials. It does not prove target connectivity, sign-in, model readiness or deployed application/runtime identity. Those require the normal protected QA run. No model/provider work or optional Runner release was dispatched. - Current local runtime dependencies use ignored isolated workspace-source views. No shared dependencies or compilers were changed. A previous local typecheck attempt hit stale reused Runner declarations. No full repository typecheck, test run or build was repeated locally; the successful exact-head full CI run above now supplies these gates. - CI on previous head `162e341cda5b627639b486b4019a1675bb018ea2` failed the Cloud/local maintenance fixture in [server shard 9](https://github.com/paperclipai/paperclip/actions/runs/41133716802/job/124331643100), plus dependent verify. The actual tested merge combined that head with master `198218bf554877fe3b7a83742eefb3b8f12392c6`. A Cloud child selected local-disk storage and hit the new shared-storage policy. This was a deterministic upstream fixture mismatch; no unchanged failed-job retry was used. - The actual #18385 repair configures a valid Cloud child, verifies local bytes independently, checks the expected provider mismatch and retains exact stored-config readback. Its own final head `b76d5dd40d7711f1b5f6b6316b1114c0aae02d57` passed 59 checks, including [server shard 9](https://github.com/paperclipai/paperclip/actions/runs/41133876636/job/124331450635). That prior result supports the repair; it does not replace this final head's fresh CI. Separate repair candidate #18392 and its source/evidence are retained as superseded history. - A local attempt to run the database fixture could not start PostgreSQL because the host exhausted macOS semaphores. A first setup attempt also failed package resolution before test execution. Skipped database cases and these setup failures are not passes. No kernel settings, shared processes or databases were changed. - The complete previous-head CI run [41138618869](https://github.com/paperclipai/paperclip/actions/runs/41138618869) finished with one underlying failure: [Runner Vitest shard 1](https://github.com/paperclipai/paperclip/actions/runs/41138618869/job/124347859445) expected native Antigravity login success but observed failure at `antigravity-auth.test.ts:101`. The lane had one failed test, 1,994 passing tests and eight skipped tests. Aggregate verify failed only because of this lane. All other 57 checks were accepted, including build, typecheck, all server/browser suites, Runner shard 2, Snyk and exact-head 5/5 reviews. No unchanged retry was requested. - The independently reviewed #18254 repair removes an invalid cross-pipe ordering assumption: a valid URL on stderr can arrive after the authenticate response on stdout during drain. Its real-child regression forces both orders. The current final source imports that actual squash; the 24 focused authentication cases pass here. Fresh full CI and exact-head review have now passed for this source, as recorded above. The earlier failed result is not relabeled. - The original live compatibility failure is retained: an older PRP v1 pack labeled builtin `bash` as operation `unknown`, causing the observer to miss the command and never send the follow-up. Its exact shape was reproduced red before repair. Generic unknown operations, semantic tools, wrappers, read-only work and foreign provenance remain rejected. The historical qualified pass used the repaired oracle; no failed result was relabeled. - Earlier CI on `6fad965d69` hit an unchanged heartbeat test's three-second adapter-call wait; its original failure and one narrow rerun remain recorded. The qualified SHA subsequently passed CI. An earlier report-catalog subprocess failure is retained. The final focused Node24 run now includes that suite and passes after the documented report feature mappings. ## Risks The fixture asks the native agent to run one bounded `sleep 90` command. The observer must see actual active native execution; a fast or noncompliant turn fails rather than passing from elapsed time. A completed result requires one or two bounded native work runs, source attribution, configured model/environment, and collection. Unknown provider prices stay unknown. This change does not alter production execution or grant new access. ## Model Used OpenAI Codex, GPT-6 family, with reasoning and code execution. The host does not expose an exact underlying model version or context-window size. ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with available version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs; existing follow-up execution fixes do not provide this deployment-targeted test - [x] I have described the issue in-PR following the enhancement template - [x] I have not referenced internal/instance-local Paperclip issues or links - [x] My branch name describes the change and contains no internal ticket or instance-derived details - [x] I have run the new focused tests locally and they pass - [x] I have added or updated tests where applicable - [x] I have updated relevant documentation - [x] I have considered and documented risks - [x] All Paperclip CI gates are green - [x] Greptile is 5/5 with no open P2s, recommendations, or follow-ups - [x] I will address all Greptile and reviewer comments before merge --------- Co-authored-by: Paperclip --- doc/evals.md | 6 + maps/feature-map/steering.md | 7 + tests/runner-e2e/PRODUCT-JOURNEYS.md | 32 ++++ tests/runner-e2e/catalog.test.ts | 2 +- .../runner-e2e/clarification-readback.spec.ts | 34 ++++ tests/runner-e2e/journey-features.ts | 2 + tests/runner-e2e/playwright-support.config.ts | 2 +- .../active-followup-evidence.ts | 82 +++++++++ .../product-journeys/active-followup-flow.ts | 110 +++++++++++ .../product-journeys/active-followup.test.ts | 174 ++++++++++++++++++ .../product-journeys/agent-flows.ts | 4 +- tests/runner-e2e/product-journeys/catalog.ts | 3 +- .../product-journeys/clarification-flow.ts | 18 +- .../clarification-readback.test.ts | 49 +++++ tests/runner-e2e/product-journeys/launch.ts | 5 +- .../product-journeys/success-contract.ts | 5 +- tests/runner-e2e/product-journeys/types.ts | 2 +- tests/runner-e2e/report-catalog.test.ts | 25 +++ 18 files changed, 550 insertions(+), 12 deletions(-) create mode 100644 tests/runner-e2e/clarification-readback.spec.ts create mode 100644 tests/runner-e2e/product-journeys/active-followup-evidence.ts create mode 100644 tests/runner-e2e/product-journeys/active-followup-flow.ts create mode 100644 tests/runner-e2e/product-journeys/active-followup.test.ts create mode 100644 tests/runner-e2e/product-journeys/clarification-readback.test.ts diff --git a/doc/evals.md b/doc/evals.md index 78c9c3ce467..bc0b7c9f729 100644 --- a/doc/evals.md +++ b/doc/evals.md @@ -76,6 +76,12 @@ one bound continuation, and an exact saved confirmation. Its four development cells use the recorded Codex and Claude models. Registration and calibration do not establish Slack, staging authentication, warm lifecycle, or live qualification. +The explicit `followup-while-running` [Product journey](../tests/runner-e2e/PRODUCT-JOURNEYS.md#active-follow-up) +checks a saved message during independently observed native work, one attributed +reply with the late requirement, and persistence after reload. Its attached and +managed targets share the same story and permit one steered run or one normal +successor. Registration and calibration are separate from a live staging pass. + [Reusable Product journeys](../tests/runner-e2e/PRODUCT-JOURNEYS.md) add explicit deployment targets and starting-state recipes. The shared task-properties journey uses no model; Cloud onboarding supplies reusable verified state. Cloud diff --git a/maps/feature-map/steering.md b/maps/feature-map/steering.md index 840fcd1ba74..611f0e56275 100644 --- a/maps/feature-map/steering.md +++ b/maps/feature-map/steering.md @@ -60,6 +60,13 @@ inspect the resulting task and run in Paperclip. The explicit `stop-redirect` [Product journey](../../tests/runner-e2e/PRODUCT-JOURNEYS.md) covers `steering/stop-and-pause` only for Stop and one new direction. It requires active work, native cancellation acknowledgement, resource cleanup, and completion without later original-turn activity. It does not qualify pause, interrupt, or all queued-message behavior. Source registration is not a live Cloud pass. +The explicit `followup-while-running` [Product journey](../../tests/runner-e2e/PRODUCT-JOURNEYS.md#active-follow-up) +covers `steering/queue`: one follow-up is saved while a bound native command is +active, its late requirement appears in one attributed final reply, the queue +empties, and the result survives reload. One steered turn or one normal successor +is allowed. Unit calibration and source registration are not live acceptance. + + Preconditions: follow the [baseline](./README.md#before-driving-a-journey). Use an agent that stays busy long enough to inspect the queue. Record the adapter, native/legacy mode, and advertised steering capability. Test supported and diff --git a/tests/runner-e2e/PRODUCT-JOURNEYS.md b/tests/runner-e2e/PRODUCT-JOURNEYS.md index 895005cd15a..98864325189 100644 --- a/tests/runner-e2e/PRODUCT-JOURNEYS.md +++ b/tests/runner-e2e/PRODUCT-JOURNEYS.md @@ -34,6 +34,7 @@ never start a server or depend on a tenant filesystem. | `protected-actions` | Native agent; four work runs; authenticated reachable review-service fixture | Approve one action and decline another on separate owned tasks; verify zero calls before either decision, one after approval, zero after decline and a truthful saved explanation | | `create-skill-studio` | Native agent with skill-write permission; one work run | Create a uniquely named company skill, open its activity card, edit in Studio, reload and verify task-card readback | | `stop-redirect` | Native agent; three work runs; qualified Docker oracle | Download a verified checkpoint, Stop an active revision, require native cancellation acknowledgement, submit a new direction once and verify completion without original-run activity | +| `followup-while-running` | Native agent; a maximum of two work runs | Save a follow-up while an independently observed native shell command is still active, then require one attributed final reply with the late requirement after reload; in-turn steering and one normal successor are accepted | | `cloud-onboarding` | Signed-in admitted Cloud account; fresh workspace; qualified bounded provider route | Create workspace in the portal, connect API key and native Codex agent in the wizard, complete a task, download and independently check its artifact | The question-resumption journey uses the same natural-language scenario, browser @@ -228,6 +229,37 @@ its disposable agent, makes owned tasks terminal, verifies no active runs, retai the records for decision-history integrity, and releases both QA leases. This is reported as retained data, rather than confirmed deletion. +## Active follow-up + +The `followup-while-running` story uses the same ordinary task composer and saved +reply observer on managed self-hosted and attached Cloud targets. The initial +request includes one bounded `sleep 90` command. It supplies no changed launch +day or final marker. The observer requires a valid running native turn and a +pending named builtin command or process execution before sending the follow-up. +Paperclip semantic tools, including completion/block/progress calls, cannot +satisfy this boundary even if a provider labels them builtin execute or wraps +their names with a native prefix. Unnamed or ambiguous builtin calls fail closed. +The observed older PRP v1 builtin `bash` shape is also supported when its +operation is `unknown` and `readOnly` is explicitly false; other unknown +operations cannot satisfy the boundary. If all owned work settles before an +active boundary is observed, the journey fails immediately rather than polling +until the campaign deadline. +The same command must remain active after exactly one user message is saved. +The task page and composer are ready before observing the boundary, so navigation is outside the +submission window. The longer bounded command leaves room for remote persistence. +Merely observing a running heartbeat or waiting for a fixed interval cannot satisfy the boundary. + +The late message changes the launch day and supplies a unique final marker. +The independent oracle requires the expected reply from one bound native run, +one or two successful work runs in total, an empty saved-message queue, Done, +and visible readback after reload. A provider can steer the original turn or +queue one normal successor; retries, duplicate replies and unrelated work fail. +Source/model/environment checks and full usage collection remain required. +Only the closed boundary receipt and attributed final reply are retained in the +private story directory. No local workspace path or provider-side fixture hook +is required. The registered source and unit calibration do not qualify a live +Cloud target; admission to scheduled testing requires its own fresh pass. + ## Evidence and lifecycle Results use the existing campaign/result schemas and dashboard. Each result diff --git a/tests/runner-e2e/catalog.test.ts b/tests/runner-e2e/catalog.test.ts index 1e98adc848b..cac4fcc784a 100644 --- a/tests/runner-e2e/catalog.test.ts +++ b/tests/runner-e2e/catalog.test.ts @@ -59,7 +59,7 @@ describe("runner E2E catalog", () => { }); it("exports deployment journeys separately from the model matrix and default selection", () => { - expect(productJourneys.map(journey => journey.id)).toEqual(["task-create", "project-task-context", "ask-question", "question-answer-resume", "task-properties", "task-search-filter", "plan-revise-accept", "build-revise", "protected-actions", "create-skill-studio", "stop-redirect", "cloud-onboarding"]); + expect(productJourneys.map(journey => journey.id)).toEqual(["task-create", "project-task-context", "ask-question", "question-answer-resume", "task-properties", "task-search-filter", "plan-revise-accept", "build-revise", "protected-actions", "create-skill-studio", "stop-redirect", "followup-while-running", "cloud-onboarding"]); expect(runnerSuites.some(suite => suite.id === "product-journeys")).toBe(false); expect(runnerMatrix.some(execution => execution.suite.id === "product-journeys")).toBe(false); expect(selectRunnerExecutions(parseRunnerSelectors(["--all"])).some(execution => execution.suite.id === "product-journeys")).toBe(false); diff --git a/tests/runner-e2e/clarification-readback.spec.ts b/tests/runner-e2e/clarification-readback.spec.ts new file mode 100644 index 00000000000..087457d5707 --- /dev/null +++ b/tests/runner-e2e/clarification-readback.spec.ts @@ -0,0 +1,34 @@ +import { expect, test } from "@playwright/test"; +import { waitForClarificationDocument } from "./product-journeys/clarification-flow.js"; + +const answer = "Use AMBERreadback."; +const selector = '[id="document-welcome-note"]:visible, [id="side-panel-content-document:welcome-note"]:visible'; + +for (const stage of ["mount", "text"] as const) { + test(`clarification readback waits beyond the default timeout for ${stage}`, async ({ page }) => { + const body = `Welcome to the afternoon meetup. ${answer}`; + await page.setContent(stage === "mount" ? "
" : '
Loading
'); + await page.evaluate(({ stage, body }) => { + setTimeout(() => { + if (stage === "mount") document.querySelector("main")!.innerHTML = `
${body}
`; + else document.querySelector("section")!.textContent = body; + }, 5_200); + }, { stage, body }); + await waitForClarificationDocument(page.locator(selector), answer, Date.now() + 8_000); + await expect(page.locator(selector)).toHaveText(body); + }); +} + +test("clarification readback rejects an exhausted campaign before waiting", async ({ page }) => { + await page.setContent(`
Welcome to the afternoon meetup. ${answer}
`); + await expect(waitForClarificationDocument(page.locator(selector), answer, Date.now() - 1)) + .rejects.toMatchObject({ code: "clarification_document_not_ready" }); +}); + +for (const invalid of ["duplicate", "wrong-reference", "wrong-choice"] as const) { + test(`clarification readback rejects ${invalid} evidence within the campaign deadline`, async ({ page }) => { + const body = invalid === "wrong-reference" ? "Welcome to the afternoon meetup. Use OLD." : invalid === "wrong-choice" ? answer : `Welcome to the afternoon meetup. ${answer}`; + await page.setContent(`
${body}
${invalid === "duplicate" ? `
${body}
` : ""}`); + await expect(waitForClarificationDocument(page.locator(selector), answer, Date.now() + 250)).rejects.toThrow(); + }); +} diff --git a/tests/runner-e2e/journey-features.ts b/tests/runner-e2e/journey-features.ts index 683f170c1e7..b0259b504d3 100644 --- a/tests/runner-e2e/journey-features.ts +++ b/tests/runner-e2e/journey-features.ts @@ -19,6 +19,8 @@ function associate(suites: readonly string[], cases: Record; +export interface ActiveFollowupBoundary { + runId: string; + executionId: string; + startedSeq: number; + sourceEventId: string; + sourceInstanceId: string; + sourceSeq: number; + startedRowSha256: string; +} + +/** A claimed run or elapsed delay is not proof that native work is active. */ +export function activeFollowupBoundary(run: Row, events: Row[]): ActiveFollowupBoundary | null { + if (!activeNativeTurnStarted(run, events) || !events.length + || events.some((event, index) => event.seq !== index + 1)) return null; + const bound = events.filter(event => { + const envelope = event.payload?.prpEvent; + return envelope && isValidNativePrpEnvelope(envelope, event.protocolSchemaVersion) + && event.runId === run.id && event.companyId === run.companyId && event.agentId === run.agentId + && envelope.runId === run.id && envelope.eventType === event.eventType && envelope.sourceKind === "runner" + && typeof envelope.sourceEventId === "string" && Boolean(envelope.sourceEventId) && envelope.sourceEventId === event.sourceEventId + && typeof envelope.sourceInstanceId === "string" && Boolean(envelope.sourceInstanceId) && envelope.sourceInstanceId === event.sourceInstanceId + && Number.isSafeInteger(envelope.sourceSeq) && envelope.sourceSeq > 0 && envelope.sourceSeq === event.sourceSeq; + }); + const latestTurn = bound.filter(event => /^turn\.(started|completed|failed|cancelled|interrupted)$/.test(event.eventType)).at(-1); + if (latestTurn?.eventType !== "turn.started") return null; + const starts = bound.filter(event => { + const payload = event.payload.prpEvent.payload; + return event.seq > latestTurn.seq && event.eventType === "tool.execution.started" && payload?.schema === "paperclip.tool.execution.v1" + && typeof payload.executionId === "string" && Boolean(payload.executionId) + && ["builtin", "process"].includes(payload.transport) + && (payload.operation === "execute" + // Older native packs classify builtin bash as unknown. Recognize only + // that named v1 shell shape; unknown semantic/custom tools still fail. + || event.protocolSchemaVersion === 1 && payload.operation === "unknown" + && payload.transport === "builtin" && payload.name === "bash" && payload.readOnly === false) + // Provider display titles vary, but Paperclip semantic calls never prove + // native command work, even when a provider maps them to builtin execute. + && payload.namespace == null + && (payload.transport === "process" || typeof payload.name === "string" && Boolean(payload.name.trim()) + && !/^(?:unknown|tool(?:[ _-]+call)?)$/i.test(payload.name.trim()) + && !/(?:^|[._:/-])paperclip(?:[._:/-]|$)/i.test(payload.name.trim())) + && payload.readOnly !== true && payload.status === "running"; + }); + const pending = starts.filter(start => { + const id = start.payload.prpEvent.payload.executionId; + return starts.filter(other => other.payload.prpEvent.payload.executionId === id).length === 1 + && !events.some(event => event.eventType === "tool.execution.completed" + && event.payload?.prpEvent?.payload?.executionId === id); + }); + const started = pending.at(-1); + if (!started) return null; + const envelope = started.payload.prpEvent; + return { runId: run.id, executionId: envelope.payload.executionId, startedSeq: started.seq, + sourceEventId: envelope.sourceEventId, sourceInstanceId: envelope.sourceInstanceId, sourceSeq: envelope.sourceSeq, + startedRowSha256: createHash("sha256").update(JSON.stringify(started)).digest("hex") }; +} + +export function sameActiveFollowupBoundary(before: ActiveFollowupBoundary, after: ActiveFollowupBoundary | null) { + return Boolean(after && Object.keys(before).every(key => before[key as keyof ActiveFollowupBoundary] === after[key as keyof ActiveFollowupBoundary])); +} + +/** Accept in-turn steering or one normal successor; reject duplicate delivery. */ +export function activeFollowupOutcome(input: { + issueId: string; agentId: string; before: ActiveFollowupBoundary; after: ActiveFollowupBoundary | null; + runs: Row[]; expectedReply: string; marker: string; replies: Array<{ runId: string; body: string }>; + storedFollowupCount: number; queuedCommentCount: number; finalIssueStatus: string; +}) { + const ids = input.runs.map(run => run.id); + const matching = input.replies.filter(reply => Boolean(input.marker) && reply.body.includes(input.marker)); + return sameActiveFollowupBoundary(input.before, input.after) + && input.storedFollowupCount === 1 && input.queuedCommentCount === 0 && input.finalIssueStatus === "done" + && input.runs.length >= 1 && input.runs.length <= 2 && new Set(ids).size === ids.length && ids.includes(input.before.runId) + && input.runs.every(run => run.agentId === input.agentId && run.status === "succeeded" && run.runtimeMode === "native" + && (run.nativeIssueId ?? run.issueId ?? run.contextSnapshot?.issueId ?? run.contextSnapshot?.taskId) === input.issueId) + && matching.length === 1 && matching[0]!.body === input.expectedReply && ids.includes(matching[0]!.runId) + && (input.runs.length === 1 || matching[0]!.runId !== input.before.runId); +} diff --git a/tests/runner-e2e/product-journeys/active-followup-flow.ts b/tests/runner-e2e/product-journeys/active-followup-flow.ts new file mode 100644 index 00000000000..2746b5ba89e --- /dev/null +++ b/tests/runner-e2e/product-journeys/active-followup-flow.ts @@ -0,0 +1,110 @@ +import { mkdir, writeFile } from "node:fs/promises"; +import path from "node:path"; +import { expect } from "@playwright/test"; +import { createTaskThroughUi, submitTaskReply } from "../user-actions.js"; +import { collectRunEvents } from "../run-observations.js"; +import { nativeFinalReplyEvents, readBoundTaskFinalReply } from "../native-task-reply.js"; +import { fixtureReplyStoredOnce, validateAgentTaskState, waitFor } from "./agent-flows.js"; +import { activeFollowupBoundary, activeFollowupOutcome, sameActiveFollowupBoundary } from "./active-followup-evidence.js"; +import { JourneyError, type JourneyContext, type JourneyEvidence, type TargetSession } from "./types.js"; + +type Row = Record; +const id = "followup-while-running"; + +export function activeFollowupStory(campaignId: string) { + const nonce = campaignId.replace(/[^a-z0-9]/gi, ""); + const reference = `BRIEF${nonce}`, marker = `UPDATED${nonce}`; + const expectedReply = `Launch Friday. Reference ${reference}. Update ${marker}.`; + return { + first: `Our launch is planned for Monday. Run the bounded command sleep 90 exactly once before answering, so I can send additional context while you work. After the command finishes, briefly acknowledge the launch day and reference ${reference}, then finish this task. No files, projects or child tasks are needed.`, + followup: `Change the launch day to Friday. When ready, reply with exactly this one line and finish: ${expectedReply}`, + reference, marker, expectedReply, + }; +} + +/** Once owned work has settled, an unobserved live boundary cannot arrive. */ +export async function observeActiveFollowupCommand(taskRuns: Row[], loadEvents: (runId: string) => Promise) { + if (taskRuns.length > 0 && taskRuns.every(run => run.status === "succeeded")) + throw new JourneyError("failed", "followup_native_command_not_active"); + const run = taskRuns.find(run => run.status === "running"); + return { run, boundary: run ? activeFollowupBoundary(run, await loadEvents(run.id)) : null }; +} + +/** The same browser/API/native story runs against managed and attached targets. */ +export async function runActiveFollowup(context: JourneyContext, session: TargetSession, campaignId: string, + evidence: JourneyEvidence, options: { deadlineAt: number; directory: string }) { + const agent = context.agent; + if (!agent || agent.runtimeMode !== "native" || !context.environmentId) throw new JourneyError("unsupported", "native_agent_and_environment_required"); + const story = activeFollowupStory(campaignId), title = `Journey QA active followup ${campaignId}`; + const check = (name: string, passed: boolean) => { + evidence.checks.push({ id: name, passed }); + if (!passed) throw new JourneyError("failed", name.replaceAll("-", "_")); + }; + await mkdir(options.directory, { recursive: true, mode: 0o700 }); + const retain = (name: string, value: unknown) => writeFile(path.join(options.directory, name), JSON.stringify(value, null, 2), { mode: 0o600 }); + const created = await createTaskThroughUi({ page: context.page, origin: context.api.origin, issuePrefix: context.company.issuePrefix, + agentName: agent.name, title, prompt: story.first, workMode: "standard", requireExplicitTitle: true, + beforeDispatch: () => session.own({ kind: "issue", name: title, journeyId: id }) }); + await session.own({ kind: "issue", name: title, id: created.issueId, journeyId: id }); + const route = `/api/issues/${created.issueId}`; + const read = async () => { + const [currentIssue, summaries, comments, queue] = await Promise.all([ + context.api.get(route), context.api.get(`${route}/runs`), context.api.get(`${route}/comments?order=asc`), + context.api.get(`${route}/queued-comments`), + ]); + if (!Array.isArray(summaries) || summaries.length > 2) throw new JourneyError("failed", "followup_run_limit_exceeded"); + const taskRuns = await Promise.all(summaries.map(summary => context.api.get(`/api/heartbeat-runs/${summary.runId}`))); + validateAgentTaskState(context, created.issueId, { currentIssue, taskRuns, comments, interactions: [], attachments: [] }, 2); + if (currentIssue.description !== story.first) throw new JourneyError("failed", "followup_initial_prompt_changed"); + return { currentIssue, taskRuns, comments, queue }; + }; + const events = (runId: string) => collectRunEvents((afterSeq, limit) => + context.api.get(`/api/heartbeat-runs/${runId}/events?afterSeq=${afterSeq}&limit=${limit}`)); + const issue = (await read()).currentIssue; + await context.page.goto(`${context.api.origin}/${context.company.issuePrefix}/issues/${issue.identifier ?? issue.id}`, { waitUntil: "domcontentloaded" }); + await expect(context.page.getByTestId("task-chat-composer-input").last()).toBeVisible({ timeout: 30_000 }); + const beforeComments = (await read()).comments; + const active = await waitFor(options.deadlineAt, async () => { + const state = await read(); + return observeActiveFollowupCommand(state.taskRuns, events); + }, value => Boolean(value.boundary), "followup_native_command_not_active"); + const before = active.boundary!; + await submitTaskReply(context.page, story.followup); + const persisted = await waitFor(Math.min(options.deadlineAt, Date.now() + 30_000), read, state => state.comments.some(comment => comment.authorUserId === context.user.id + && !beforeComments.some(old => old.id === comment.id)), "followup_not_persisted"); + const afterRun = persisted.taskRuns.find(run => run.id === before.runId)!; + const after = activeFollowupBoundary(afterRun, await events(before.runId)); + await retain("followup-boundary.json", { before, after }); + check("followup-native-command-active", sameActiveFollowupBoundary(before, after)); + check("followup-stored-once", fixtureReplyStoredOnce(beforeComments, persisted.comments, context.user.id, story.followup)); + + const final = await waitFor(options.deadlineAt, async () => { + const state = await read(); + const replies = [] as Array<{ runId: string; body: string }>; + for (const run of state.taskRuns.filter(run => run.status === "succeeded")) { + const stream = await events(run.id); + const decision = run.resultJson?.presentationDecision?.commentId; + const posted = state.comments.filter(comment => comment.id === decision && comment.authorAgentId === agent.id && comment.createdByRunId === run.id); + if (posted.length) replies.push(...posted.map(comment => ({ runId: run.id, body: comment.body }))); + else replies.push(...nativeFinalReplyEvents(stream, run).map(reply => ({ runId: run.id, body: reply.body }))); + } + return { state, replies }; + }, value => activeFollowupOutcome({ issueId: created.issueId, agentId: agent.id, before, after, + runs: value.state.taskRuns, expectedReply: story.expectedReply, marker: story.marker, replies: value.replies, + storedFollowupCount: value.state.comments.filter(comment => comment.authorUserId === context.user.id + && comment.body?.replaceAll("\\_", "_") === story.followup).length, + queuedCommentCount: Array.isArray(value.state.queue.entries) ? value.state.queue.entries.length : -1, + finalIssueStatus: value.state.currentIssue.status }), "followup_final_outcome_missing"); + check("followup-consumed-once", true); + const finalRunId = final.replies.find(reply => reply.body === story.expectedReply)!.runId; + const companies = await context.api.get>("/api/companies?scope=accessible"); + if (!Array.isArray(companies) || !companies.some(company => company.id === context.company.id) || companies.some(company => !company.issuePrefix)) throw new JourneyError("failed", "reply_company_context_missing"); + const reply = await readBoundTaskFinalReply({ page: context.page, runId: finalRunId, agentId: agent.id, + expectedText: story.expectedReply, deadlineAt: options.deadlineAt, knownCompanyPrefixes: companies.map(company => company.issuePrefix), + load: async () => { const latest = await read(); return { run: latest.taskRuns.find(run => run.id === finalRunId)!, comments: latest.comments, events: await events(finalRunId) }; } }); + await retain("followup-final.json", { before, after, runIds: final.state.taskRuns.map(run => run.id), reply }); + check("followup-final-answer-persisted", reply.observedAfterReload === true); + check("followup-one-or-two-native-runs", final.state.taskRuns.length >= 1 && final.state.taskRuns.length <= 2); + await expect(context.page.getByText(story.expectedReply, { exact: true }).last()).toBeVisible(); + check("configured-model-and-environment", true); +} diff --git a/tests/runner-e2e/product-journeys/active-followup.test.ts b/tests/runner-e2e/product-journeys/active-followup.test.ts new file mode 100644 index 00000000000..728160c610c --- /dev/null +++ b/tests/runner-e2e/product-journeys/active-followup.test.ts @@ -0,0 +1,174 @@ +import { describe, expect, it } from "vitest"; +import { canonicalProviderEventsFromCodex, canonicalProviderEventsFromAcpxRuntimeEvent, canonicalProviderEventsFromOpenCodePart } from "../../../packages/paperclip-runner/src/provider-events.js"; +import { activeFollowupBoundary, activeFollowupOutcome, sameActiveFollowupBoundary } from "./active-followup-evidence.js"; +import { activeFollowupStory, observeActiveFollowupCommand } from "./active-followup-flow.js"; +import { journeyRequiredChecks } from "./success-contract.js"; +import { parseTargetConfig, preparationPreview, agentJourneyRunCountValid } from "./catalog.js"; +import { validateRetainedRunnerResult } from "../result-validation.js"; + +const run = { id: "run", companyId: "company", agentId: "agent", issueId: "issue", runtimeMode: "native", status: "running" }; +function event(seq: number, type: string, payload = {}): Record { + return { seq, eventType: type, runId: run.id, companyId: run.companyId, agentId: run.agentId, + protocolSchemaVersion: 2, sourceEventId: `source-${seq}`, sourceInstanceId: "source", sourceSeq: seq, + payload: { prpEvent: { schema: "paperclip.prp.event.v2", schemaVersion: 2, eventType: type, runId: run.id, + sourceKind: "runner", sourceEventId: `source-${seq}`, sourceInstanceId: "source", sourceSeq: seq, payload } } }; +} +const command = { schema: "paperclip.tool.execution.v1", executionId: "wait", transport: "builtin", operation: "execute", name: "bash", status: "running" }; +const events = () => [event(1, "turn.started"), event(2, "tool.execution.started", command)]; +const boundary = () => activeFollowupBoundary(run, events())!; +// Synthetic identity and filler rows; the v1 operation/transport/name/readOnly +// shape and native/source sequence positions match observed deployment metadata. +function legacyEvents(change: Record = {}): Record[] { + const rows: Record[] = Array.from({ length: 81 }, (_, index) => ({ seq: index + 1, eventType: "adapter.message" })); + const bind = (seq: number, sourceSeq: number, type: string, payload: Record) => { + const row = event(seq, type, payload); + row.protocolSchemaVersion = 1; row.sourceSeq = sourceSeq; + Object.assign(row.payload.prpEvent, { schema: "paperclip.prp.event.v1", schemaVersion: 1, sourceSeq }); + return row; + }; + rows[70] = bind(71, 9, "turn.started", { status: "inProgress" }); + rows[80] = bind(81, 12, "tool.execution.started", { ...command, namespace: null, operation: "unknown", readOnly: false, ...change }); + return rows; +} +const outcome = () => ({ issueId: "issue", agentId: "agent", before: boundary(), after: boundary(), + runs: [{ ...run, status: "succeeded" }], expectedReply: "Friday UPDATED", marker: "UPDATED", replies: [{ runId: "run", body: "Friday UPDATED" }], + storedFollowupCount: 1, queuedCommentCount: 0, finalIssueStatus: "done" }); + +describe("deployment-independent active follow-up", () => { + it("withholds the changed requirement and result marker until the follow-up", () => { + const story = activeFollowupStory("campaign-123"); + expect(story.first).toContain("sleep 90"); expect(story.first).toContain(story.reference); + expect(story.first).not.toContain(story.marker); expect(story.first).not.toContain("Friday"); + expect(story.followup).toContain(story.expectedReply); + }); + it("requires a bound running native turn and an actual pending shell execution", () => { + expect(boundary()).toMatchObject({ runId: "run", executionId: "wait", startedSeq: 2 }); + for (const rows of [[], events().slice(0, 1), [...events(), event(3, "tool.execution.completed", { ...command, status: "completed" })], + [...events(), event(3, "turn.completed")], [...events(), event(3, "turn.started")], [events()[0]!, event(2, "tool.execution.started", { ...command, operation: "read" })], + [events()[0]!, event(2, "tool.execution.started", { ...command, readOnly: true })], + [events()[0]!, event(2, "tool.execution.started", { ...command, name: "paperclip_finish" })], + [events()[0]!, event(2, "tool.execution.started", { ...command, name: "paperclip_block" })], + [events()[0]!, event(2, "tool.execution.started", { ...command, namespace: "paperclip" })], + [events()[0]!, event(2, "tool.execution.started", { ...command, name: null })], + [events()[0]!, event(2, "tool.execution.started", { ...command, transport: "mcp" })], + [events()[0]!, event(3, "tool.execution.started", command)], + [...events(), event(3, "tool.execution.started", command)]]) expect(activeFollowupBoundary(run, rows)).toBeNull(); + for (const field of ["companyId", "agentId", "runId", "sourceEventId", "sourceInstanceId", "sourceSeq", "protocolSchemaVersion"]) { + const rows = events(); rows[1]![field] = "wrong"; expect(activeFollowupBoundary(run, rows)).toBeNull(); + } + const rows = events(); rows[0]!.payload.prpEvent.sourceKind = "control_plane"; + expect(activeFollowupBoundary(run, rows)).toBeNull(); + for (const changed of [{ status: "succeeded" }, { runtimeMode: "legacy" }]) expect(activeFollowupBoundary({ ...run, ...changed }, events())).toBeNull(); + }); + it("accepts canonical Codex command text and ACP display titles without provider-name coupling", () => { + const codex = canonicalProviderEventsFromCodex("item/started", { + item: { id: "wait", type: "commandExecution", command: "sleep 90", status: "inProgress" }, + })[0]!; + const acp = canonicalProviderEventsFromAcpxRuntimeEvent({ + type: "tool_call", tag: "tool_call", toolCallId: "wait", title: "Bounded settlement fixture", kind: "execute", status: "pending", + } as never, "fallback")[0]!; + expect(codex.payload).toMatchObject({ transport: "process", operation: "execute", name: "sleep 90", status: "running" }); + expect(acp.payload).toMatchObject({ transport: "builtin", operation: "execute", name: "Bounded settlement fixture", readOnly: false, status: "running" }); + for (const mapped of [codex, acp]) { + expect(activeFollowupBoundary(run, [event(1, "turn.started"), event(2, mapped.eventType, mapped.payload)])).toMatchObject({ runId: "run", startedSeq: 2 }); + } + }); + it("accepts the real OpenCode shell fixture and rejects semantic calls normalized as builtin execute", () => { + const mapped = (tool: string, callID = "") => canonicalProviderEventsFromOpenCodePart({ + id: "wait", type: "tool", tool, callID, state: { status: "pending", input: { command: "sleep 90" } }, + })[0]!; + const shell = mapped("bash"); + expect(shell.payload).toMatchObject({ transport: "builtin", operation: "execute", name: "bash", readOnly: false, status: "running" }); + expect(activeFollowupBoundary(run, [event(1, "turn.started"), event(2, shell.eventType, shell.payload)])).toMatchObject({ runId: "run", startedSeq: 2 }); + for (const semantic of [mapped("paperclip_finish"), mapped("paperclip_block"), mapped("paperclip_report_progress"), + mapped("paperclip_paperclip_finish"), mapped("unknown", "functions.paperclip_block:0"), + mapped("functions.paperclip_finish"), mapped("mcp__paperclip__paperclip_block"), mapped("paperclip.paperclip_report_progress")]) { + expect(semantic.payload).toMatchObject({ transport: "builtin", operation: "execute", readOnly: false, status: "running" }); + expect(activeFollowupBoundary(run, [event(1, "turn.started"), event(2, semantic.eventType, semantic.payload)])).toBeNull(); + } + for (const name of [null, "", " ", "unknown", "tool", "tool call", "tool_call", "tool-call"]) { + expect(activeFollowupBoundary(run, [event(1, "turn.started"), event(2, shell.eventType, { ...shell.payload, name })])).toBeNull(); + } + }); + it("recognizes the observed legacy v1 builtin bash shape without admitting unknown operations generally", () => { + const before = activeFollowupBoundary(run, legacyEvents()); + expect(before).toMatchObject({ runId: "run", executionId: "wait", startedSeq: 81, sourceSeq: 12 }); + for (const change of [{ name: "unknown" }, { name: "custom-command" }, { name: "paperclip_finish" }, { name: "paperclip_block" }, + { name: "paperclip_report_progress" }, { name: "functions.paperclip_finish" }, { readOnly: true }, { readOnly: undefined }, + { transport: "process" }, { namespace: "paperclip" }]) expect(activeFollowupBoundary(run, legacyEvents(change))).toBeNull(); + for (const field of ["companyId", "agentId", "runId", "sourceEventId", "sourceInstanceId", "sourceSeq", "protocolSchemaVersion"]) { + const rows = legacyEvents(); rows[80]![field] = "foreign"; + expect(activeFollowupBoundary(run, rows)).toBeNull(); + } + expect(activeFollowupBoundary(run, [events()[0]!, event(2, "tool.execution.started", { ...command, operation: "unknown", readOnly: false })])).toBeNull(); + const after = legacyEvents(); + after.push(event(82, "tool.execution.progressed", { ...after[80]!.payload.prpEvent.payload })); + expect(sameActiveFollowupBoundary(before!, activeFollowupBoundary(run, after))).toBe(true); + after.push(event(83, "tool.execution.completed", { ...after[80]!.payload.prpEvent.payload, status: "completed" })); + expect(activeFollowupBoundary(run, after)).toBeNull(); + }); + it("fails promptly after owned work settles without an observed active boundary", async () => { + let eventReads = 0; + const load = async () => { eventReads++; return legacyEvents(); }; + for (const taskRuns of [[{ ...run, status: "succeeded" }], [{ ...run, status: "succeeded" }, { ...run, id: "second", status: "succeeded" }]]) + await expect(observeActiveFollowupCommand(taskRuns, load)).rejects.toMatchObject({ outcome: "failed", code: "followup_native_command_not_active" }); + expect(eventReads).toBe(0); + for (const taskRuns of [[], [{ ...run, status: "queued" }], [{ ...run, status: "succeeded" }, { ...run, id: "second", status: "queued" }]]) + expect((await observeActiveFollowupCommand(taskRuns, load)).boundary).toBeNull(); + expect(eventReads).toBe(0); + expect((await observeActiveFollowupCommand([run], load)).boundary).toMatchObject({ runId: "run", startedSeq: 81 }); + expect(eventReads).toBe(1); + }); + it("proves the same execution remains active after the user message is stored", () => { + expect(sameActiveFollowupBoundary(boundary(), boundary())).toBe(true); + for (const changed of [null, { ...boundary(), executionId: "other" }, { ...boundary(), startedRowSha256: "changed" }]) + expect(sameActiveFollowupBoundary(boundary(), changed)).toBe(false); + }); + it("accepts one steered run or one normal successor with the unique late answer", () => { + expect(activeFollowupOutcome(outcome())).toBe(true); + const value = outcome(); value.runs.push({ ...value.runs[0]!, id: "successor" }); value.replies[0]!.runId = "successor"; + expect(activeFollowupOutcome(value)).toBe(true); + }); + it("rejects lost input, duplicate delivery/work, stale answers and unfinished queues", () => { + const changes = [ + (v: ReturnType) => { v.storedFollowupCount = 0; }, + (v: ReturnType) => { v.storedFollowupCount = 2; }, + (v: ReturnType) => { v.queuedCommentCount = 1; }, + (v: ReturnType) => { v.finalIssueStatus = "in_progress"; }, + (v: ReturnType) => { v.replies = []; }, + (v: ReturnType) => { v.replies.push(v.replies[0]!); }, + (v: ReturnType) => { v.replies[0]!.body = "Monday"; }, + (v: ReturnType) => { v.replies.push({ runId: "run", body: "UPDATED again" }); }, + (v: ReturnType) => { v.replies[0]!.runId = "other"; }, + (v: ReturnType) => { v.runs[0]!.status = "failed"; }, + (v: ReturnType) => { v.runs[0]!.agentId = "other"; }, + (v: ReturnType) => { v.runs[0]!.issueId = "other"; }, + (v: ReturnType) => { v.runs = []; }, + (v: ReturnType) => { v.runs.push(v.runs[0]!); }, + (v: ReturnType) => { v.runs.push({ ...v.runs[0]!, id: "successor" }); }, + ]; + for (const change of changes) { const value = outcome(); change(value); expect(activeFollowupOutcome(value)).toBe(false); } + }); + it("allows a maximum of two declared runs while preserving every other exact bound", () => { + for (const count of [1, 2]) expect(agentJourneyRunCountValid("followup-while-running", count)).toBe(true); + for (const count of [undefined, 0, 3, NaN, 1.5]) expect(agentJourneyRunCountValid("followup-while-running", count)).toBe(false); + expect(agentJourneyRunCountValid("build-revise", 1)).toBe(false); + const config = { version: 1, id: "qa", mode: "existing-cloud", adapterModule: "/private/provider.mjs", journeys: ["followup-while-running"], + executionEnvironment: "daytona", baseline: { features: {}, connections: [] }, limits: { durationMs: 600_000, modelBudgetCents: 100, maxRuns: 2 } }; + expect(preparationPreview(parseTargetConfig(config)).journeys[0]).toMatchObject({ id: "followup-while-running", paid: true }); + expect(() => parseTargetConfig({ ...config, limits: { ...config.limits, maxRuns: 1 } })).toThrow("agent_journeys_require_bounded_runs"); + }); + it("retains one-run and two-run passes only with the whole success contract", () => { + const result = { schema: "paperclip.runner-e2e.result/v2", executionId: "product-journeys.qa.configured-agent.daytona.followup-while-running", + suiteId: "product-journeys", suiteDefinitionHash: "test", profileId: "configured-agent", environmentId: "daytona", caseId: "followup-while-running", + provider: "opencode", model: "configured-model", runtimeMode: "native", status: "passed", cleanup: "passed", attempt: 1, + startedAt: "2026-01-01T00:00:00Z", finishedAt: "2026-01-01T00:00:01Z", durationMs: 1000, runIds: ["run"], + productJourney: { outcome: "passed", startingState: "recipe", privacy: "private", resourceDisposition: "retained", + phases: [{ phase: "journey", outcome: "passed" }], checks: journeyRequiredChecks["followup-while-running"]!.map(id => ({ id, passed: true })), + deployment: { kind: "cloud", targetId: "qa", origin: "https://example.test", commit: "a".repeat(40), cloudRelease: "b".repeat(40), runtimeImage: "pinned", configurationHash: "test" } } }; + for (const runIds of [["run"], ["run", "successor"]]) expect(() => validateRetainedRunnerResult({ ...result, runIds })).not.toThrow(); + for (const runIds of [[], ["a", "b", "c"]]) expect(() => validateRetainedRunnerResult({ ...result, runIds })).toThrow("result.agentJourneyRun"); + for (const check of result.productJourney.checks) expect(() => validateRetainedRunnerResult({ ...result, productJourney: { ...result.productJourney, + checks: result.productJourney.checks.filter(row => row.id !== check.id) } })).toThrow("result.productJourney"); + }); +}); diff --git a/tests/runner-e2e/product-journeys/agent-flows.ts b/tests/runner-e2e/product-journeys/agent-flows.ts index 37ccb8995bf..0cc797c0ae5 100644 --- a/tests/runner-e2e/product-journeys/agent-flows.ts +++ b/tests/runner-e2e/product-journeys/agent-flows.ts @@ -18,7 +18,7 @@ import { agentJourneyRuns } from "./catalog.js"; import { JourneyError, type JourneyContext, type JourneyEvidence, type TargetSession } from "./types.js"; type Row = Record; -type AgentJourney = Exclude; +type AgentJourney = Exclude; interface TaskState { currentIssue: Row & { id: string; status: string }; taskRuns: Array; comments: Row[]; interactions: Array; attachments: StoryDelivery[] } interface Options { deadlineAt: number; directory: string; capture(id: string, label: string, file: string): Promise } const active = (run: Row) => ["queued", "running"].includes(run.status); @@ -46,7 +46,7 @@ export function validateAgentTaskState(context: JourneyContext, issueId: string, } } -async function waitFor(deadlineAt: number, load: () => Promise, accept: (value: T) => boolean, code: string): Promise { +export async function waitFor(deadlineAt: number, load: () => Promise, accept: (value: T) => boolean, code: string): Promise { while (Date.now() < deadlineAt) { const value = await load(); if (accept(value)) return value; diff --git a/tests/runner-e2e/product-journeys/catalog.ts b/tests/runner-e2e/product-journeys/catalog.ts index 3d40c2cdc66..3013851a360 100644 --- a/tests/runner-e2e/product-journeys/catalog.ts +++ b/tests/runner-e2e/product-journeys/catalog.ts @@ -1,5 +1,5 @@ import { agentJourneyRuns } from "./success-contract.js"; -export { agentJourneyRuns } from "./success-contract.js"; +export { agentJourneyRuns, agentJourneyRunCountValid } from "./success-contract.js"; import type { JourneyId, TargetConfig } from "./types.js"; export const productJourneys = [ { id: "task-create", label: "Create a task for yourself and reopen it", requires: ["company", "board-user", "task-write"], capabilities: ["tasks/create"], paid: false }, @@ -13,6 +13,7 @@ export const productJourneys = [ { id: "protected-actions", label: "Approve and decline protected service actions", requires: ["company", "board-user", "task-write", "agent", "review-service"], capabilities: ["questions-and-approvals/tool-review"], paid: true }, { id: "create-skill-studio", label: "Create a company skill and persist a Skill Studio edit", requires: ["company", "board-user", "task-write", "agent", "skill-write"], capabilities: ["skills/authoring"], paid: true }, { id: "stop-redirect", label: "Stop active work and send a new direction once", requires: ["company", "board-user", "task-write", "agent"], capabilities: ["steering/stop-and-pause"], paid: true }, + { id: "followup-while-running", label: "Send a follow-up during active work and verify its saved result", requires: ["company", "board-user", "task-write", "agent"], capabilities: ["steering/queue"], paid: true }, { id: "cloud-onboarding", label: "Create a Cloud workspace and complete a first task", requires: ["cloud-account", "fresh-workspace", "provider-key"], capabilities: ["onboarding/first-task"], paid: true }, ] as const; diff --git a/tests/runner-e2e/product-journeys/clarification-flow.ts b/tests/runner-e2e/product-journeys/clarification-flow.ts index 932efbebf92..800a747284d 100644 --- a/tests/runner-e2e/product-journeys/clarification-flow.ts +++ b/tests/runner-e2e/product-journeys/clarification-flow.ts @@ -1,6 +1,6 @@ import { mkdir, writeFile } from "node:fs/promises"; import path from "node:path"; -import { expect } from "@playwright/test"; +import { expect, type Locator } from "@playwright/test"; import { continuationScenario } from "../continuation-cases.js"; import { continuationAnswerCommitted, continuationCheckpointReady } from "../continuation-readiness.js"; import type { ContinuationCheckpoint } from "../continuation-scoring.js"; @@ -13,6 +13,19 @@ import { JourneyError, type JourneyContext, type JourneyEvidence, type TargetSes type Row = Record; interface Options { deadlineAt: number; directory: string; capture(id: string, label: string, file: string): Promise } +export async function waitForClarificationDocument(visible: Locator, answer: string, deadlineAt: number) { + const remaining = () => { + const timeout = deadlineAt - Date.now(); + if (!Number.isFinite(timeout) || timeout <= 0) throw new JourneyError("failed", "clarification_document_not_ready"); + return timeout; + }; + await expect(visible).toHaveCount(1, { timeout: remaining() }); + await expect(visible).toBeVisible({ timeout: remaining() }); + await expect(visible).toContainText(answer, { timeout: remaining() }); + await expect(visible).toContainText(/afternoon/i, { timeout: remaining() }); + remaining(); +} + export async function readClarificationState(context: JourneyContext, issueId: string) { const route = `/api/issues/${issueId}`; const [currentIssue, summaries, interactions, documents, attachments, children] = await Promise.all([ @@ -96,8 +109,7 @@ export async function runClarificationJourney(context: JourneyContext, session: await context.page.goto(new URL(href, context.api.origin).href, { waitUntil: "domcontentloaded" }); await context.page.reload({ waitUntil: "domcontentloaded" }); const visible = context.page.locator(`[id=${JSON.stringify(`document-${document.key}`)}]:visible, [id=${JSON.stringify(`side-panel-content-document:${document.key}`)}]:visible`); - await expect(visible).toHaveCount(1); await expect(visible).toBeVisible(); - await expect(visible).toContainText(scenario.answer); await expect(visible).toContainText(/afternoon/i); + await waitForClarificationDocument(visible, scenario.answer, options.deadlineAt); const persisted = await context.api.get(`/api/issues/${issue.id}/documents/${encodeURIComponent(document.key)}`); readback = { issueId: persisted.issueId, key: persisted.key, revisionId: persisted.latestRevisionId, body: persisted.body, visibleBody: await visible.innerText() }; await options.capture("reloaded", "Saved answers after reload", "clarification-reloaded.png"); diff --git a/tests/runner-e2e/product-journeys/clarification-readback.test.ts b/tests/runner-e2e/product-journeys/clarification-readback.test.ts new file mode 100644 index 00000000000..5ec51654630 --- /dev/null +++ b/tests/runner-e2e/product-journeys/clarification-readback.test.ts @@ -0,0 +1,49 @@ +import { afterEach, expect, it, vi } from "vitest"; +import type { Locator } from "@playwright/test"; + +const clock = vi.hoisted(() => ({ now: 0, locator: {} })); +const answer = "Use AMBERreadback."; +const body = `Welcome to the afternoon meetup. ${answer}`; + +function stateAt(now: number) { + return { count: now >= 100 ? 1 : 0, visible: now >= 100, + text: now >= 1_300 ? body : now >= 600 ? answer : "Loading" }; +} + +vi.mock("@playwright/test", async importOriginal => { + const actual = await importOriginal(); + const wait = (accept: (state: ReturnType) => boolean, options: { timeout: number }) => { + if (!Number.isFinite(options.timeout) || options.timeout <= 0) throw Error("Invalid wait budget"); + if (accept(stateAt(clock.now))) return; + const deadline = clock.now + options.timeout; + const readyAt = [100, 600, 1_300].find(time => time > clock.now && time <= deadline && accept(stateAt(time))); + clock.now = readyAt ?? deadline; + if (readyAt === undefined) throw Error("Document not ready within wait budget"); + }; + return { ...actual, expect: (locator: Locator) => { + if (locator !== clock.locator) throw Error("Unexpected document locator"); + return { + async toHaveCount(count: number, options: { timeout: number }) { wait(state => state.count === count, options); }, + async toBeVisible(options: { timeout: number }) { wait(state => state.visible, options); }, + async toContainText(text: string | RegExp, options: { timeout: number }) { + wait(state => typeof text === "string" ? state.text.includes(text) : text.test(state.text), options); + }, + }; + } }; +}); + +import { waitForClarificationDocument } from "./clarification-flow.js"; + +afterEach(() => vi.restoreAllMocks()); + +it("shares one document deadline and retains rejection evidence through a later scheduler stall", async () => { + clock.now = 0; + vi.spyOn(Date, "now").mockImplementation(() => clock.now); + const observed = await waitForClarificationDocument(clock.locator as Locator, answer, 1_000).then( + () => ({ outcome: "resolved", at: clock.now, text: stateAt(clock.now).text }), + (error: Error) => ({ outcome: "rejected", error: error.message, at: clock.now, text: stateAt(clock.now).text }), + ); + clock.now += 10_000; + expect(stateAt(clock.now).text).toBe(body); + expect(observed).toEqual({ outcome: "rejected", error: "Document not ready within wait budget", at: 1_000, text: answer }); +}); diff --git a/tests/runner-e2e/product-journeys/launch.ts b/tests/runner-e2e/product-journeys/launch.ts index 4be973f1e81..72706d53ba9 100644 --- a/tests/runner-e2e/product-journeys/launch.ts +++ b/tests/runner-e2e/product-journeys/launch.ts @@ -14,6 +14,7 @@ import { taskSearchFilter } from "./task-search-filter.js"; import { askQuestion, taskCreate } from "./basic-flows.js"; import { projectTaskContext } from "./project-context-flow.js"; import { runAgentJourney } from "./agent-flows.js"; +import { runActiveFollowup } from "./active-followup-flow.js"; import { runClarificationJourney } from "./clarification-flow.js"; import { JourneyError, type JourneyContext, type JourneyEvidence, type JourneyOutcome, type TargetLifecycleInput, type TargetProvider, type TargetSession } from "./types.js"; import type { RunnerE2EResult } from "../types.js"; @@ -106,7 +107,7 @@ export async function launchProductJourneys(args: string[], repositoryRoot: stri try { sha = execFileSync("git", ["rev-parse", "HEAD"], { cwd: repositoryRoot, encoding: "utf8", stdio: ["ignore", "pipe", "ignore"] }).trim(); } catch { /* Packaged harnesses may not include Git metadata. */ } const source = resolveRunnerE2ESource({ sha, ref: null, workflowRunUrl: null }); const definitionHash = createHash("sha256").update(JSON.stringify(productJourneys)); - for (const name of ["catalog.ts", "task-properties.ts", "task-search-filter.ts", "task-search-evidence.ts", "basic-flows.ts", "basic-evidence.ts", "project-context-flow.ts", "project-context-readback.ts", "project-context-evidence.ts", "agent-flows.ts", "clarification-flow.ts", "clarification-evidence.ts", "../question-resume-ui.ts", "../question-resume-scoring.ts", "../runtime-question-readiness.ts", "../chat-flow.ts", "../continuation-readiness.ts", "../continuation-cases.ts", "success-contract.ts", "product-screenshot.ts", "cloud-onboarding.ts", "onboarding-connect.ts", "onboarding-first-task.ts", "workspace-deferral.ts", "onboarding-pre-acceptance.ts", "artifact-evidence.ts", "fixtures.ts", "launch.ts", "types.ts", "collection.ts", "local-target.ts"]) definitionHash.update(await readFile(path.join(import.meta.dirname, name))); + for (const name of ["catalog.ts", "task-properties.ts", "task-search-filter.ts", "task-search-evidence.ts", "basic-flows.ts", "basic-evidence.ts", "project-context-flow.ts", "project-context-readback.ts", "project-context-evidence.ts", "agent-flows.ts", "active-followup-flow.ts", "active-followup-evidence.ts", "clarification-flow.ts", "clarification-evidence.ts", "../question-resume-ui.ts", "../question-resume-scoring.ts", "../runtime-question-readiness.ts", "../chat-flow.ts", "../continuation-readiness.ts", "../continuation-cases.ts", "success-contract.ts", "product-screenshot.ts", "cloud-onboarding.ts", "onboarding-connect.ts", "onboarding-first-task.ts", "workspace-deferral.ts", "onboarding-pre-acceptance.ts", "artifact-evidence.ts", "fixtures.ts", "launch.ts", "types.ts", "collection.ts", "local-target.ts"]) definitionHash.update(await readFile(path.join(import.meta.dirname, name))); for (const file of ["catalog.ts", "api.ts", "run-observations.ts", "native-task-reply.ts", "native-event-envelope.ts", "user-actions.ts", "plan-revision-flow.ts", "skill-studio-flow.ts", "protected-action-flow.ts", "story-artifact.ts", "stop-redirect-flow.ts", "everyday-cases.ts", "everyday-delivery.ts", "everyday-decisions.ts", "connection-guidance-evidence.ts", "everyday-artifact.py"]) definitionHash.update(await readFile(path.join(import.meta.dirname, "..", file))); definitionHash.update(runnerSuites.flatMap(suite => suite.tasks).find(task => task.id === "ask-question")!.buildPrompt("definition")); const suiteDefinitionHash = definitionHash.digest("hex"); @@ -203,6 +204,8 @@ export async function launchProductJourneys(args: string[], repositoryRoot: stri result.model = ready.agent.model; result.runtimeMode = ready.agent.runtimeMode; if (id === "ask-question") await askQuestion(ready, session!, campaignId, evidence); + else if (id === "followup-while-running") await runActiveFollowup(ready, session!, campaignId, evidence, + { deadlineAt, directory: path.join(outputDir, "private-stories", id) }); else if (id === "question-answer-resume") await runClarificationJourney(ready, session!, campaignId, evidence, { deadlineAt, directory: path.join(outputDir, "private-stories", id), capture: async (_id, _label, file) => { diff --git a/tests/runner-e2e/product-journeys/success-contract.ts b/tests/runner-e2e/product-journeys/success-contract.ts index f49db32a938..67d37578e3e 100644 --- a/tests/runner-e2e/product-journeys/success-contract.ts +++ b/tests/runner-e2e/product-journeys/success-contract.ts @@ -1,11 +1,11 @@ /** Closed, target-independent success requirements used by dispatch and retained evidence. */ export const agentJourneyRuns = { "ask-question": 1, "plan-revise-accept": 3, "build-revise": 2, - "protected-actions": 4, "create-skill-studio": 1, "stop-redirect": 3, "question-answer-resume": 3 } as const; + "followup-while-running": 2, "protected-actions": 4, "create-skill-studio": 1, "stop-redirect": 3, "question-answer-resume": 3 } as const; export function agentJourneyRunCountValid(id: string, count: unknown): boolean { if (!Object.hasOwn(agentJourneyRuns, id) || !Number.isSafeInteger(count)) return false; const maximum = agentJourneyRuns[id as keyof typeof agentJourneyRuns]; - const minimum = id === "question-answer-resume" ? 1 : maximum; + const minimum = id === "question-answer-resume" || id === "followup-while-running" ? 1 : maximum; return (count as number) >= minimum && (count as number) <= maximum; } export const journeyRequiredChecks: Record = { @@ -24,5 +24,6 @@ export const journeyRequiredChecks: Record = { "create-skill-studio": ["created-skill-persisted", "skill-creation-attributed", "skill-studio-edit-persisted", "configured-model-and-environment"], "stop-redirect": ["checkpoint-artifact-verified", "active-work-stopped", "stop-native-terminal-retained", "new-direction-completed", "stopped-run-remained-terminal", "stopped-run-no-later-events", "stop-preserved-download", "redirect-consumed-once", "user-reply-stored-once", "configured-model-and-environment"], + "followup-while-running": ["followup-native-command-active", "followup-stored-once", "followup-consumed-once", "followup-final-answer-persisted", "followup-one-or-two-native-runs", "configured-model-and-environment"], "cloud-onboarding": ["fresh-workspace", "ui-connection-and-native-agent", "first-task-complete", "downloadable-artifact-verified", "native-trace-verified"], }; diff --git a/tests/runner-e2e/product-journeys/types.ts b/tests/runner-e2e/product-journeys/types.ts index 0f899efca5e..24f57c2cf7f 100644 --- a/tests/runner-e2e/product-journeys/types.ts +++ b/tests/runner-e2e/product-journeys/types.ts @@ -3,7 +3,7 @@ import type { ConnectionApi } from "../connection-target.js"; export type JourneyOutcome = "passed" | "failed" | "blocked" | "unsupported"; export type JourneyId = "task-properties" | "task-create" | "project-task-context" | "ask-question" | "cloud-onboarding" - | "question-answer-resume" | "task-search-filter" | "plan-revise-accept" | "build-revise" | "protected-actions" | "create-skill-studio" | "stop-redirect"; + | "question-answer-resume" | "task-search-filter" | "followup-while-running" | "plan-revise-accept" | "build-revise" | "protected-actions" | "create-skill-studio" | "stop-redirect"; export type TargetMode = "managed-self-hosted" | "existing-cloud" | "new-cloud"; export interface DeploymentIdentity { kind: "self-hosted" | "cloud"; diff --git a/tests/runner-e2e/report-catalog.test.ts b/tests/runner-e2e/report-catalog.test.ts index 28929783eb1..09d6bd6d6d7 100644 --- a/tests/runner-e2e/report-catalog.test.ts +++ b/tests/runner-e2e/report-catalog.test.ts @@ -11,6 +11,7 @@ import { buildRunnerCampaign } from "./history.js"; import { renderRunnerE2EDashboard } from "./dashboard.js"; import { regenerateRunnerDashboard } from "./dashboard-regenerate.js"; import { journeyCapabilities, journeyFeatureIds } from "./journey-features.js"; +import { journeyRequiredChecks } from "./product-journeys/success-contract.js"; import type { RunnerE2EResult } from "./types.js"; const temporary: string[] = []; @@ -91,6 +92,30 @@ describe("trusted report catalog discovery", () => { expect(journeyCapabilities({ ...product, task: { ...known.task, id: "task-search-filter" } }).map(capability => capability.id)).toEqual(["tasks/organize"]); }); + it.each([ + { caseId: "question-answer-resume", feature: "questions-and-approvals", capabilities: ["questions-and-approvals/questions", "questions-and-approvals/continuation"] }, + { caseId: "followup-while-running", feature: "steering", capabilities: ["steering/queue"] }, + ])("reports deployment journey $caseId in its feature coverage", ({ caseId, feature, capabilities }) => { + const known = runnerMatrix.find(execution => execution.id === "core-compatibility.legacy-codex.local.ask-question")!; + const execution = { ...known, id: `product-journeys.${known.profile.id}.${known.environment.id}.${caseId}`, + suite: { ...known.suite, id: "product-journeys" }, task: { ...known.task, id: caseId } }; + expect(journeyCapabilities(execution).map(capability => capability.id)).toEqual(capabilities); + expect(journeyFeatureIds(execution)).toEqual([feature]); + expect(journeyCapabilities({ ...execution, suite: { ...execution.suite, id: "unrecognized-suite" } })).toEqual([]); + const page = renderRunnerE2EDashboard({ title: "Deployment coverage", generatedAt: "2026-10-20T12:00:00Z", + expected: [execution.id], catalog: [execution], entries: [{ valid: true, errors: [], + result: { ...result(), executionId: execution.id, suiteId: execution.suite.id, profileId: execution.profile.id, + environmentId: execution.environment.id, caseId, provider: "opencode", runIds: ["run-1"], + productJourney: { targetId: "qa", outcome: "passed", startingState: "recipe", privacy: "private", resourceDisposition: "retained", + phases: [{ phase: "journey", outcome: "passed" }], checks: journeyRequiredChecks[caseId]!.map(id => ({ id, passed: true })), + deployment: { kind: "cloud", targetId: "qa", origin: "https://qa.example.test", commit: "a".repeat(40), + cloudRelease: "b".repeat(40), runtimeImage: "immutable-image", configurationHash: "config" } } } }] }); + expect(page).toContain(`data-execution-id="${execution.id}"`); + expect(page).toContain(`data-report-features="${feature}"`); + expect(page).toContain(`title="${capabilities.join(", ")}"`); + expect(page).toContain("1/1 passed"); + }); + it("counts the project/task deployment journey toward projects and task creation coverage", () => { const known = runnerMatrix.find(execution => execution.id === "core-compatibility.legacy-codex.local.ask-question")!; const project = { ...known, suite: { ...known.suite, id: "product-journeys" }, task: { ...known.task, id: "project-task-context" } }; From 594a4ee657b6194ec31c26bd0e088cd94bfd77d9 Mon Sep 17 00:00:00 2001 From: Dotta Date: Fri, 30 Oct 2026 18:41:23 -0500 Subject: [PATCH 50/70] fix(external-objects): fence selected plugin results Revalidate the exact selected plugin before result persistence even when an allowed fallback still supports the object. Preserve due debt and token ownership so a fresh allowed resolver can proceed. Co-Authored-By: Paperclip --- .../2026-10-30-external-objects-sleep.md | 10 +++- .../external-object-refresh-work.test.ts | 54 ++++++++++++++++++ server/src/services/external-objects.ts | 55 +++++++++++-------- 3 files changed, 96 insertions(+), 23 deletions(-) diff --git a/doc/plans/2026-10-30-external-objects-sleep.md b/doc/plans/2026-10-30-external-objects-sleep.md index 41bf59440d1..e5401596a0e 100644 --- a/doc/plans/2026-10-30-external-objects-sleep.md +++ b/doc/plans/2026-10-30-external-objects-sleep.md @@ -18,6 +18,14 @@ ready plugin with `external.objects.read`, a matching provider/object declaratio and no company setting that disables it. Built-in resolvers remain available independently of optional plugin support. +A plugin result carries its selected plugin ID inside the host owner. Persistence +rechecks that exact plugin's readiness, read capability, object declaration and +same-company enabled setting in the result update. Another available resolver +cannot authorize an obsolete plugin response. A rejected response releases only +its own claim; the due row remains for a fresh allowed resolver. Plugin RPC +payloads and public result shapes do not contain this host selection context. + + Terminal objects are passive. The existing exception remains: GitHub `not_found` snapshots without `data.probeVerified=true` need a repository-visibility probe before their terminal verdict is definitive. A null deadline retains its @@ -94,7 +102,7 @@ reactivation in the tested process; it does not establish Cloud alarm coverage, host/database suspension or deployed savings. -## Host cancellation and explicit legacy entry points +## Host cancellation and registered ownership The resolver contract accepts an optional host `AbortSignal`. The worker passes its coordinator signal through refresh to the resolver. This reuses the narrow diff --git a/server/src/__tests__/external-object-refresh-work.test.ts b/server/src/__tests__/external-object-refresh-work.test.ts index 4c1817b9722..8aab42b35b1 100644 --- a/server/src/__tests__/external-object-refresh-work.test.ts +++ b/server/src/__tests__/external-object-refresh-work.test.ts @@ -257,6 +257,60 @@ const active: ExternalObjectResolveResult = { ok: true, snapshot: { expect(calls).toHaveBeenCalledOnce(); }); + it.each(["company setting", "status", "uninstall", "manifest", "other company"] as const)( + "revalidates the selected plugin after %s changes even while the built-in fallback remains available", async change => { + clock(); const registry = pluginRegistryService(db); + const manifest: PaperclipPluginManifestV1 = { + id: `paperclip.selected-${randomUUID()}`, apiVersion: 1, version: "1.0.0", displayName: "Selected provider", + description: "Selected resolver fixture", author: "Paperclip", categories: ["connector"], + capabilities: ["external.objects.detect", "external.objects.read"], entrypoints: { worker: "worker.js" }, + objectReferences: [{ providerKey: "github", displayName: "GitHub", objectTypes: ["pull_request"], urlPatterns: ["https://github.com/:owner/:repo/pull/:id"] }], + }; + const plugin = await registry.install({ packageName: "selected-provider-fixture" }, manifest); + await registry.updateStatus(plugin!.id, { status: "ready" }); + let finishPlugin!: (result: ExternalObjectResolveResult) => void; + const pluginResult = new Promise(resolve => { finishPlugin = resolve; }); + let selected = false; + const calls = vi.fn(async (_id: string, method: string) => { + if (method === "detectExternalObjects") return { detections: [] }; + selected = true; return pluginResult; + }); + let finishFallback!: () => void; + const fallbackResult = new Promise(resolve => { finishFallback = resolve; }); + const fallback = vi.fn(async () => { + await fallbackResult; + return new Response(JSON.stringify({ state: "closed", merged: true, title: "Allowed fallback" }), { status: 200 }); + }); + const svc = externalObjectService(db, { pluginWorkerManager: { call: calls } as unknown as PluginWorkerManager, + github: { fetch: fallback, credentialProvider: () => ({ token: "simulated-company-token", source: "company_secret" }) } }); + try { + await start(svc); const f = await fixture(); + await db.update(issues).set({ description: "https://github.com/acme/app/pull/42" }).where(eq(issues.id, f.issueId)); + await svc.syncIssue(f.issueId); await settle(() => selected); + if (change === "company setting") await registry.upsertCompanySettings(plugin!.id, f.companyId, { enabled: false, settingsJson: {} }); + else if (change === "status") await registry.updateStatus(plugin!.id, { status: "error" }); + else if (change === "uninstall") await registry.uninstall(plugin!.id, true); + else if (change === "manifest") await registry.update(plugin!.id, { manifest: { ...manifest, capabilities: [], objectReferences: [] } }); + else { + const other = await fixture(); + await registry.upsertCompanySettings(plugin!.id, other.companyId, { enabled: false, settingsJson: {} }); + } + finishPlugin({ ok: true, snapshot: { statusKey: "plugin_result", statusCategory: "succeeded", statusTone: "success", isTerminal: true } }); + if (change === "other company") { + await settle(async () => (await row(f.companyId))?.statusKey === "plugin_result"); await quiet(); + expect(fallback).not.toHaveBeenCalled(); + } else { + await settle(async () => fallback.mock.calls.length > 0 || Boolean((await row(f.companyId))?.lastResolvedAt)); + const beforeFallback = await row(f.companyId); + expect(beforeFallback.isTerminal).toBe(false); expect(beforeFallback.statusKey).toBeNull(); expect(beforeFallback.lastResolvedAt).toBeNull(); + expect(fallback).toHaveBeenCalledOnce(); + finishFallback(); + await settle(async () => (await row(f.companyId))?.statusKey === "merged"); await quiet(); + expect((await row(f.companyId)).displayTitle).toBe("acme/app#42: Allowed fallback"); + } + } finally { finishPlugin(terminal); finishFallback(); } + }); + it("reactivates refresh after ordinary budget pause and budget raise", async () => { clock(); const svc = service(); provider.mockResolvedValue(active); await start(svc); const f = await fixture(); await svc.syncIssue(f.issueId); diff --git a/server/src/services/external-objects.ts b/server/src/services/external-objects.ts index a4d26014d26..81e10a208f2 100644 --- a/server/src/services/external-objects.ts +++ b/server/src/services/external-objects.ts @@ -346,7 +346,7 @@ async function resolveViaPluginProvider( db: Db, pluginWorkerManager: PluginWorkerManager | undefined, object: ExternalObjectRecord, -): Promise { +): Promise<{ pluginId: string; result: PluginExternalObjectResolveResult } | null> { if (!pluginWorkerManager) return null; const providers = await readyObjectReferencePlugins(db, object.companyId); for (const provider of providers) { @@ -354,23 +354,27 @@ async function resolveViaPluginProvider( if (!manifest.capabilities.includes("external.objects.read")) continue; if (!manifestProvidesObject(manifest, object)) continue; try { - return await pluginWorkerManager.call(provider.id, "resolveExternalObject", { + const result = await pluginWorkerManager.call(provider.id, "resolveExternalObject", { companyId: object.companyId, providerKey: object.providerKey, objectType: object.objectType, externalId: object.externalId, object: objectSnapshot(object), }); + return { pluginId: provider.id, result }; } catch (err) { logger.warn( { err, pluginId: provider.id, pluginKey: provider.pluginKey, objectId: object.id }, "plugin external object resolver failed", ); return { - ok: false, - liveness: "unreachable", - errorCode: "plugin_resolver_failed", - errorMessage: err instanceof Error ? err.message : String(err), + pluginId: provider.id, + result: { + ok: false, + liveness: "unreachable", + errorCode: "plugin_resolver_failed", + errorMessage: err instanceof Error ? err.message : String(err), + }, }; } } @@ -402,12 +406,9 @@ export function externalObjectService( ]; const resolverRegistry = createExternalObjectResolverRegistry(resolvers); - function automaticRefreshWhere(database: Db = db) { - const supported = resolvers.map(resolver => and( - eq(externalObjects.providerKey, resolver.providerKey), - resolver.objectType ? eq(externalObjects.objectType, resolver.objectType) : undefined, - )!); - if (opts.pluginWorkerManager) supported.push(exists(database.select({ id: plugins.id }).from(plugins).where(and( + function pluginRefreshWhere(database: Db, selectedPluginId?: string) { + return exists(database.select({ id: plugins.id }).from(plugins).where(and( + selectedPluginId ? eq(plugins.id, selectedPluginId) : undefined, eq(plugins.status, "ready"), sql`${plugins.manifestJson}->'capabilities' ? 'external.objects.read'`, sql`exists (select 1 from jsonb_array_elements(coalesce(${plugins.manifestJson}->'objectReferences', '[]'::jsonb)) as reference @@ -418,7 +419,15 @@ export function externalObjectService( eq(pluginCompanySettings.companyId, externalObjects.companyId), eq(pluginCompanySettings.enabled, false), )))), - )))); + ))); + } + + function automaticRefreshWhere(database: Db = db) { + const supported = resolvers.map(resolver => and( + eq(externalObjects.providerKey, resolver.providerKey), + resolver.objectType ? eq(externalObjects.objectType, resolver.objectType) : undefined, + )!); + if (opts.pluginWorkerManager) supported.push(pluginRefreshWhere(database)); return and( refreshableObjectWhere(), isNotNull(externalObjects.nextRefreshAt), supported.length ? or(...supported) : sql`false`, @@ -863,12 +872,14 @@ export function externalObjectService( } async function persistRefresh(object: ExternalObjectRecord, token: string, - patch: Partial, scheduled = false) { + patch: Partial, scheduled = false, selectedPluginId?: string) { return db.transaction(async tx => { await notifyExternalObjectRefresh(tx, object.companyId, object.id); const rows = await tx.update(externalObjects).set(patch) - .where(and(refreshOwnerWhere(object, token), scheduled ? automaticRefreshWhere(tx as unknown as Db) : undefined)).returning(); - if (scheduled && rows.length === 0) await tx.update(externalObjects) + .where(and(refreshOwnerWhere(object, token), + scheduled ? automaticRefreshWhere(tx as unknown as Db) : undefined, + selectedPluginId ? pluginRefreshWhere(tx as unknown as Db, selectedPluginId) : undefined)).returning(); + if ((scheduled || selectedPluginId) && rows.length === 0) await tx.update(externalObjects) .set({ refreshStartedAt: null, refreshToken: null }).where(refreshOwnerWhere(object, token)); return rows; }); @@ -881,9 +892,9 @@ export function externalObjectService( refreshToken: string, ) { input.signal?.throwIfAborted(); - const pluginResult = await observeRefreshCancellation(resolveViaPluginProvider(db, opts.pluginWorkerManager, object), input.signal); - const resolver = pluginResult ? null : resolverRegistry.find(object); - if (!pluginResult && !resolver) { + const pluginResolution = await observeRefreshCancellation(resolveViaPluginProvider(db, opts.pluginWorkerManager, object), input.signal); + const resolver = pluginResolution ? null : resolverRegistry.find(object); + if (!pluginResolution && !resolver) { const [updated] = await persistRefresh(object, refreshToken, { liveness: object.isTerminal ? object.liveness : visibleLiveness(object, now) === "fresh" ? "stale" : object.liveness, nextRefreshAt: addSeconds(now, DEFAULT_RETRY_AFTER_SECONDS), @@ -895,7 +906,7 @@ export function externalObjectService( return { object: toObjectPayload(updated ?? object, now), refreshed: false, reason: "no_resolver" as const }; } - const result = pluginResult ?? await observeRefreshCancellation( + const result = pluginResolution?.result ?? await observeRefreshCancellation( resolver!.resolve({ companyId: object.companyId, object, signal: input.signal }), input.signal); input.signal?.throwIfAborted(); if (!result.ok) { @@ -913,7 +924,7 @@ export function externalObjectService( refreshStartedAt: null, refreshToken: null, updatedAt: now, - }, input.scheduled); + }, input.scheduled, pluginResolution?.pluginId); if (!updated) return refreshSupersededResult(object, now); publishLiveEvent({ companyId: object.companyId, @@ -950,7 +961,7 @@ export function externalObjectService( const [updated] = await persistRefresh(object, refreshToken, { ...patch, lastChangedAt: objectChanged(object, { ...object, ...patch }) ? now : object.lastChangedAt, - }, input.scheduled); + }, input.scheduled, pluginResolution?.pluginId); if (!updated) return refreshSupersededResult(object, now); const next = updated ?? object; if (objectChanged(object, next) && input.actor) { From 9e904ee5194f6f126701f1798faf792695a1d327 Mon Sep 17 00:00:00 2001 From: Nicky Leach Date: Fri, 30 Oct 2026 16:49:20 -0700 Subject: [PATCH 51/70] perf(server): overlap independent heartbeat preparation reads (#18372) --- doc/observability.md | 8 + .../heartbeat-execution-configuration.test.ts | 57 ++++++- .../__tests__/heartbeat-task-context.test.ts | 100 ++++++++++- server/src/services/heartbeat.ts | 47 +----- .../heartbeat/execution-configuration.ts | 47 ++++-- .../heartbeat/run-preparation.test.ts | 43 +++++ .../src/services/heartbeat/run-preparation.ts | 155 +++++++----------- server/src/services/heartbeat/task-context.ts | 92 +++++++---- .../runner-api.integration.test.ts | 2 +- 9 files changed, 354 insertions(+), 197 deletions(-) diff --git a/doc/observability.md b/doc/observability.md index e2bb4065caf..996d3a807f5 100644 --- a/doc/observability.md +++ b/doc/observability.md @@ -328,6 +328,14 @@ once known, `paperclip.native_reconciliation.candidates` (a count). A failed pass marks the pass span and the phase that was running with an error status. Faster passes record nothing. +Heartbeat configuration preparation overlaps independent reads. Skill preparation +runs alongside connector assignments; standing connection instructions wait for +assignment catalog repairs, but can overlap the skill branch. These spans can +finish out of order. Do not add their durations as serial elapsed time. The +historical `skills.prepare` span ends when the skill branch settles, rather than +when its concurrent branches finish. Each preparation attempt settles its started +branches before it returns an error or releases acquired credentials. + ### Detailed execution phases The `paperclip.execution` tracer adds these content-free spans beneath the current diff --git a/server/src/__tests__/heartbeat-execution-configuration.test.ts b/server/src/__tests__/heartbeat-execution-configuration.test.ts index 509a9cfa7e6..2cd52f84849 100644 --- a/server/src/__tests__/heartbeat-execution-configuration.test.ts +++ b/server/src/__tests__/heartbeat-execution-configuration.test.ts @@ -19,11 +19,20 @@ import { secretService } from "../services/secrets.js"; import { instanceSettingsService } from "../services/instance-settings.js"; import { aiConnectionService } from "../services/ai-connections.js"; import * as aiRuntime from "../services/ai-connection-runtime.js"; +import * as connectorRuntime from "../services/connector-runtime.js"; +import * as assignedMcp from "../services/native-runtime/assigned-mcp-tools.js"; import * as sentry from "../sentry.js"; import { waitForPendingRunFailureReports } from "../services/run-failure-report.js"; import { recoveryPostgresSupport, startRecoveryPostgres } from "./helpers/recovery-postgres.js"; +function deferred() { + let resolve!: (value: T) => void; + const promise = new Promise(complete => { resolve = complete; }); + return { promise, resolve }; +} + const skillFailures = vi.hoisted(() => new Map()); +const skillWaits = vi.hoisted(() => new Map Promise>()); describe("Runner home mode during recovery preparation", () => { it.each([ @@ -52,6 +61,7 @@ vi.mock("../services/company-skills.js", async importOriginal => { return { ...service, withRuntimeSkillEntries: async (companyId: string, options: Parameters[1], consume: (entries: Awaited>) => T | Promise) => { + await skillWaits.get(companyId)?.(); const failure = skillFailures.get(companyId); if (failure) throw new Error(failure); return service.withRuntimeSkillEntries(companyId, options, consume); @@ -91,7 +101,7 @@ describePostgres("heartbeat execution configuration through adapter dispatch", ( testEnvironment: async () => ({ adapterType, status: "pass", checks: [], testedAt: new Date().toISOString() }), }); }, 60_000); - afterEach(() => { vi.restoreAllMocks(); vi.unstubAllEnvs(); skillFailures.clear(); }); + afterEach(() => { vi.restoreAllMocks(); vi.unstubAllEnvs(); skillFailures.clear(); skillWaits.clear(); }); afterAll(async () => { await heartbeat?.drainActiveRunExecutions(); unregisterServerAdapter(adapterType); @@ -288,6 +298,45 @@ describePostgres("heartbeat execution configuration through adapter dispatch", ( const [unchangedAgent] = await db.select().from(agents).where(eq(agents.id, f.agentId)); expect(unchangedAgent.adapterConfig).toMatchObject({ paperclipSkillSync: { desiredSkills: [other.key] } }); }, 60_000); + it("overlaps skills with assignments and dependent connection instructions before composition", async () => { + const f = await seed(); + const skillGate = deferred(); + skillWaits.set(f.companyId, () => skillGate.promise); + const assignments = vi.spyOn(connectorRuntime, "resolveConnectorAssignments"); + const instructions = vi.spyOn(assignedMcp, "resolveAssignedConnectionInstructionsForRun"); + const configured = await configuration(f, { provider: "codex", model: "gpt-6.1-sol", homeMode: "isolated" }); + const opening = configured.prepare(); + try { + await vi.waitFor(() => { + expect(assignments).toHaveBeenCalledOnce(); + expect(instructions).toHaveBeenCalledOnce(); + }); + } finally { + skillGate.resolve(); + } + expect(await opening).toMatchObject({ prepared: true, runtimeConfig: { homeMode: "isolated", model: "gpt-6.1-sol" } }); + }); + + it("settles concurrent configuration reads before exposing the first preparation failure", async () => { + const f = await seed(); + const assignmentGate = deferred>>(); + const assignments = vi.spyOn(connectorRuntime, "resolveConnectorAssignments").mockReturnValue(assignmentGate.promise); + const instructions = vi.spyOn(assignedMcp, "resolveAssignedConnectionInstructionsForRun").mockRejectedValue(new Error("later instruction failure")); + skillFailures.set(f.companyId, "first skill failure"); + const configured = await configuration(f, { provider: "codex", homeMode: "isolated" }); + let settled = false; + const opening = configured.prepare(); + void opening.then(() => { settled = true; }, () => { settled = true; }); + try { + await vi.waitFor(() => expect(assignments).toHaveBeenCalledOnce()); + await Promise.resolve(); + expect(settled).toBe(false); + expect(instructions).not.toHaveBeenCalled(); + } finally { + assignmentGate.resolve([]); + } + await expect(opening).rejects.toThrow("first skill failure"); + }); it("does not install new environment declarations when recovering a saved skill selection", async () => { const f = await seed(), pluginId = randomUUID(), pluginKey = `fixture.${randomUUID()}`; @@ -445,11 +494,11 @@ describePostgres("heartbeat execution configuration through adapter dispatch", ( const runIdHash = createHash("sha256").update(finished.id).digest("hex").slice(0, 12); const steps = exporter.getFinishedSpans().filter(span => span.name.startsWith("heartbeat.prepare.configuration_") && span.attributes["paperclip.task.run.run_id"] === runIdHash); - expect(steps.map(span => span.name)).toEqual([ + expect(steps.map(span => span.name).sort()).toEqual([ "configuration_mentions", "configuration_github_identity", "configuration_secrets", "configuration_ai_runtime", "configuration_skills", "configuration_connector_assignments", "configuration_connector_skills", "configuration_connector_delivery", "configuration_connection_instructions", "configuration_agent_identity", - ].map(step => `heartbeat.prepare.${step}`)); + ].map(step => `heartbeat.prepare.${step}`).sort()); for (const span of steps) { expect(span.status).toEqual({ code: 0 }); expect(span.attributes).toEqual({ "paperclip.task.run.run_id": runIdHash }); @@ -494,7 +543,7 @@ describePostgres("heartbeat execution configuration through adapter dispatch", ( && span.attributes["paperclip.task.run.run_id"] === runIdHash); expect(steps.find(span => span.name === "heartbeat.prepare.configuration_ai_runtime")?.status.code).toBe(0); expect(steps.find(span => span.name === "heartbeat.prepare.configuration_skills")?.status).toEqual({ code: 2 }); - expect(steps.some(span => span.name === "heartbeat.prepare.configuration_connector_assignments")).toBe(false); + expect(steps.some(span => span.name === "heartbeat.prepare.configuration_connector_assignments")).toBe(true); for (const span of steps) { expect(span.attributes).toEqual({ "paperclip.task.run.run_id": runIdHash }); expect(span.events).toEqual([]); diff --git a/server/src/__tests__/heartbeat-task-context.test.ts b/server/src/__tests__/heartbeat-task-context.test.ts index 5449d7def6c..e073db37676 100644 --- a/server/src/__tests__/heartbeat-task-context.test.ts +++ b/server/src/__tests__/heartbeat-task-context.test.ts @@ -1,10 +1,11 @@ import { createHash, randomUUID } from "node:crypto"; import { eq } from "drizzle-orm"; -import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { afterAll, beforeAll, describe, expect, it, vi } from "vitest"; import { agents, authUsers, companies, companyMemberships, createDb, heartbeatRuns, issueComments, issues } from "@paperclipai/db"; import { FAST_RESPONSE_AGENT_GUIDANCE } from "@paperclipai/shared"; import { registerServerAdapter, unregisterServerAdapter, type AdapterExecutionContext } from "../adapters/index.js"; import { heartbeatService } from "../services/heartbeat.js"; +import { prepareHeartbeatTaskContext } from "../services/heartbeat/task-context.js"; import { documentService } from "../services/documents.js"; import { ISSUE_CONTINUATION_SUMMARY_DOCUMENT_KEY } from "../services/issue-continuation-summary.js"; import { createRunSecretRedactionRegistry } from "../services/run-secret-redaction.js"; @@ -19,6 +20,7 @@ if (!support.supported) console.warn(`Skipping heartbeat task context tests: ${s describePostgres("heartbeat task context delivered to an adapter", () => { const adapterType = "task_context_test"; const captured = new Map(); + const inspectPreparation = new Map void>(); let temporary: Awaited>; let db: ReturnType; let heartbeat: ReturnType; @@ -33,6 +35,7 @@ describePostgres("heartbeat task context delivered to an adapter", () => { type: adapterType, supportsLocalAgentJwt: false, execute: async (input) => { + inspectPreparation.get(input.agent.id)?.(); captured.set(input.runId, input); return { exitCode: 0, signal: null, timedOut: false, sessionId: "context-session", resultJson: {} }; }, @@ -118,6 +121,101 @@ describePostgres("heartbeat task context delivered to an adapter", () => { }); }); + it("reads selected comments once for both wake projections", async () => { + const f = await seed(); + const commentId = randomUUID(); + await db.insert(issueComments).values({ + id: commentId, companyId: f.companyId, issueId: f.issueId, authorType: "user", authorUserId: "context-user", + body: "selected wake canary", + }); + const select = vi.spyOn(db, "select"); + inspectPreparation.set(f.agentId, () => { + const commentReads = select.mock.calls.filter(([projection]) => projection + && projection.body === issueComments.body && projection.id === issueComments.id && projection.deletedAt === issueComments.deletedAt); + expect(commentReads).toHaveLength(1); + }); + try { + const { context } = await execute(f, { wakeCommentId: commentId, wakeCommentIds: [commentId] }); + expect(context.paperclipWakeComment).toMatchObject({ id: commentId, body: "selected wake canary" }); + expect(context.paperclipWake).toMatchObject({ comments: [{ id: commentId, body: "selected wake canary" }] }); + } finally { + inspectPreparation.delete(f.agentId); + select.mockRestore(); + } + }); + + it("cancels deleted-only queued comments before adapter dispatch", async () => { + const f = await seed(), commentId = randomUUID(); + await db.insert(issueComments).values({ id: commentId, companyId: f.companyId, issueId: f.issueId, + authorType: "user", authorUserId: "context-user", body: "discarded wake", deletedAt: new Date() }); + const run = await heartbeat.wakeup(f.agentId, { source: "on_demand", triggerDetail: "manual", reason: "issue_commented", + payload: { issueId: f.issueId }, contextSnapshot: { issueId: f.issueId, wakeCommentId: commentId, wakeCommentIds: [commentId] } }); + expect(run).not.toBeNull(); + await heartbeat.drainActiveRunExecutions(); + const finished = await heartbeat.getRun(run!.id); + expect(finished).toMatchObject({ status: "cancelled", error: "Queued messages were discarded before dispatch" }); + expect(captured.has(run!.id)).toBe(false); + }); + + it("preserves deleted comment metadata without delivering its body", async () => { + const f = await seed(); + const commentId = randomUUID(); + await db.insert(issueComments).values({ id: commentId, companyId: f.companyId, issueId: f.issueId, + authorType: "user", authorUserId: "context-user", body: "deleted wake canary", deletedAt: new Date(), + presentation: { fixture: "deleted" }, metadata: { fixture: "deleted" } }); + const [run] = await db.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: f.agentId, + status: "running", issueId: f.issueId }).returning(); + const [agent] = await db.select().from(agents).where(eq(agents.id, f.agentId)); + const [issueRef] = await db.select().from(issues).where(eq(issues.id, f.issueId)); + const context: Record = { issueId: f.issueId, wakeCommentId: commentId, wakeCommentIds: [commentId] }; + await prepareHeartbeatTaskContext(db, { + run, agent, task: { issueRef, issueContext: null, context, taskSession: null }, + policy: { trustPreset: { kind: "standard" }, experimentalInstanceSettings: {} }, + services: { issuesSvc: { getAncestors: async () => [] }, getPinnedSkillTestContext: async () => null }, + }); + expect(context.paperclipWakeComment).toMatchObject({ id: commentId, body: "", presentation: null, metadata: null }); + expect(context.paperclipWake).toMatchObject({ comments: [{ id: commentId, body: "", presentation: null, metadata: null }] }); + expect(context.paperclipTaskMarkdown).not.toContain("deleted wake canary"); + }); + + it.each([false, true])("uses admitted skill-test scope when recovery is %s", async (recovering) => { + const f = await seed(); + const [run] = await db.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: f.agentId, + status: "running", issueId: f.issueId }).returning(); + const [agent] = await db.select().from(agents).where(eq(agents.id, f.agentId)); + const [issueRef] = await db.select().from(issues).where(eq(issues.id, f.issueId)); + const context: Record = { issueId: f.issueId, paperclipAgentJwtScope: { kind: "skill_test", issueId: f.issueId } }; + const pinned = vi.fn(async () => null); + const preparing = prepareHeartbeatTaskContext(db, { + run, agent, task: { issueRef, issueContext: null, context, taskSession: null }, + policy: { recovering, trustPreset: { kind: "standard" }, experimentalInstanceSettings: {} }, + services: { issuesSvc: { getAncestors: async () => [] }, getPinnedSkillTestContext: pinned }, + }); + if (recovering) await expect(preparing).rejects.toThrow("skill test's saved version is unavailable"); + else await preparing; + expect(pinned).toHaveBeenCalledTimes(recovering ? 1 : 0); + expect(context.paperclipAgentJwtScope).toEqual({ kind: recovering ? "skill_test" : "standard", ...(recovering ? { issueId: f.issueId } : {}) }); + }); + + it("retains the context issue scope when no issue reference is available", async () => { + const f = await seed(); + const otherIssueId = randomUUID(), commentId = randomUUID(); + await db.insert(issues).values({ id: otherIssueId, companyId: f.companyId, title: "Other issue", issueNumber: 2 }); + await db.insert(issueComments).values({ id: commentId, companyId: f.companyId, issueId: otherIssueId, + authorType: "user", authorUserId: "context-user", body: "other issue canary" }); + const [run] = await db.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: f.agentId, + status: "running", issueId: f.issueId }).returning(); + const [agent] = await db.select().from(agents).where(eq(agents.id, f.agentId)); + const context: Record = { issueId: f.issueId, wakeCommentId: commentId, wakeCommentIds: [commentId] }; + await prepareHeartbeatTaskContext(db, { + run, agent, task: { issueRef: null, issueContext: null, context, taskSession: null }, + policy: { trustPreset: { kind: "standard" }, experimentalInstanceSettings: {} }, + services: { issuesSvc: { getAncestors: async () => [] }, getPinnedSkillTestContext: async () => null }, + }); + expect(JSON.stringify(context.paperclipWake)).not.toContain("other issue canary"); + expect(context.paperclipWakeComment).toBeUndefined(); + }); + it("quarantines low-trust summaries and comments before preparing higher-trust prompts", async () => { const f = await seed(); const sourceTrust = buildLowTrustSourceTrust({ issueId: f.issueId }); diff --git a/server/src/services/heartbeat.ts b/server/src/services/heartbeat.ts index eb655fdef25..20b8ebe11c4 100644 --- a/server/src/services/heartbeat.ts +++ b/server/src/services/heartbeat.ts @@ -2007,8 +2007,11 @@ export function heartbeatService( const dispatchIssueId = readNonEmptyString(parseObject(run.contextSnapshot).issueId); const resumingAdmittedConversationTurn = !!runOptions.nativeLeaseOwner && typeof run.contextSnapshot?.conversationSessionGeneration === "number"; - if (dispatchIssueId && isConversation(await getIssueExecutionContext(run.companyId, dispatchIssueId)) - && !resumingAdmittedConversationTurn && !(await instanceSettings.getExperimental()).enableAgentChat) { + const conversationIdentity = dispatchIssueId && !resumingAdmittedConversationTurn + ? await db.select({ conversationAgentId: issues.conversationAgentId, conversationUserId: issues.conversationUserId }) + .from(issues).where(and(eq(issues.companyId, run.companyId), eq(issues.id, dispatchIssueId))).then(rows => rows[0] ?? null) + : null; + if (isConversation(conversationIdentity) && !(await instanceSettings.getExperimental()).enableAgentChat) { await setRunStatus(run.id, "cancelled", { finishedAt: new Date(), error: "Agent Chat is disabled", errorCode: "agent_chat_disabled" }); await setWakeupStatus(run.wakeupRequestId, "cancelled", { finishedAt: new Date() }); await releaseIssueExecutionAndPromote((await getRun(run.id))!, { suppressImmediateRecovery: true }); @@ -2204,44 +2207,6 @@ export function heartbeatService( context[PAPERCLIP_EXTERNAL_CHAT_EXECUTION_BOUND_KEY] = true; } const wakeCommentId = deriveCommentId(context, null); - const wakeCommentContext = - issueContext && wakeCommentId - ? await db - .select({ - id: issueComments.id, - body: issueComments.body, - authorType: issueComments.authorType, - authorAgentId: issueComments.authorAgentId, - authorUserId: issueComments.authorUserId, - presentation: issueComments.presentation, - metadata: issueComments.metadata, - deletedAt: issueComments.deletedAt, - deletedByType: issueComments.deletedByType, - deletedByAgentId: issueComments.deletedByAgentId, - deletedByUserId: issueComments.deletedByUserId, - deletedByRunId: issueComments.deletedByRunId, - sourceTrust: issueComments.sourceTrust, - }) - .from(issueComments) - .where( - and( - eq(issueComments.id, wakeCommentId), - eq(issueComments.issueId, issueContext.id), - eq(issueComments.companyId, agent.companyId), - ), - ) - .then((rows) => { - const row = rows[0] ?? null; - return row?.deletedAt - ? { - ...row, - body: "", - presentation: null, - metadata: null, - } - : row; - }) - : null; let issueAssigneeOverrides = issueContext && issueContext.assigneeAgentId === agent.id ? parseIssueAssigneeAdapterOverrides( @@ -2610,7 +2575,7 @@ export function heartbeatService( continuationSummary, executionContinuation, paperclipWakePayload, safeWakeComments, safeWakeCommentContext, pinnedSkillTest, } = await timePreparation("task_context", () => prepareHeartbeatTaskContext(db, { run, agent, - task: { issueRef, issueContext, wakeCommentContext, context, taskSession }, + task: { issueRef, issueContext, context, taskSession }, policy: { trustPreset, experimentalInstanceSettings, recovering: Boolean(runOptions.nativeRestartRecovery || runOptions.nativeLeaseOwner) }, services: { issuesSvc, getPinnedSkillTestContext }, })); diff --git a/server/src/services/heartbeat/execution-configuration.ts b/server/src/services/heartbeat/execution-configuration.ts index 9dd767a3811..fb9626b932f 100644 --- a/server/src/services/heartbeat/execution-configuration.ts +++ b/server/src/services/heartbeat/execution-configuration.ts @@ -139,28 +139,27 @@ export async function prepareHeartbeatExecutionConfiguration(db: Db, input: Hear stripWorkspaceRuntimeFromExecutionRunConfig(mergedConfig); const skillActor: AuthorizationActor = { type: "agent", agentId: agent.id, companyId: agent.companyId, runId: run.id, source: "agent_jwt", onBehalfOfUserId: responsibleUserId === "local-board" ? null : responsibleUserId }; - const runScopedMentionedSkillKeys = - preconfiguredHome || pinnedSkillTest ? [] : await time("configuration_mentions", () => resolveRunScopedMentionedSkillKeys({ + const [mentionedSkills, githubIdentity] = await Promise.allSettled([ + preconfiguredHome || pinnedSkillTest ? Promise.resolve([]) : time("configuration_mentions", () => resolveRunScopedMentionedSkillKeys({ db, companyId: agent.companyId, issueId, commentIds: extractWakeCommentIds(context), actor: skillActor, - })); + })), + time("configuration_github_identity", () => resolveManagedGitHubIdentitySelection(db, agent.companyId, { + agentId: agent.id, responsibleUserId, allowStandingDelegation: false, + })), + ]); + if (mentionedSkills.status === "rejected") throw mentionedSkills.reason; + if (githubIdentity.status === "rejected") throw githubIdentity.reason; + const runScopedMentionedSkillKeys = mentionedSkills.value; const runScopedSkillKeys = !pinnedSkillTest && acceptedPlanContinuationWake && !acceptedPlanWakeRoutingDecision?.suppressAcceptedContinuation ? [...runScopedMentionedSkillKeys, ACCEPTED_PLAN_CONVERSION_SKILL_KEY] : runScopedMentionedSkillKeys; - const githubSelection = await time("configuration_github_identity", () => resolveManagedGitHubIdentitySelection( - db, - agent.companyId, - { - agentId: agent.id, - responsibleUserId, - allowStandingDelegation: false, - }, - )); + const githubSelection = githubIdentity.value; const useHostGitHub = !githubSelection.configured && trustPreset.kind === "standard" && @@ -274,6 +273,7 @@ export async function prepareHeartbeatExecutionConfiguration(db: Db, input: Hear ); const nativeRunnerPreparationSpans: NativeRunHistoricalSpan[] = []; const skillsPrepareStartedAtMs = Date.now(); + let skillsPrepareEndedAtMs = skillsPrepareStartedAtMs; let skillDeliveryConfig = effectiveResolvedConfig; const prepareSkills = async () => { @@ -346,24 +346,35 @@ export async function prepareHeartbeatExecutionConfiguration(db: Db, input: Hear throw error; } }; - const runtimeSkillEntries = preconfiguredHome ? [] : await time("configuration_skills", prepareSkills); + const skillsPreparation = (preconfiguredHome ? Promise.resolve([]) : time("configuration_skills", prepareSkills)) + .finally(() => { skillsPrepareEndedAtMs = Date.now(); }); + const assignmentsPreparation = preconfiguredHome ? Promise.resolve([]) : time("configuration_connector_assignments", () => resolveConnectorAssignments(db, { companyId: agent.companyId, agentId: agent.id, runId: run.id, issueId: typeof context.issueId === "string" ? context.issueId : undefined })); + const [preparedSkills, preparedAssignments, preparedInstructions] = await Promise.allSettled([ + skillsPreparation, + assignmentsPreparation, + assignmentsPreparation.then(() => time("configuration_connection_instructions", () => resolveAssignedConnectionInstructionsForRun(db, { companyId: agent.companyId, agentId: agent.id, runId: run.id }))), + ]); + if (preparedSkills.status === "rejected") throw preparedSkills.reason; + if (preparedAssignments.status === "rejected") throw preparedAssignments.reason; + if (preparedInstructions.status === "rejected") throw preparedInstructions.reason; + const runtimeSkillEntries = preparedSkills.value; nativeRunnerPreparationSpans.push({ name: "skills.prepare", parentName: "task.prepare", startedAtMs: skillsPrepareStartedAtMs, - endedAtMs: Date.now(), + endedAtMs: skillsPrepareEndedAtMs, }); - const connectorAssignments = preconfiguredHome ? [] : await time("configuration_connector_assignments", () => resolveConnectorAssignments(db, { companyId: agent.companyId, agentId: agent.id, runId: run.id, issueId: typeof context.issueId === "string" ? context.issueId : undefined })); + const connectorAssignments = preparedAssignments.value; const connectorSkillConfig = await time("configuration_connector_skills", () => applyConnectorSkills(skillDeliveryConfig, runtimeSkillEntries, connectorAssignments)); // Both CLI adapters and native context materialization use the same resolved set. runtimeSkillEntries.splice(0, runtimeSkillEntries.length, ...connectorSkillConfig.paperclipRuntimeSkills); const connectorDelivery = await time("configuration_connector_delivery", () => prepareConnectorSkillDelivery(connectorSkillConfig, agent.adapterType)); // Always replace this runtime-only field; caller wake data cannot supply skills. context.paperclipWake = { ...parseObject(context.paperclipWake), connectorSkillInstructions: connectorDelivery.instructions }; - let runtimeConfig = await time("configuration_connection_instructions", () => prepareConnectionInstructionDelivery({ - resolve: () => resolveAssignedConnectionInstructionsForRun(db, { companyId: agent.companyId, agentId: agent.id, runId: run.id }), + let runtimeConfig = await prepareConnectionInstructionDelivery({ + resolve: async () => preparedInstructions.value, context, config: connectorDelivery.config, native: agent.adapterType === "paperclip_runner", - })); + }); const agentIdentity = supportsManagedAgentIdentity(agent.adapterType, agent.adapterConfig, run.runtimeMode === "native" ? run.driverKind : undefined) ? await time("configuration_agent_identity", () => agentIdentityService(db).ensureAgentIdentity(agent.companyId, agent.id)) : undefined; diff --git a/server/src/services/heartbeat/run-preparation.test.ts b/server/src/services/heartbeat/run-preparation.test.ts index 10a447e1a74..7d5a070d91d 100644 --- a/server/src/services/heartbeat/run-preparation.test.ts +++ b/server/src/services/heartbeat/run-preparation.test.ts @@ -14,6 +14,7 @@ import { ConfigurationIncompleteFailure, createHeartbeatRunPreparation, resolveExecutionRunAdapterConfig, + readHeartbeatWakeComments, } from "./run-preparation.js"; // Existing integration suites exercise the real database and dispatch path. @@ -38,6 +39,12 @@ function databaseWithResults(...results: unknown[][]) { return { db: { select } as unknown as Db, select, scopes }; } +function deferred() { + let resolve!: (value: T) => void; + const promise = new Promise(complete => { resolve = complete; }); + return { promise, resolve }; +} + const emptyRoutineContext = { routineId: null, env: null, responsibleUserId: null }; describe("heartbeat run preparation module", () => { @@ -120,6 +127,42 @@ describe("heartbeat run preparation module", () => { expect(database.select).toHaveBeenCalledTimes(2); }); + it("scopes one selected-comment snapshot for both wake consumers", async () => { + const database = databaseWithResults([{ id: "comment-one", body: "selected wake" }]); + expect(await readHeartbeatWakeComments(database.db, "company-one", "issue-one", ["comment-one"])) + .toEqual([{ id: "comment-one", body: "selected wake" }]); + expect(database.scopes[0]).toEqual(expect.arrayContaining(["company-one", "issue-one", "comment-one"])); + expect(await readHeartbeatWakeComments(database.db, "company-one", "issue-one", [])).toEqual([]); + expect(database.select).toHaveBeenCalledOnce(); + }); + + it("settles independent binding checks before surfacing a failure or resolving values", async () => { + const projectCheck = deferred<[]>(); + const firstFailure = new Error("agent binding check failed"); + const secretsSvc = { + resolveAdapterConfigForRuntime: vi.fn(), resolveEnvBindings: vi.fn(), + collectMissingRuntimeBindings: vi.fn(async (_company, _env, context) => { + if (context.consumerType === "agent") throw firstFailure; + return context.consumerType === "project" ? projectCheck.promise : []; + }), + collectMissingAdapterConfigRuntimeBindings: vi.fn(async () => []), + }; + let settled = false; + const opening = resolveExecutionRunAdapterConfig({ + companyId: "company-one", agentId: "agent-one", projectId: "project-one", + environmentId: "environment-one", environmentEnv: { SHARED: "environment" }, + executionRunConfig: { env: { SHARED: "agent" } }, projectEnv: { SHARED: "project" }, secretsSvc, + }); + void opening.then(() => { settled = true; }, () => { settled = true; }); + await vi.waitFor(() => expect(secretsSvc.collectMissingRuntimeBindings).toHaveBeenCalledTimes(3)); + expect(secretsSvc.collectMissingAdapterConfigRuntimeBindings).toHaveBeenCalledOnce(); + expect(settled).toBe(false); + expect(secretsSvc.resolveAdapterConfigForRuntime).not.toHaveBeenCalled(); + expect(secretsSvc.resolveEnvBindings).not.toHaveBeenCalled(); + projectCheck.resolve([]); + await expect(opening).rejects.toBe(firstFailure); + }); + it("keeps missing scoped credentials as a pre-dispatch failure", async () => { const secretsSvc = { resolveAdapterConfigForRuntime: vi.fn(), diff --git a/server/src/services/heartbeat/run-preparation.ts b/server/src/services/heartbeat/run-preparation.ts index bf288599a51..bfbe6c862c1 100644 --- a/server/src/services/heartbeat/run-preparation.ts +++ b/server/src/services/heartbeat/run-preparation.ts @@ -394,76 +394,31 @@ export async function resolveExecutionRunAdapterConfig(input: { // no binding before resolving any secret value. Missing bindings short-circuit // to a configuration-incomplete blocker routed to a human owner instead of a // dispatched-then-failed run (which previously surfaced as opaque setup_failed). - if (typeof input.secretsSvc.collectMissingRuntimeBindings === "function") { - const missingBindings: MissingRuntimeBinding[] = []; - if (environmentEnv && input.environmentId) { - missingBindings.push( - ...(await input.secretsSvc.collectMissingRuntimeBindings( - input.companyId, - environmentEnv, - { - consumerType: "environment", - consumerId: input.environmentId, - responsibleUserId: input.responsibleUserId ?? null, - }, - )), - ); - } + const collectMissing = input.secretsSvc.collectMissingRuntimeBindings; + if (typeof collectMissing === "function") { + const checks: Array<() => Promise> = []; + const checkEnv = (env: unknown, consumerType: "environment" | "agent" | "project" | "routine", consumerId: string) => { + checks.push(() => collectMissing(input.companyId, env, { + consumerType, consumerId, responsibleUserId: input.responsibleUserId ?? null, + })); + }; + if (environmentEnv && input.environmentId) checkEnv(environmentEnv, "environment", input.environmentId); if (input.agentId) { - missingBindings.push( - ...(await input.secretsSvc.collectMissingRuntimeBindings( - input.companyId, - parseObject(executionRunConfig.env), - { - consumerType: "agent", - consumerId: input.agentId, - responsibleUserId: input.responsibleUserId ?? null, - }, - )), - ); - if ( - typeof input.secretsSvc.collectMissingAdapterConfigRuntimeBindings === - "function" - ) { - missingBindings.push( - ...(await input.secretsSvc.collectMissingAdapterConfigRuntimeBindings( - input.companyId, - executionRunConfig, - input.adapterType ?? null, - { - consumerType: "agent", - consumerId: input.agentId, - responsibleUserId: input.responsibleUserId ?? null, - }, - )), - ); + checkEnv(parseObject(executionRunConfig.env), "agent", input.agentId); + const collectAdapterMissing = input.secretsSvc.collectMissingAdapterConfigRuntimeBindings; + if (typeof collectAdapterMissing === "function") { + checks.push(() => collectAdapterMissing(input.companyId, executionRunConfig, input.adapterType ?? null, { + consumerType: "agent", consumerId: input.agentId!, responsibleUserId: input.responsibleUserId ?? null, + })); } } - if (projectEnv && input.projectId) { - missingBindings.push( - ...(await input.secretsSvc.collectMissingRuntimeBindings( - input.companyId, - projectEnv, - { - consumerType: "project", - consumerId: input.projectId, - responsibleUserId: input.responsibleUserId ?? null, - }, - )), - ); - } - if (routineEnv && input.routineId) { - missingBindings.push( - ...(await input.secretsSvc.collectMissingRuntimeBindings( - input.companyId, - routineEnv, - { - consumerType: "routine", - consumerId: input.routineId, - responsibleUserId: input.responsibleUserId ?? null, - }, - )), - ); + if (projectEnv && input.projectId) checkEnv(projectEnv, "project", input.projectId); + if (routineEnv && input.routineId) checkEnv(routineEnv, "routine", input.routineId); + const results = await Promise.allSettled(checks.map(check => Promise.resolve().then(check))); + const missingBindings: MissingRuntimeBinding[] = []; + for (const result of results) { + if (result.status === "rejected") throw result.reason; + missingBindings.push(...result.value); } if (requiredScopedEnvBinding) { const requiredEnvKeys = new Set(requiredScopedEnvBinding.keys); @@ -2244,12 +2199,45 @@ export async function attestReviewedExternalChatRun(input: { throw new Error("reviewed_chat_execution_binding_not_ready"); } +export async function readHeartbeatWakeComments(db: Db, companyId: string, issueId: string | null, commentIds: string[]) { + if (commentIds.length === 0) return []; + return db + .select({ + id: issueComments.id, + companyId: issueComments.companyId, + issueId: issueComments.issueId, + body: issueComments.body, + origin: issueComments.origin, + authorType: issueComments.authorType, + authorAgentId: issueComments.authorAgentId, + authorUserId: issueComments.authorUserId, + presentation: issueComments.presentation, + metadata: issueComments.metadata, + deletedAt: issueComments.deletedAt, + deletedByType: issueComments.deletedByType, + deletedByAgentId: issueComments.deletedByAgentId, + deletedByUserId: issueComments.deletedByUserId, + deletedByRunId: issueComments.deletedByRunId, + sourceTrust: issueComments.sourceTrust, + createdAt: issueComments.createdAt, + }) + .from(issueComments) + .where( + and( + eq(issueComments.companyId, companyId), + issueId ? eq(issueComments.issueId, issueId) : undefined, + inArray(issueComments.id, commentIds), + ), + ); +} + export async function buildPaperclipWakePayload(input: { db: Db; companyId: string; agentId?: string | null; runId?: string | null; contextSnapshot: Record; + wakeCommentRows?: Awaited>; continuationSummary?: { key: string; title: string | null; @@ -2314,36 +2302,9 @@ export async function buildPaperclipWakePayload(input: { ) return null; - const commentRows = - commentIds.length === 0 - ? [] - : await input.db - .select({ - id: issueComments.id, - issueId: issueComments.issueId, - body: issueComments.body, - origin: issueComments.origin, - authorType: issueComments.authorType, - authorAgentId: issueComments.authorAgentId, - authorUserId: issueComments.authorUserId, - presentation: issueComments.presentation, - metadata: issueComments.metadata, - deletedAt: issueComments.deletedAt, - deletedByType: issueComments.deletedByType, - deletedByAgentId: issueComments.deletedByAgentId, - deletedByUserId: issueComments.deletedByUserId, - deletedByRunId: issueComments.deletedByRunId, - sourceTrust: issueComments.sourceTrust, - createdAt: issueComments.createdAt, - }) - .from(issueComments) - .where( - and( - eq(issueComments.companyId, input.companyId), - issueId ? eq(issueComments.issueId, issueId) : undefined, - inArray(issueComments.id, commentIds), - ), - ); + const commentRows = input.wakeCommentRows + ? input.wakeCommentRows.filter(row => row.companyId === input.companyId && (!issueId || row.issueId === issueId) && commentIds.includes(row.id)) + : await readHeartbeatWakeComments(input.db, input.companyId, issueId, commentIds); const commentsById = new Map( commentRows.map((comment) => [comment.id, comment]), diff --git a/server/src/services/heartbeat/task-context.ts b/server/src/services/heartbeat/task-context.ts index 85bdd65ca61..9c1fb122dd4 100644 --- a/server/src/services/heartbeat/task-context.ts +++ b/server/src/services/heartbeat/task-context.ts @@ -1,5 +1,5 @@ import { createHash } from "node:crypto"; -import type { Db, agents, agentTaskSessions, heartbeatRuns, issueComments } from "@paperclipai/db"; +import type { Db, agents, agentTaskSessions, heartbeatRuns } from "@paperclipai/db"; import { FAST_RESPONSE_AGENT_GUIDANCE, fastResponseHistoryBody } from "@paperclipai/shared"; import type { PaperclipTurnContext } from "@paperclipai/adapter-utils/server-utils"; import { parseObject } from "../../adapters/utils.js"; @@ -15,8 +15,9 @@ import { getTaskPlanContext } from "../task-plan-context.js"; import { caseExecutionMarkdown, getCaseExecutionContext } from "../case-context.js"; import type { TrustPresetResolution } from "../trust-preset-resolver.js"; import { liveVoiceExecutionGuidance } from "../voice/voice-execution-guidance.js"; -import { buildPaperclipWakePayload, ConfigurationIncompleteFailure, PAPERCLIP_WAKE_PAYLOAD_KEY, type createHeartbeatRunPreparation } from "./run-preparation.js"; +import { buildPaperclipWakePayload, ConfigurationIncompleteFailure, readHeartbeatWakeComments, PAPERCLIP_WAKE_PAYLOAD_KEY, type createHeartbeatRunPreparation } from "./run-preparation.js"; import { buildPaperclipTaskMarkdown } from "./task-markdown.js"; +import { deriveCommentId, extractWakeCommentIds } from "../../modules/run-dispatch/index.js"; import { workspacePreferenceService } from "../workspace-preferences.js"; type RunPreparation = ReturnType; @@ -30,8 +31,6 @@ export interface HeartbeatTaskContextInput { "id" | "identifier" | "title" | "status" | "priority" | "workMode" | "conversationAgentId" | "description" | "projectId"> | null; issueContext: IssueContext | null; - wakeCommentContext: Pick | null; context: Record; taskSession: Pick | null; }; @@ -53,35 +52,9 @@ function readNonEmptyString(value: unknown): string | null { /** Rebuilds task prompt fields on the run's context; acquires no execution resources. */ export async function prepareHeartbeatTaskContext(db: Db, input: HeartbeatTaskContextInput) { const { run, agent } = input; - const { issueRef, issueContext, wakeCommentContext, context, taskSession } = input.task; + const { issueRef, issueContext, context, taskSession } = input.task; const { trustPreset, experimentalInstanceSettings } = input.policy; const { issuesSvc, getPinnedSkillTestContext } = input.services; - const continuationSummary = issueRef && !isConversation(issueContext) - ? await getIssueContinuationSummaryDocument(db, issueRef.id) - : null; - const exposeLowTrustRaw = trustPreset.kind === "low_trust_review"; - const safeContinuationSummary = - continuationSummary && !exposeLowTrustRaw - ? redactQuarantinedBodyForHigherTrust(continuationSummary) - : continuationSummary; - const safeWakeCommentContext = - wakeCommentContext && !exposeLowTrustRaw - ? sanitizeQuarantinedCommentForHigherTrust(wakeCommentContext) - : wakeCommentContext; - const issueAncestors = issueRef - ? await issuesSvc.getAncestors(issueRef.id) - : []; - if (continuationSummary) { - context.paperclipContinuationSummary = { - key: safeContinuationSummary!.key, - title: safeContinuationSummary!.title, - body: safeContinuationSummary!.body, - sourceTrust: safeContinuationSummary!.sourceTrust ?? null, - updatedAt: safeContinuationSummary!.updatedAt.toISOString(), - }; - } else { - delete context.paperclipContinuationSummary; - } const admittedScope = parseObject(context.paperclipAgentJwtScope); if (!input.policy.recovering || (admittedScope.kind !== "standard" && admittedScope.kind !== "skill_test")) { const savedIssue = parseObject(context.paperclipIssue); @@ -93,10 +66,58 @@ export async function prepareHeartbeatTaskContext(db: Db, input: HeartbeatTaskCo : { kind: "standard" }; } const isSkillTest = parseObject(context.paperclipAgentJwtScope).kind === "skill_test"; - const pinnedSkillTestContext = - isSkillTest && issueRef - ? await getPinnedSkillTestContext(agent.companyId, issueRef.id) - : null; + const wakeCommentId = deriveCommentId(context, null); + const commentIds = [...new Set([...extractWakeCommentIds(context), ...(wakeCommentId ? [wakeCommentId] : [])])]; + const [summary, ancestors, pinnedSkillTestRead, wakeComments] = await Promise.allSettled([ + issueRef && !isConversation(issueContext) ? getIssueContinuationSummaryDocument(db, issueRef.id) : Promise.resolve(null), + issueRef ? issuesSvc.getAncestors(issueRef.id) : Promise.resolve([]), + isSkillTest && issueRef ? getPinnedSkillTestContext(agent.companyId, issueRef.id) : Promise.resolve(null), + readHeartbeatWakeComments(db, agent.companyId, readNonEmptyString(context.issueId), commentIds), + ]); + if (summary.status === "rejected") throw summary.reason; + if (ancestors.status === "rejected") throw ancestors.reason; + if (pinnedSkillTestRead.status === "rejected") throw pinnedSkillTestRead.reason; + if (wakeComments.status === "rejected") throw wakeComments.reason; + const wakeCommentRows = wakeComments.value; + const selectedWakeComment = issueRef && wakeCommentId ? wakeCommentRows.find(comment => comment.id === wakeCommentId && comment.issueId === issueRef.id) ?? null : null; + const wakeCommentContext = selectedWakeComment ? { + id: selectedWakeComment.id, + body: selectedWakeComment.deletedAt ? "" : selectedWakeComment.body, + authorType: selectedWakeComment.authorType, + authorAgentId: selectedWakeComment.authorAgentId, + authorUserId: selectedWakeComment.authorUserId, + presentation: selectedWakeComment.deletedAt ? null : selectedWakeComment.presentation, + metadata: selectedWakeComment.deletedAt ? null : selectedWakeComment.metadata, + deletedAt: selectedWakeComment.deletedAt, + deletedByType: selectedWakeComment.deletedByType, + deletedByAgentId: selectedWakeComment.deletedByAgentId, + deletedByUserId: selectedWakeComment.deletedByUserId, + deletedByRunId: selectedWakeComment.deletedByRunId, + sourceTrust: selectedWakeComment.sourceTrust, + } : null; + const continuationSummary = summary.value; + const issueAncestors = ancestors.value; + const pinnedSkillTestContext = pinnedSkillTestRead.value; + const exposeLowTrustRaw = trustPreset.kind === "low_trust_review"; + const safeContinuationSummary = + continuationSummary && !exposeLowTrustRaw + ? redactQuarantinedBodyForHigherTrust(continuationSummary) + : continuationSummary; + const safeWakeCommentContext = + wakeCommentContext && !exposeLowTrustRaw + ? sanitizeQuarantinedCommentForHigherTrust(wakeCommentContext) + : wakeCommentContext; + if (continuationSummary) { + context.paperclipContinuationSummary = { + key: safeContinuationSummary!.key, + title: safeContinuationSummary!.title, + body: safeContinuationSummary!.body, + sourceTrust: safeContinuationSummary!.sourceTrust ?? null, + updatedAt: safeContinuationSummary!.updatedAt.toISOString(), + }; + } else { + delete context.paperclipContinuationSummary; + } if (isSkillTest && !pinnedSkillTestContext) { throw new ConfigurationIncompleteFailure("The skill test's saved version is unavailable.", { configurationIncomplete: { reason: "skill_unavailable", actionUrl: "/skills" }, @@ -132,6 +153,7 @@ export async function prepareHeartbeatTaskContext(db: Db, input: HeartbeatTaskCo agentId: agent.id, runId: run.id, contextSnapshot: context, + wakeCommentRows, continuationSummary, issueSummary: issueRef ? { diff --git a/server/src/services/native-runtime/runner-api.integration.test.ts b/server/src/services/native-runtime/runner-api.integration.test.ts index 4e1831b734a..fdb5b7f70a2 100644 --- a/server/src/services/native-runtime/runner-api.integration.test.ts +++ b/server/src/services/native-runtime/runner-api.integration.test.ts @@ -497,7 +497,7 @@ else if(m.id!==undefined) send({id:m.id,result:{}}); const [agent] = await server.db.select().from(agents).where(eq(agents.id, fixture.agentId)); const context = structuredClone(run.contextSnapshot ?? {}); await prepareHeartbeatTaskContext(server.db, { run, agent, - task: { issueRef: issue, issueContext: null, wakeCommentContext: null, context, taskSession: null }, + task: { issueRef: issue, issueContext: null, context, taskSession: null }, policy: { recovering: true, trustPreset: { kind: "standard" }, experimentalInstanceSettings: {} }, services: { issuesSvc: { getAncestors: async () => [] }, getPinnedSkillTestContext: createHeartbeatRunPreparation(server.db).getPinnedSkillTestContext }, From cd4caaab266ecd857563be32e7aec9974fc26e4a Mon Sep 17 00:00:00 2001 From: Dotta Date: Fri, 30 Oct 2026 18:49:20 -0500 Subject: [PATCH 52/70] fix: batch status-card source notifications Coalesce committed company hints in a fixed one-second window without flooring real deadlines. Verify one hundred ordinary issue/comment writes, company isolation, deadline preemption, retained hints and settled SQL quietness. Await actual evaluation I/O in fake-clock acceptance instead of jumping across connection timers. Co-Authored-By: Paperclip --- doc/plans/2026-10-30-status-cards-sleep.md | 12 ++ .../status-card-evaluation-work.test.ts | 130 +++++++++++++++++- server/src/services/status-card-evaluation.ts | 4 + 3 files changed, 139 insertions(+), 7 deletions(-) diff --git a/doc/plans/2026-10-30-status-cards-sleep.md b/doc/plans/2026-10-30-status-cards-sleep.md index dbf229cdace..5b98962698b 100644 --- a/doc/plans/2026-10-30-status-cards-sleep.md +++ b/doc/plans/2026-10-30-status-cards-sleep.md @@ -123,3 +123,15 @@ five minutes keeps its remaining time without extension. This bounds migration compatibility delay while preserving prior crash/failure recovery. After the first successful evaluation all newly saved deadlines have explicit ownership. No recurring compatibility scan is added. + +Source notifications use the shared deadline helper's fixed one-second batching +window. The first hint after idle can reconcile immediately. Later hints merge +by company until that window ends. The window does not slide with each new +write, so a sustained source stream cannot postpone reconciliation forever. +One second keeps reactive cards prompt (the removed heartbeat default was 30s) +and bounds burst SQL. No minimum spacing is set: an already-known policy, +expiry, or retry deadline can preempt batching. The window alone creates no +scheduled work after settlement. Real issue/comment producer acceptance measures +due/next queries and actual searches under 100 writes, preserves both companies' +hints, dispatches another company's saved deadline inside the batching window, +and checks SQL quietness after authorized results. diff --git a/server/src/__tests__/status-card-evaluation-work.test.ts b/server/src/__tests__/status-card-evaluation-work.test.ts index 4e00e326855..6bb466dea89 100644 --- a/server/src/__tests__/status-card-evaluation-work.test.ts +++ b/server/src/__tests__/status-card-evaluation-work.test.ts @@ -1,5 +1,7 @@ import { budgetService } from "../services/budgets.js"; import { documentService } from "../services/documents.js"; +import * as companySearch from "../services/company-search.js"; +import { setImmediate as immediate } from "node:timers/promises"; import { randomUUID } from "node:crypto"; import { eq, sql } from "drizzle-orm"; import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; @@ -24,6 +26,7 @@ const support = external ? { supported: true } : await getEmbeddedPostgresTestSu let draining = false; let companyId: string, agentId: string; const errors: unknown[] = []; + const evaluations = new Set>(); let compiledQuery = companySearchQuerySchema.parse({ scope: "issues", status: ["blocked"] }); const actor = { agentId: null, userId: "board-user" }; const policy = (mode: "manual" | "reactive" | "interval" = "reactive") => statusCardRefreshPolicySchema.parse({ @@ -60,16 +63,29 @@ const support = external ? { supported: true } : await getEmbeddedPostgresTestSu await db.delete(instanceSettings); await db.delete(agents); await db.delete(companyMemberships); await db.delete(companies); }); afterAll(async () => { await db?.$client.end({ timeout: 1 }); await temporary?.cleanup(); }); - async function start() { + async function start(service = statusCardEvaluationService(db, { wakeup })) { coordinator = createDeliveryWorkCoordinator({ owner: db, canRun: () => !draining, canReconcile: () => true, onError: error => errors.push(error) }); - await registerStatusCardEvaluationWork(coordinator, statusCardEvaluationService(db, { wakeup }), () => !draining).ready; + await registerStatusCardEvaluationWork(coordinator, { ...service, run: signal => { + const work = service.run(signal); + evaluations.add(work); + void work.then(() => evaluations.delete(work), () => evaluations.delete(work)); + return work; + } }, () => !draining).ready; } async function settled() { - await vi.waitFor(async () => { - await vi.advanceTimersByTimeAsync(30_000); - expect(idleWorkSnapshot().active).toBe(0); - }, { timeout: 5000 }); + // Do not leap across PostgreSQL connect timers while real I/O is pending. + // Await actual domain work, then require several settled event-loop turns + // so a transient zero hold cannot stand in for transaction reconciliation. + for (let turn = 0; turn < 3; turn++) { + await vi.waitFor(async () => { + await Promise.allSettled([...evaluations]); + await vi.advanceTimersByTimeAsync(500); + await Promise.allSettled([...evaluations]); + expect(idleWorkSnapshot().active).toBe(0); + }, { timeout: 5000 }); + await immediate(); + } } async function quiet() { await settled(); @@ -128,11 +144,111 @@ const support = external ? { supported: true } : await getEmbeddedPostgresTestSu expect(errors).toEqual([]); }); + it("coalesces one hundred committed source writes without delaying another company's saved deadline", async () => { + const reactive = await card(); + const originalCompany = companyId, originalAgent = agentId; + companyId = randomUUID(); agentId = randomUUID(); + await db.insert(companies).values({ id: companyId, name: "Other", issuePrefix: `O${companyId.slice(0, 6)}` }); + await db.insert(agents).values({ id: agentId, companyId, name: "Other summarizer", role: "general", status: "idle", adapterType: "process", adapterConfig: {} }); + const other = await card("interval"); + const otherIssue = await issueService(db).create(companyId, { title: "Other interval blocker", status: "blocked", priority: "medium" }); + const otherReactive = await card(); + companyId = originalCompany; agentId = originalAgent; + const service = statusCardEvaluationService(db, { wakeup }); + const originalRun = service.run; + let running: Promise | undefined; + const run = vi.spyOn(service, "run").mockImplementation(signal => { + const result = originalRun(signal); running = result; return result; + }); + await start(service); await settled(); + await vi.advanceTimersByTimeAsync(1000); + // Give each committed producer and any actual domain work time to settle. + // This prevents incidental overlap from hiding per-write reconstruction. + const flush = async () => { + for (let i = 0; i < 10; i++) { + await running; await immediate(); + } + }; + await flush(); + const select = vi.spyOn(db, "select"); + const factory = companySearch.companySearchService; + const searchedCompanies: string[] = []; + vi.spyOn(companySearch, "companySearchService").mockImplementation(owner => { + const search = factory(owner); + const actual = search.search; + search.search = async (...args) => { searchedCompanies.push(args[0]); return actual(...args); }; + return search; + }); + run.mockClear(); + const burstStarted = Date.now(); + for (let i = 0; i < 50; i++) { + const issue = await issueService(db).create(companyId, { title: `Burst blocker ${i}`, status: "blocked", priority: "medium" }); + await flush(); + await issueService(db).addComment(issue.id, `Human update ${i}`, { userId: "board-user" }); + await flush(); + } + await flush(); + // One eager reconciliation after idle and one bounded batched continuation; + // the producers themselves are not counted as background SQL. + const dueReads = () => select.mock.calls.filter(([fields]) => fields && "card" in fields && fields.card === statusCards).length; + const nextReads = () => select.mock.calls.filter(([fields]) => fields && "at" in fields && fields.at === statusCards.nextEvalAt).length; + const elapsedMs = Date.now() - burstStarted; + const maximum = Math.ceil(elapsedMs / 1000) + 2; + console.info("status-card burst", { elapsedMs, maximum, runs: run.mock.calls.length, due: dueReads(), next: nextReads(), searches: searchedCompanies.length }); + expect(run.mock.calls.length).toBeLessThanOrEqual(maximum); + expect(dueReads()).toBeLessThanOrEqual(maximum); expect(nextReads()).toBeLessThanOrEqual(maximum); + expect(searchedCompanies.length).toBeLessThanOrEqual(maximum); + expect(searchedCompanies.length).toBeGreaterThan(0); + expect(new Set(searchedCompanies)).toEqual(new Set([originalCompany])); + expect(wakeup).not.toHaveBeenCalled(); + // Re-register with an existing durable deadline inside the coalescing + // window. A company hint must not make that real deadline wait one second. + await coordinator!.stop(); coordinator = undefined; + const dueAt = Date.now() + 500; + await db.update(statusCards).set({ nextEvalAt: new Date(dueAt), nextEvalReason: "refresh" }).where(eq(statusCards.id, other.id)); + await start(service); await flush(); + await issueService(db).addComment(otherIssue.id, "Another company's coalesced human update", { userId: "board-user" }); + await issueService(db).addComment((await db.select({ id: issues.id }).from(issues).where(eq(issues.companyId, originalCompany)))[0]!.id, + "More changes while another company is due", { userId: "board-user" }); + await flush(); + await vi.advanceTimersByTimeAsync(500); await flush(); + await vi.waitFor(() => expect(wakeup).toHaveBeenCalledOnce()); + expect(Date.now()).toBeLessThan(dueAt + 500); + expect(wakeup.mock.calls[0]![0]).not.toBe(originalAgent); + const generated = (await statusCardService(db).getById(other.id))!; + expect(generated.generatingIssueId).toBeTruthy(); + expect((await statusCardService(db).getById(reactive.id))!.pendingChangeCount).toBeGreaterThan(0); + expect((await statusCardService(db).getById(otherReactive.id))!.pendingChangeCount).toBeGreaterThan(0); + expect(searchedCompanies).toContain(other.companyId); + // Complete all ordinary results, then prove there is no recurring batch + // timer once the requested work has settled. + companyId = other.companyId; agentId = other.agentId!; + await writeback(other.id, generated.generatingIssueId!); + await statusCardService(db).update((await statusCardService(db).getById(other.id))!, { archived: true }, actor); + companyId = originalCompany; agentId = originalAgent; + await vi.advanceTimersByTimeAsync(120_000); await settled(); await flush(); + await vi.waitFor(async () => expect((await statusCardService(db).getById(reactive.id))!.generatingIssueId).toBeTruthy()); + const pending = (await statusCardService(db).getById(reactive.id))!; + expect(pending.generatingIssueId).toBeTruthy(); + await writeback(reactive.id, pending.generatingIssueId!); + const otherPending = (await statusCardService(db).getById(otherReactive.id))!; + expect(otherPending.generatingIssueId).toBeTruthy(); + companyId = other.companyId; agentId = other.agentId!; + await writeback(otherReactive.id, otherPending.generatingIssueId!); + companyId = originalCompany; agentId = originalAgent; + select.mockRestore(); + await quiet(); expect(errors).toEqual([]); + }, 30_000); + it("cancels archived or disabled future work and replaces an earlier interval deadline", async () => { await start(); let created = await card("interval"); await settled(); const old = coordinator!.nextWakeAt(); created = await statusCardService(db).update(created, { refreshPolicy: { ...policy("interval"), intervalMinutes: 1 } }, actor); - await settled(); expect(coordinator!.nextWakeAt()).toBeLessThan(old!); + await vi.waitFor(async () => { + await vi.advanceTimersByTimeAsync(100); + await Promise.allSettled([...evaluations]); + expect(coordinator!.nextWakeAt()).toBeLessThan(old!); + }); await statusCardService(db).update(created, { archived: true }, actor); await quiet(); expect(wakeup).not.toHaveBeenCalled(); expect(errors).toEqual([]); }); diff --git a/server/src/services/status-card-evaluation.ts b/server/src/services/status-card-evaluation.ts index 3e33c0d0885..e6086535fc9 100644 --- a/server/src/services/status-card-evaluation.ts +++ b/server/src/services/status-card-evaluation.ts @@ -90,5 +90,9 @@ export function registerStatusCardEvaluationWork( ) { return registerDeadlineWork(coordinator, DELIVERY_QUEUES.statusCardEvaluation, { run: service.run, next: service.next, onSettled: service.changed, canRun, retryMs: 30_000, + // Batch source hints for at most one second after reconciliation. Do not + // use minimumSpacingMs: known policy/retry deadlines must preempt batching. + // This window alone arms no timer when the feature has no pending work. + notificationCoalesceMs: 1_000, }); } From 105a67b52c66a738c0a8eb7807332b99931ac45d Mon Sep 17 00:00:00 2001 From: Devin Foley Date: Fri, 30 Oct 2026 16:54:04 -0700 Subject: [PATCH 53/70] Reuse signed master images for npm canaries (#18404) ## Thinking Path > - Paperclip is the open source app people use to manage AI agents for work. > - Release workflows publish npm packages and multi-platform images. > - A master push can finish its image before the npm canary dispatch starts. > - The dispatch currently builds that same source again on two runners. > - This PR verifies and reuses the signed master image by digest. > - Missing images retain the native build path. Channel promotion still follows npm. ## Linked Issues or Issue Description **What existing behavior does this improve?** The Docker workflow dispatched after npm canary publication. **Current behavior** Every canary dispatch builds both architectures, even when master has already published the same source. This consumes runner capacity and delays the image channel update. **Proposed behavior** Resolve the full-SHA master image. Verify its platform index, digest and signed source identity. Retag that immutable digest. Build normally when the image does not exist. **Reason and benefit** Remove duplicate image builds while keeping missing-image recovery and the current npm channel check. A recent duplicate used about 25 runner-minutes. The saving depends on whether the master image exists. ## What Changed - Add a canary-only reuse job before the native matrix. - Require signed provenance from the canonical Docker workflow on master at the exact source and signer commit. Read the proof from GHCR. - Reuse the validated digest or build both platforms when the full-SHA image is absent. Stop on registry and verification errors. - Keep the serialized npm dist-tag promotion after either path. Keep other build lanes independent. - Add failure tests, register them in CI and update release documentation. ## Verification - Passed 603 tests: `node --test '.github/scripts/tests/*.test.mjs' scripts/standard-image-contract.test.mjs scripts/reuse-canary-image.test.mjs scripts/__tests__/release-verify-workflow.test.mjs scripts/cloud-source-verification.test.mjs`. - Passed `actionlint -shellcheck='' .github/workflows/docker.yml .github/workflows/pr-trusted.yml` and `git diff --check`. - Full actionlint reports the same six existing shellcheck findings as the unchanged master workflow. No new findings. - A published master image passed the exact provenance command. This was a read-only check. No images were retagged. - Gitleaks and manual diff inspection found no secrets or private references. - Attempted `pnpm -r typecheck`, `pnpm test:run` and `pnpm build`. They cannot start because this isolated checkout has no installed dependencies (`tsc` and `tsx` are absent). Repository CI passed on `3c46f0152d4db94a7f393bc79f466aa6c2c745e2` after the documented browser-test rerun. All checks are complete. The original failure and prior baseline occurrence remain recorded in the PR comments; its root cause remains unknown. ## Risks - The image can appear just before its attestation. Verification then fails closed. Rerun after the master producer completes. - Reuse preserves the first build's resolved dependencies and tool versions. It intentionally does not rebuild the same source to refresh floating inputs. - A skipped or unfinished master image still needs a full build. No speedup is claimed for that path. - End-to-end reuse and runner savings require observation after merge. No live publication was performed. ## Model Used OpenAI `gpt-6.1-sol`, through the Codex harness in T3 Code, with xhigh reasoning effort and shell/code tools. The context window size is not exposed by this runtime. ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above - [x] I have either (a) linked existing issues with `Fixes: #` / `Closes #` / `Refs #` OR (b) described the issue in-PR following the relevant issue template - [x] I have not referenced internal/instance-local Paperclip issues or links (only public GitHub `#NNN` / `github.com/paperclipai/paperclip` URLs) - [x] My branch name describes the change (e.g. `docs/...`, `fix/...`) and contains no internal Paperclip ticket id or instance-derived details - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [x] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [x] All Paperclip CI gates are green - [x] Greptile is 5/5 with no open P2s, recommendations, or follow-ups - [x] I will address all Greptile and reviewer comments before requesting merge Co-authored-by: Paperclip --- .../tests/docker-canary-promotion.test.mjs | 2 +- .github/workflows/docker.yml | 39 ++++++++-- .github/workflows/pr-trusted.yml | 2 +- doc/RELEASING.md | 23 +++++- scripts/reuse-canary-image.mjs | 34 +++++++++ scripts/reuse-canary-image.test.mjs | 75 +++++++++++++++++++ scripts/standard-image-contract.mjs | 6 +- scripts/standard-image-contract.test.mjs | 14 ++++ 8 files changed, 183 insertions(+), 12 deletions(-) create mode 100644 scripts/reuse-canary-image.mjs create mode 100644 scripts/reuse-canary-image.test.mjs diff --git a/.github/scripts/tests/docker-canary-promotion.test.mjs b/.github/scripts/tests/docker-canary-promotion.test.mjs index a9b0727465f..fe80e25168b 100644 --- a/.github/scripts/tests/docker-canary-promotion.test.mjs +++ b/.github/scripts/tests/docker-canary-promotion.test.mjs @@ -11,7 +11,7 @@ const script = job.split(" run: |\n")[1].split("\n").map(line => line.rep const sha = "a".repeat(40); test("canary promotion waits for the standard manifest and keeps its serialized channel", () => { - assert.match(job, /needs: \[merge-and-push\]/); + assert.match(job, /needs: \[reuse_canary_image, merge-and-push\]/); assert.match(job, /group: docker-canary-channel-promotion/); assert.match(job, /cancel-in-progress: false/); }); diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index ed52dfa700d..ec7c71000e5 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -33,6 +33,35 @@ concurrency: cancel-in-progress: false jobs: + reuse_canary_image: + if: github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && startsWith(github.ref, 'refs/tags/canary/v') + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + contents: read + packages: write + attestations: read + outputs: + reused: ${{ steps.reuse.outputs.reused }} + steps: + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + - name: Setup Node.js + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 + with: + node-version: 24 + - name: Login to GitHub Container Registry + uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f + with: + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Reuse the verified master image when present + id: reuse + env: + GH_TOKEN: ${{ github.token }} + run: node scripts/reuse-canary-image.mjs + # Multi-arch by native runner, not QEMU. # # This was one job building linux/amd64,linux/arm64 together on an x86 @@ -56,6 +85,8 @@ jobs: # digest; `merge` assembles the manifest list. Both architectures remain # available to self-hosted installations and downstream image composers. build-and-push: + needs: [reuse_canary_image] + if: ${{ !cancelled() && (needs.reuse_canary_image.result == 'skipped' || (needs.reuse_canary_image.result == 'success' && needs.reuse_canary_image.outputs.reused == 'false')) }} strategy: # Independent legs: one architecture failing should still publish # nothing, but it must not also hide the other's logs behind a @@ -277,6 +308,7 @@ jobs: # tags. Nothing is publicly tagged until this runs, so a half-published # multi-arch image is not a state anything can pull. merge-and-push: + if: ${{ !cancelled() && needs.build-and-push.result == 'success' }} needs: build-and-push runs-on: ubuntu-latest timeout-minutes: 20 @@ -407,11 +439,8 @@ jobs: # promotion is gated on its own completed pushes). Every interleaving # converges the Docker channel onto the npm channel. promote_canary_channel: - if: startsWith(github.ref, 'refs/tags/canary/v') - # merge-and-push, not build-and-push: the per-arch legs push untagged - # digests, and the production `sha-*` tags this promotion retags only - # exist once the manifest merge has named them. - needs: [merge-and-push] + if: ${{ !cancelled() && startsWith(github.ref, 'refs/tags/canary/v') && (needs.merge-and-push.result == 'success' || (needs.reuse_canary_image.result == 'success' && needs.reuse_canary_image.outputs.reused == 'true')) }} + needs: [reuse_canary_image, merge-and-push] runs-on: ubuntu-latest timeout-minutes: 10 permissions: diff --git a/.github/workflows/pr-trusted.yml b/.github/workflows/pr-trusted.yml index b1b73c1f085..dae203d6216 100644 --- a/.github/workflows/pr-trusted.yml +++ b/.github/workflows/pr-trusted.yml @@ -202,7 +202,7 @@ jobs: run: node "$CI_NODE_TEST" ./scripts/__tests__/e2e-shard.test.mjs - name: Test release verify workflow wiring - run: node "$CI_NODE_TEST" ./scripts/__tests__/release-verify-workflow.test.mjs ./scripts/__tests__/eval-schedule-monitor.test.mjs ./scripts/__tests__/prepare-runner-release-image.test.mjs ./scripts/cloud-source-verification.test.mjs ./scripts/standard-image-contract.test.mjs + run: node "$CI_NODE_TEST" ./scripts/__tests__/release-verify-workflow.test.mjs ./scripts/__tests__/eval-schedule-monitor.test.mjs ./scripts/__tests__/prepare-runner-release-image.test.mjs ./scripts/cloud-source-verification.test.mjs ./scripts/standard-image-contract.test.mjs ./scripts/reuse-canary-image.test.mjs - name: Test standalone package build concurrency run: node "$CI_NODE_TEST" ./scripts/__tests__/build-standalone-concurrency.test.mjs diff --git a/doc/RELEASING.md b/doc/RELEASING.md index dbf40956a50..70e28fe798f 100644 --- a/doc/RELEASING.md +++ b/doc/RELEASING.md @@ -86,7 +86,7 @@ internal traceability tag. moves them to the versioned name - canaries, nightlies, and betas never create GitHub Releases - canaries, nightlies, and betas never require changelog generation -- Docker `:latest` moves only on stable releases; master builds publish +- Docker `:latest` moves only on stable releases; canary dispatches promote `:canary`, nightly builds `:nightly`, and beta builds `:beta` ## TL;DR @@ -310,12 +310,12 @@ The workflow: ## Docker Image Tags -[`docker.yml`](../.github/workflows/docker.yml) publishes both the self-hosted -image and the `-cloud` variant with the same lane mapping: +[`docker.yml`](../.github/workflows/docker.yml) publishes the standard image: | Build ref | Tags | | --- | --- | -| `master` push | `:canary`, `:sha-` | +| `master` push | `:sha-`, `:sha-` | +| `canary/v*` dispatch | `:sha-`, then `:canary` for the current npm canary | | `nightly/v*` tag | `:nightly`, `:sha-` | | `beta/v*` tag | `:beta`, `:sha-` | | `v*` tag (stable) | `:latest`, `:YYYY.MDD.P`, `:YYYY.MDD`, `:sha-` | @@ -325,6 +325,21 @@ suppresses push-triggered workflow runs for those pushes. The release jobs therefore dispatch `docker.yml` explicitly at the new tag ref; the tag mapping keys off `github.ref` either way. +Canary dispatches first look up the full-SHA master image. Reuse requires a +valid multi-platform index with both `linux/amd64` and `linux/arm64`, a matching +registry digest, and signed provenance from this repository's `docker.yml` +on `master` at the exact source commit. The image is retagged by immutable +digest. This preserves the original build inputs and skips both native builds +and the manifest merge. A missing full-SHA image runs the normal build instead; +registry errors or invalid provenance fail the run. Rerun after the master +image's attestation finishes if the image became visible before its proof. + +Both paths retain the serialized promotion that resolves the current npm +`canary` dist-tag before moving Docker `:canary`. Nightly, beta, stable, and +master builds retain their existing build paths. Downstream image builds can +continue to consume the signed full-SHA master image independently of npm +canary publication. + The Docker build matrix and the release preview image builder configure BuildKit to check `mirror.gcr.io` for Docker Hub images first. This covers both the Dockerfile frontend and base images. BuildKit keeps Docker Hub as diff --git a/scripts/reuse-canary-image.mjs b/scripts/reuse-canary-image.mjs new file mode 100644 index 00000000000..3a43524dd0b --- /dev/null +++ b/scripts/reuse-canary-image.mjs @@ -0,0 +1,34 @@ +import assert from "node:assert/strict"; +import { execFileSync } from "node:child_process"; +import { appendFile } from "node:fs/promises"; +import { pathToFileURL } from "node:url"; +import { resolveStandardImageDigest } from "./standard-image-contract.mjs"; + +export async function reuseCanaryImage(sha, { + resolve = source => resolveStandardImageDigest(source, fetch, { missing: "return-null" }), + run = (command, args) => execFileSync(command, args, { stdio: "inherit" }), +} = {}) { + assert.match(sha ?? "", /^[a-f0-9]{40}$/); + const digest = await resolve(sha); + if (digest === null) return false; + assert.match(digest, /^sha256:[a-f0-9]{64}$/); + const image = `ghcr.io/paperclipai/paperclip@${digest}`; + await run("gh", ["attestation", "verify", `oci://${image}`, + "--repo", "paperclipai/paperclip", "--bundle-from-oci", + "--signer-workflow", "paperclipai/paperclip/.github/workflows/docker.yml", + "--source-ref", "refs/heads/master", "--source-digest", sha, + "--signer-digest", sha, "--deny-self-hosted-runners"]); + await run("docker", ["buildx", "imagetools", "create", "-t", + `ghcr.io/paperclipai/paperclip:sha-${sha.slice(0, 7)}`, image]); + return true; +} + +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + const reused = await reuseCanaryImage(process.env.GITHUB_SHA); + if (process.env.GITHUB_OUTPUT) await appendFile(process.env.GITHUB_OUTPUT, `reused=${reused}\n`); + const summary = reused + ? "Reused the signed multi-platform master image. Skipped both image builds and the manifest merge." + : "No full-SHA master image exists. Building both platforms for this canary."; + console.log(summary); + if (process.env.GITHUB_STEP_SUMMARY) await appendFile(process.env.GITHUB_STEP_SUMMARY, `${summary}\n`); +} diff --git a/scripts/reuse-canary-image.test.mjs b/scripts/reuse-canary-image.test.mjs new file mode 100644 index 00000000000..26dfbc83c5f --- /dev/null +++ b/scripts/reuse-canary-image.test.mjs @@ -0,0 +1,75 @@ +import assert from "node:assert/strict"; +import { readFileSync } from "node:fs"; +import test from "node:test"; +import { reuseCanaryImage } from "./reuse-canary-image.mjs"; + +const sha = "a".repeat(40); +const digest = `sha256:${"b".repeat(64)}`; + +test("reuses only an exact-source, master-signed multi-platform image by digest", async () => { + const calls = []; + const reused = await reuseCanaryImage(sha, { + resolve: async source => { assert.equal(source, sha); return digest; }, + run: async (...args) => calls.push(args), + }); + assert.equal(reused, true); + assert.deepEqual(calls, [ + ["gh", ["attestation", "verify", `oci://ghcr.io/paperclipai/paperclip@${digest}`, + "--repo", "paperclipai/paperclip", "--bundle-from-oci", + "--signer-workflow", "paperclipai/paperclip/.github/workflows/docker.yml", + "--source-ref", "refs/heads/master", "--source-digest", sha, + "--signer-digest", sha, "--deny-self-hosted-runners"]], + ["docker", ["buildx", "imagetools", "create", "-t", + "ghcr.io/paperclipai/paperclip:sha-aaaaaaa", `ghcr.io/paperclipai/paperclip@${digest}`]], + ]); +}); + +test("a missing master image enables the fallback without retagging", async () => { + assert.equal(await reuseCanaryImage(sha, { + resolve: async () => null, + run: () => assert.fail("A missing image must not be promoted"), + }), false); +}); + +test("registry and provenance failures stop promotion rather than bypass verification", async () => { + await assert.rejects(reuseCanaryImage(sha, { + resolve: async () => { throw new Error("HTTP 403"); }, + run: () => assert.fail("A registry failure must not be promoted"), + }), /HTTP 403/); + let commands = 0; + await assert.rejects(reuseCanaryImage(sha, { + resolve: async () => digest, + run: command => { + assert.equal(command, "gh"); + commands++; + throw new Error("Untrusted source or signature"); + }, + }), /Untrusted source/); + assert.equal(commands, 1); +}); + +test("invalid source refs and digests cannot reach external commands", async () => { + for (const invalid of ["master", "a".repeat(7), "a".repeat(40) + "\n"]) { + await assert.rejects(reuseCanaryImage(invalid, { + resolve: () => assert.fail("Invalid source accepted"), + })); + } + await assert.rejects(reuseCanaryImage(sha, { + resolve: async () => "mutable:tag", + run: () => assert.fail("Invalid digest accepted"), + })); +}); + +test("canary reuse gates both native builds and still runs convergent promotion", () => { + const workflow = readFileSync(new URL("../.github/workflows/docker.yml", import.meta.url), "utf8"); + const reuse = workflow.split(" reuse_canary_image:\n")[1].split(" build-and-push:\n")[0]; + assert.match(reuse, /github.repository_id == '1170821064'.*startsWith\(github.ref, 'refs\/tags\/canary\/v'\)/); + assert.match(reuse, /node scripts\/reuse-canary-image.mjs/); + const build = workflow.split(" build-and-push:\n")[1].split(" merge-and-push:\n")[0]; + assert.match(build, /needs: \[reuse_canary_image\]/); + assert.match(build, /!cancelled\(\).*result == 'skipped'.*result == 'success'.*outputs.reused == 'false'/); + assert.match(build, /platform: linux\/amd64/); + assert.match(build, /platform: linux\/arm64/); + const promotion = workflow.split(" promote_canary_channel:\n")[1]; + assert.match(promotion, /!cancelled\(\).*needs.merge-and-push.result == 'success'.*outputs.reused == 'true'/); +}); diff --git a/scripts/standard-image-contract.mjs b/scripts/standard-image-contract.mjs index 62a8249e2da..5e0ae3cc24c 100644 --- a/scripts/standard-image-contract.mjs +++ b/scripts/standard-image-contract.mjs @@ -21,6 +21,7 @@ export function standardImageDigest(bytes) { /** Read registry bytes directly; CLI stdout formatting is not artifact content. */ export async function resolveStandardImageDigest(sha, fetchImpl = fetch, { sleep = ms => new Promise(resolve => setTimeout(resolve, ms)), + missing = "error", } = {}) { assert.match(sha ?? "", /^[a-f0-9]{40}$/); const request = async (url, headers, consume, allowNotFound = false) => { @@ -35,6 +36,7 @@ export async function resolveStandardImageDigest(sha, fetchImpl = fetch, { } if (!response.ok) { await response.body?.cancel(); + if (allowNotFound && response.status === 404 && missing === "return-null") return null; const transient = [408, 425, 429].includes(response.status) || response.status >= 500 || (allowNotFound && response.status === 404); assert.ok(transient && attempt < 5, `Registry lookup failed: HTTP ${response.status}`); const retryAfter = Number(response.headers.get("retry-after")); @@ -53,7 +55,7 @@ export async function resolveStandardImageDigest(sha, fetchImpl = fetch, { }; const { token } = await request("https://ghcr.io/token?service=ghcr.io&scope=repository:paperclipai/paperclip:pull", {}, response => response.json()); assert.ok(typeof token === "string" && token, "Missing public pull token"); - const { raw, expected } = await request(`https://ghcr.io/v2/paperclipai/paperclip/manifests/sha-${sha}`, { + const manifest = await request(`https://ghcr.io/v2/paperclipai/paperclip/manifests/sha-${sha}`, { authorization: `Bearer ${token}`, accept: "application/vnd.oci.image.index.v1+json, application/vnd.docker.distribution.manifest.list.v2+json", }, async response => { assert.ok(response.body, "Standard image index has no body"); @@ -71,6 +73,8 @@ export async function resolveStandardImageDigest(sha, fetchImpl = fetch, { } finally { await reader.cancel(); } return { raw: Buffer.concat(chunks), expected: response.headers.get("docker-content-digest") }; }, true); + if (manifest === null) return null; + const { raw, expected } = manifest; const digest = standardImageDigest(raw); assert.equal(digest, expected, "Registry digest does not match the index bytes"); return digest; diff --git a/scripts/standard-image-contract.test.mjs b/scripts/standard-image-contract.test.mjs index af2d4462977..d61f3fb862d 100644 --- a/scripts/standard-image-contract.test.mjs +++ b/scripts/standard-image-contract.test.mjs @@ -77,3 +77,17 @@ test("permanent registry errors fail immediately and transient errors exhaust a assert.equal(calls, expectedCalls); } }); + +test("optional lookup falls back only for a missing manifest", async () => { + for (const status of [401, 403, 404, 503]) { + let calls = 0; + const lookup = resolveStandardImageDigest("a".repeat(40), async url => { + if (url.includes("/token?")) return Response.json({ token: "public-fixture" }); + calls++; + return new Response(null, { status }); + }, { missing: "return-null", sleep: async () => {} }); + if (status === 404) assert.equal(await lookup, null); + else await assert.rejects(lookup, /Registry lookup failed/); + assert.equal(calls, status === 503 ? 5 : 1); + } +}); From 0fe6e038c2e3541b2ad14192b312f054b45b9665 Mon Sep 17 00:00:00 2001 From: Dotta Date: Fri, 30 Oct 2026 18:58:39 -0500 Subject: [PATCH 54/70] Restore independent recovery duties and bound committed-source work --- doc/plans/2026-10-30-issue-recovery-sleep.md | 8 +- .../src/__tests__/company-portability.test.ts | 1 + .../issue-recovery-deadlines.test.ts | 111 ++++++++++++++---- .../issue-recovery-work-lifecycle.test.ts | 68 +++++++++-- .../services/delivery-work-notifications.ts | 2 + server/src/services/heartbeat.ts | 4 +- .../services/issue-recovery-notifications.ts | 4 +- server/src/services/issues.ts | 43 +++---- .../native-workspace-finalizer.test.ts | 10 +- server/src/services/recovery/service.ts | 32 +++-- server/src/services/recovery/work.ts | 70 +++++++++-- server/src/services/scoped-deadline-work.ts | 20 +++- server/src/services/task-watchdog-work.ts | 25 +++- 13 files changed, 312 insertions(+), 86 deletions(-) diff --git a/doc/plans/2026-10-30-issue-recovery-sleep.md b/doc/plans/2026-10-30-issue-recovery-sleep.md index 18670cc66b8..f191d5dca42 100644 --- a/doc/plans/2026-10-30-issue-recovery-sleep.md +++ b/doc/plans/2026-10-30-issue-recovery-sleep.md @@ -1,14 +1,18 @@ # Issue recovery without periodic scans -Four existing owners reconstruct their current obligations once at startup. Committing source writes invalidate only their company. The coordinator keeps scopes, per-issue deadlines, and per-entry failures in memory. Its pending check does not query the database. +The issue recovery owners reconstruct their current obligations once at startup. Committing source writes invalidate only their company. The coordinator keeps scopes, per-issue deadlines, and per-entry failures in memory. Its pending check does not query the database. | Owner | Durable truth and producer boundary | Deadline or event | Exit | | --- | --- | --- | --- | | Stranded assignment | Visible assigned todo/in-progress/review issues; issue create/update/checkout/release; run lifecycle and execution control; agent/company/budget/configuration; interaction, approval and subtree hold changes | A committed eligibility change. A transient continuation uses the latest finish plus the existing exponential delay. | An execution path, durable wait, escalation, or current ineligibility. | +| Orphan blocker assignment | Agent-created unassigned todo/blocked blockers with a nonterminal dependent; issue, relation, agent and company changes | Once per changed company, independent of assignment candidates. | Assigned, no longer blocking, or creator cannot execute until its next eligibility change. | +| Active recovery actions | Active/escalated action records, including actions on completed sources; action, issue, run and wait changes | Once per changed company, independent of assignment candidates. | Resolved, restored path, explicit hold or a separately owned retry. | | Resolved dependency | Blocked assigned issues and company-scoped blocker relations; blocker status or relation changes through issue service and import; run/control and pause gate changes | The committing change. A failed enqueue retains that issue with error backoff. | Existing idempotent wake, live path, unresolved dependency or deliberate self-block. | | Task watchdog | Active issue watchdogs and their current source subtree; normal watchdog upsert/disable; issue/comment/document/work-product and wait changes | Exact source creation plus the existing 15-second first-run grace. An incomplete scan retains the watchdog with error backoff. | Live path, open/reviewed reusable review, disabled configuration, or ineligible source. | | Stale issue lock | Existing checkout/execution run references; issue and run lifecycle changes | A committing ownership change, or the fresh owning controller's saved lease expiry. | No stale reference, a valid owner, or the existing guarded terminalization and lock clear. | +Reconstruction uses keyset pages of at most100rows. Repeated company hints coalesce reconstruction for30seconds. Already-known entry deadlines preempt that spacing without an early probe on each comment. Scheduling suppression is read once per pass. A failed read keeps scoped backoff debt. The two subsidiary company repairs do not run inside the per-issue loop. Legacy direct aggregate API calls retain both repairs. + The owner methods accept company and issue/watchdog identity. A narrowed lock check still reads other same-company references before using terminal-issue authority. A stale terminal alias cannot end an active issue's shared run. The controller lease owner supplies its fresh expiry; the existing write predicate revalidates the observed boot ID and database-clock expiry. Producer coverage uses the existing run-dispatch, execution-control, issue-review-state, native status reassessment and external conversation settlement boundaries. Direct gaps have explicit signals inside their owning transaction: document writes/import, attachment removal, blocker import, recovery-action adoption/mutation, conversation pause release, Telegram failure comment replacement, native artifact title/metadata updates and company deletion. Metadata-only comment classification that does not change timestamp, visibility or execution eligibility needs no additional watchdog activation. Approval links and decisions use their existing issue-review/continuation effects. Work-product create/update/delete/import uses its existing native status boundary. @@ -16,3 +20,5 @@ Producer coverage uses the existing run-dispatch, execution-control, issue-revie The new owners replace only these four startup and periodic calls. Deferred comments, silent active runs and residual orphan/resource recovery retain their existing backstop until the next slice. Every previously migrated native-owner skip remains unchanged. A controlled process test prints an execution proof but cannot apply a task disposition. The test operator completes that task through issue service. If the existing continuation policy already saved a disposition-repair retry, the test preserves it until its stored deadline. Only after all provider work has joined does it advance the clock; dispatch then revalidates the completed task and cancels the retry. The final quiet proof requires no registered deadlines, no tracked in-flight work and no owner SQL over twenty former polling minutes. It does not claim a saved future retry is already drained. + +Active action repair has no separate grace or lease countdown. Its delayed disposition repair first writes a scheduled-retry run, then mirrors the date into action wakePolicy.retryAt and timeoutAt. The dispatch owner owns that saved run deadline. Other skipped gates wait for their existing execution, wait, pause, budget, or finalizer producer; the company action worker does not add a polling fallback. diff --git a/server/src/__tests__/company-portability.test.ts b/server/src/__tests__/company-portability.test.ts index fdf00b828af..f6dbbf552b6 100644 --- a/server/src/__tests__/company-portability.test.ts +++ b/server/src/__tests__/company-portability.test.ts @@ -4168,6 +4168,7 @@ describe("company portability", () => { const insertedRelationValues: Array> = []; const monitorUpdates: Array> = []; const db = { + transaction: async (write: (tx: unknown) => Promise) => write(db), insert: () => ({ values: (rows: Array>) => ({ onConflictDoNothing: async () => { diff --git a/server/src/__tests__/issue-recovery-deadlines.test.ts b/server/src/__tests__/issue-recovery-deadlines.test.ts index 4faac4ab535..b3d65f4dbcc 100644 --- a/server/src/__tests__/issue-recovery-deadlines.test.ts +++ b/server/src/__tests__/issue-recovery-deadlines.test.ts @@ -2,14 +2,18 @@ import { idleWorkSnapshot } from "../services/task-admission.js"; import { randomUUID } from "node:crypto"; import { and, eq, sql } from "drizzle-orm"; import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; -import { agents, authUsers, companies, companyMemberships, createDb, heartbeatRuns, issues, issueWatchdogs } from "@paperclipai/db"; +import { agents, authUsers, companies, companyMemberships, createDb, heartbeatRuns, issueRecoveryActions, issueRelations, issues, issueWatchdogs } from "@paperclipai/db"; import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js"; import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; import { heartbeatService } from "../services/heartbeat.js"; import { issueService } from "../services/issues.js"; +import { issueRecoveryActionService } from "../services/issue-recovery-actions.js"; +import { notifyIssueRecovery } from "../services/issue-recovery-notifications.js"; import { taskWatchdogService } from "../services/task-watchdogs.js"; import { createDeliveryWorkCoordinator } from "../services/delivery-work-coordinator.js"; +vi.setConfig({ testTimeout: 90_000 }); + const external = resolveExternalTestDatabaseUrl("PAPERCLIP_ISSUE_RECOVERY_TEST_DATABASE_URL"); const support = external ? { supported: true } : await getEmbeddedPostgresTestSupport(); describe.skipIf(!support.supported)("issue recovery production producers and deadlines", () => { @@ -81,13 +85,14 @@ describe.skipIf(!support.supported)("issue recovery production producers and dea expect(wake, detail).toBeNull(); expect(extraDeadlines.map(next => next())).toEqual(extraDeadlines.map(() => null)); expect(idleWorkSnapshot().active).toBe(0); - }, { timeout: 10_000 }); + }, { timeout: 40_000 }); } async function quiet() { await settled(); const reads = vi.spyOn(db, "select"); const executes = vi.spyOn(db, "execute"); - vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"] }); - try { await vi.advanceTimersByTimeAsync(20 * 60_000); expect(reads).not.toHaveBeenCalled(); expect(executes).not.toHaveBeenCalled(); } + const selected = reads.mock.calls.length, executed = executes.mock.calls.length; + if (!vi.isFakeTimers()) vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"] }); + try { await vi.advanceTimersByTimeAsync(20 * 60_000); expect(reads).toHaveBeenCalledTimes(selected); expect(executes).toHaveBeenCalledTimes(executed); } finally { vi.useRealTimers(); reads.mockRestore(); executes.mockRestore(); } } async function finish(issueId: string) { @@ -95,12 +100,10 @@ describe.skipIf(!support.supported)("issue recovery production producers and dea await heartbeat.drainActiveRunExecutions(); const saved = await db.select().from(heartbeatRuns).where(and(eq(heartbeatRuns.status, "scheduled_retry"), sql`${heartbeatRuns.contextSnapshot}->>'issueId' = ${issueId}`)); + expect(await db.select().from(heartbeatRuns).where(sql`${heartbeatRuns.status} in ('queued', 'running')`)).toHaveLength(0); + if (!vi.isFakeTimers()) vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"] }); if (saved.length > 0) { const due = Math.max(...saved.map(run => run.scheduledRetryAt!.getTime())); - expect(await db.select().from(heartbeatRuns).where(and( - sql`${heartbeatRuns.status} in ('queued', 'running')`, - sql`${heartbeatRuns.contextSnapshot}->>'issueId' = ${issueId}`))).toHaveLength(0); - vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"] }); await vi.advanceTimersByTimeAsync(Math.max(0, due - Date.now()) + 1); await vi.waitFor(async () => { const remaining = await db.select().from(heartbeatRuns).where(and(eq(heartbeatRuns.status, "scheduled_retry"), @@ -120,20 +123,85 @@ describe.skipIf(!support.supported)("issue recovery production producers and dea const run = runs.find(row => row.status === "running" || row.status === "succeeded"); expect(run).toBeDefined(); runId = run!.id; }, { timeout: 15_000 }); - await heartbeat.drainActiveRunExecutions(); + await vi.waitFor(async () => { + await heartbeat.drainActiveRunExecutions(); + const [current] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, runId!)); + expect(current.status, JSON.stringify({ error: current.error, code: current.errorCode, stderr: current.stderrExcerpt })).toBe("succeeded"); + }, { timeout: 15_000 }); const [run] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, runId!)); - expect(run.status, JSON.stringify({ error: run.error, code: run.errorCode, stderr: run.stderrExcerpt, result: run.resultJson })).toBe("succeeded"); expect(run.stdoutExcerpt).toContain("recovery-proof"); + expect(run.stdoutExcerpt).toContain("recovery-proof"); return run; } it("keeps empty, configured idle, and terminal history quiet", async () => { await fixture(); await start(); await quiet(); }); + it("creates a blocking child through the ordinary root service transaction", async () => { + const f = await fixture(); + const parent = await issueService(db).create(f.companyId, { title: "Parent", status: "todo" }); + const child = await issueService(db).createChild(parent.id, { title: "Blocking child", status: "todo", blockParentUntilDone: true }); + expect(child.parentBlockerAdded).toBe(true); + expect(await db.select().from(issueRelations).where(and(eq(issueRelations.companyId, f.companyId), + eq(issueRelations.issueId, child.issue.id), eq(issueRelations.relatedIssueId, parent.id)))).toHaveLength(1); + }); + it.each(["startup", "committed"])("restores blocker-only and completed-action debt with no assignment candidates (%s)", async mode => { + const f = await fixture(); + if (mode === "committed") { await start(); await quiet(); } + const blocker = await issueService(db).create(f.companyId, { title: "Orphan blocker", status: "blocked", createdByAgentId: f.agentId }); + const dependent = await issueService(db).create(f.companyId, { title: "Blocked dependent", status: "blocked", blockedByIssueIds: [blocker.id] }); + const completedSource = await issueService(db).create(f.companyId, { title: "Completed source", status: "done" }); + const action = await issueRecoveryActionService(db).upsertSourceScoped({ companyId: f.companyId, sourceIssueId: completedSource.id, + kind: "stranded_assigned_issue", ownerType: "agent", ownerAgentId: f.agentId, cause: "stranded_assigned_issue", + fingerprint: "source-terminal", nextAction: "Restore work", wakePolicy: { type: "wake_owner" }, evidence: {} }); + if (mode === "startup") await start(); + await vi.waitFor(async () => { + expect((await db.select().from(issues).where(eq(issues.id, blocker.id)))[0].assigneeAgentId).toBe(f.agentId); + expect((await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.id, action.id)))[0].status).toBe("resolved"); + }, { timeout: 40_000 }); + await finish(blocker.id); await finish(dependent.id); await quiet(); + }); + it("bounds subsidiary scans for100 assignment candidates and coalesces repeated company writes", async () => { + const f = await fixture(), foreign = await fixture(); + await db.update(agents).set({ lifecycleState: "preparing" }).where(eq(agents.id, f.agentId)); + await db.insert(issues).values(Array.from({ length: 100 }, (_, index) => ({ companyId: f.companyId, + title: `Waiting setup ${index}`, status: "todo", assigneeAgentId: f.agentId }))); + const [source] = await db.insert(issues).values({ companyId: f.companyId, title: "Done source", status: "done" }).returning(); + const action = await issueRecoveryActionService(db).upsertSourceScoped({ companyId: f.companyId, sourceIssueId: source.id, + kind: "stranded_assigned_issue", ownerType: "agent", ownerAgentId: f.agentId, cause: "stranded_assigned_issue", + fingerprint: "query-budget", nextAction: "Restore work", wakePolicy: { type: "wake_owner" }, evidence: {} }); + const scans = { blockers: 0, actions: 0, loads: 0 }; + const originalSelect = db.select.bind(db); + vi.spyOn(db, "select").mockImplementation(((fields?: Record) => { + const query = originalSelect(fields as never); const from = query.from.bind(query); + query.from = ((table: unknown) => { + if (table === issues && fields?.id === issues.id && fields.companyId === issues.companyId && Object.keys(fields).length === 2) scans.loads++; + if (table === issueRelations && fields?.createdByAgentId === issues.createdByAgentId) scans.blockers++; + if (table === issueRecoveryActions && fields?.action === issueRecoveryActions) scans.actions++; + return from(table as never); + }) as typeof query.from; + return query; + }) as typeof db.select); + await start(); await settled(); + expect(scans.actions).toBe(1); expect(scans.blockers).toBe(0); expect(scans.loads).toBeLessThanOrEqual(8); + expect((await originalSelect().from(issueRecoveryActions).where(eq(issueRecoveryActions.id, action.id)))[0].status).toBe("resolved"); + const [foreignSource] = await db.insert(issues).values({ companyId: foreign.companyId, title: "Foreign done", status: "done" }).returning(); + const [foreignAction] = await db.insert(issueRecoveryActions).values({ companyId: foreign.companyId, sourceIssueId: foreignSource.id, + kind: "stranded_assigned_issue", ownerType: "board", cause: "stranded_assigned_issue", fingerprint: "foreign-query-budget", nextAction: "Inspect" }).returning(); + const before = { ...scans }; + for (let index = 0; index < 100; index++) await db.transaction(tx => notifyIssueRecovery(tx, { companyId: f.companyId })); + await new Promise(resolve => setTimeout(resolve, 100)); + expect(scans.loads).toBe(before.loads); + await settled(); + expect(scans.actions).toBe(before.actions); expect(scans.blockers).toBe(before.blockers); + expect(scans.loads).toBeLessThanOrEqual(before.loads + 4); + expect((await originalSelect().from(issueRecoveryActions).where(eq(issueRecoveryActions.id, foreignAction.id)))[0].status).toBe("active"); + expect(await originalSelect().from(heartbeatRuns)).toHaveLength(0); await quiet(); + }); it("dispatches an ordinary new assignment after commit and returns to quiet after completion", async () => { const f = await fixture(); await start(); await quiet(); const issue = await issueService(db).create(f.companyId, { title: "Recover assignment", status: "todo", assigneeAgentId: f.agentId }); await completed(issue.id); await finish(issue.id); await quiet(); - }, 45_000); + }, 90_000); it("reacts to blocker resolution once and dispatches the durable dependency wake", async () => { const f = await fixture(); const blocker = await issueService(db).create(f.companyId, { title: "Blocker", status: "todo" }); @@ -147,7 +215,7 @@ describe.skipIf(!support.supported)("issue recovery production producers and dea await finish(dependent.id); await quiet(); expect(await db.select().from(heartbeatRuns).where(and(eq(heartbeatRuns.companyId, f.companyId), sql`${heartbeatRuns.contextSnapshot}->>'wakeReason' = 'issue_blockers_resolved'`))).toHaveLength(1); - }, 45_000); + }, 90_000); it("uses ordinary watchdog configuration and subtree closure to dispatch a reusable review", async () => { const f = await fixture(); const source = await issueService(db).create(f.companyId, { title: "Finished subtree", status: "done" }); @@ -163,8 +231,9 @@ describe.skipIf(!support.supported)("issue recovery production producers and dea expect(await db.select().from(heartbeatRuns).where(and(eq(heartbeatRuns.companyId, f.companyId), sql`${heartbeatRuns.contextSnapshot}->>'wakeReason' = 'task_watchdog_stopped_subtree'`))).toHaveLength(1); await finish(reviewId!); - await taskWatchdogService(db).disableForIssue(f.companyId, source.id); await quiet(); - }, 45_000); + await taskWatchdogService(db).disableForIssue(f.companyId, source.id); + if (!vi.isFakeTimers()) vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"] }); await quiet(); + }, 90_000); it("bounds ordinary writes to their company and reconstructs foreign stale debt after restart", async () => { const selected = await fixture(); await start(); await quiet(); const foreign = await fixture(); @@ -198,19 +267,19 @@ describe.skipIf(!support.supported)("issue recovery production producers and dea })).rejects.toThrow("rollback control"); await quiet(); expect(await db.select().from(issues).where(eq(issues.title, "Nested rollback"))).toHaveLength(0); - }, 60_000); + }, 90_000); it("retains the first committed assignment during drain and executes on resume", async () => { const f = await fixture(); await start(); await quiet(); draining = true; const issue = await issueService(db).create(f.companyId, { title: "Drained assignment", status: "todo", assigneeAgentId: f.agentId }); await new Promise(resolve => setTimeout(resolve, 100)); const reads = vi.spyOn(db, "select"); - vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"] }); + if (!vi.isFakeTimers()) vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"] }); try { await vi.advanceTimersByTimeAsync(90_000); expect(reads).not.toHaveBeenCalled(); } finally { vi.useRealTimers(); reads.mockRestore(); } expect(await db.select().from(heartbeatRuns)).toHaveLength(0); draining = false; worker.wake(); dispatch.wake(); await completed(issue.id); await finish(issue.id); await quiet(); - }, 45_000); + }, 90_000); it("retains a fresh subtree's exact first-run grace then disarms on disable", async () => { const f = await fixture(); const source = await issueService(db).create(f.companyId, { title: "Fresh unassigned subtree", status: "todo" }); @@ -218,7 +287,8 @@ describe.skipIf(!support.supported)("issue recovery production producers and dea await taskWatchdogService(db).upsertForIssue(f.companyId, source.id, { agentId: f.agentId }); await vi.waitFor(() => expect(coordinator.nextWakeAt()).toBe(source.createdAt.getTime() + 15_000), { timeout: 5_000 }); expect(await db.select().from(issues).where(eq(issues.originKind, "task_watchdog"))).toHaveLength(0); - await taskWatchdogService(db).disableForIssue(f.companyId, source.id); await quiet(); + await taskWatchdogService(db).disableForIssue(f.companyId, source.id); + if (!vi.isFakeTimers()) vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"] }); await quiet(); }); it("waits for a live foreign controller's exact stored lease before clearing a terminal lock", async () => { const f = await fixture(); @@ -232,6 +302,7 @@ describe.skipIf(!support.supported)("issue recovery production producers and dea await vi.waitFor(() => expect(coordinator.nextWakeAt()).toBe(lease.getTime()), { timeout: 3_000 }); expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, run.id)))[0].status).toBe("running"); await vi.waitFor(async () => expect((await db.select().from(issues).where(eq(issues.id, source.id)))[0].executionRunId).toBeNull(), { timeout: 12_000 }); - expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, run.id)))[0].status).toBe("succeeded"); await quiet(); - }, 30_000); + expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, run.id)))[0].status).toBe("succeeded"); + if (!vi.isFakeTimers()) vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"] }); await quiet(); + }, 90_000); }); diff --git a/server/src/__tests__/issue-recovery-work-lifecycle.test.ts b/server/src/__tests__/issue-recovery-work-lifecycle.test.ts index 4d0647f4406..52f465daa8e 100644 --- a/server/src/__tests__/issue-recovery-work-lifecycle.test.ts +++ b/server/src/__tests__/issue-recovery-work-lifecycle.test.ts @@ -1,6 +1,6 @@ import { PgDialect } from "drizzle-orm/pg-core"; import { afterEach, describe, expect, it, vi } from "vitest"; -import { issueWatchdogs } from "@paperclipai/db"; +import { issueRecoveryActions, issueWatchdogs } from "@paperclipai/db"; import { installDatabaseWorkSignals } from "../../../packages/db/src/work-signals.js"; import { createDeliveryWorkCoordinator } from "../services/delivery-work-coordinator.js"; import { notifyIssueRecovery } from "../services/issue-recovery-notifications.js"; @@ -12,29 +12,34 @@ afterEach(async () => { for (const stop of stops.splice(0)) await stop(); vi.use function fixture() { vi.useFakeTimers(); vi.setSystemTime(1_000_000); const dialect = new PgDialect(); - const rows = { issues: [] as Array<{ id: string; companyId: string }>, watchdogs: [] as Array<{ id: string; companyId: string }> }; + const rows = { issues: [] as Array<{ id: string; companyId: string }>, watchdogs: [] as Array<{ id: string; companyId: string }>, blockers: [] as Array<{ id: string; companyId: string }>, actions: [] as Array<{ id: string; companyId: string }> }; const execute = vi.fn(async (query: any) => { if (dialect.sqlToQuery(query).sql.includes("pg_current_xact_id")) return [{ xid: "42" }]; throw new Error("unexpected SQL"); }); - const load = vi.fn((table: unknown) => table === issueWatchdogs ? rows.watchdogs : rows.issues); + const load = vi.fn((table: unknown, distinct = false) => distinct + ? table === issueRecoveryActions ? rows.actions : rows.blockers + : table === issueWatchdogs ? rows.watchdogs : rows.issues); + const query = (distinct = false) => ({ from: (table: unknown) => ({ where: () => ({ orderBy: () => ({ limit: async () => load(table, distinct) }) }) }) }); const db: any = installDatabaseWorkSignals({ execute, - select: () => ({ from: (table: unknown) => ({ where: async () => load(table) }) }), - transaction: async (callback: (tx: any) => Promise) => callback({ execute, select: db.select, transaction: db.transaction }), + select: () => query(), selectDistinct: () => query(true), + transaction: async (callback: (tx: any) => Promise) => callback({ execute, select: db.select, selectDistinct: db.selectDistinct, transaction: db.transaction }), }); const state = { draining: false, standby: false, suppressed: false, deadline: null as number | null, fail: false, incomplete: false }; const check = vi.fn(async (scope: { onDeferredDeadline(at: number): void }) => { if (state.fail) throw new Error("database unavailable"); if (state.deadline !== null && state.deadline > Date.now()) scope.onDeferredDeadline(state.deadline); }); - const owner = { stranded: check, dependencies: async (scope: any) => { await check(scope); return { enqueueFailed: 0 }; }, + const orphanBlockers = vi.fn(async () => {}), recoveryActions = vi.fn(async () => {}); + const suppression = vi.fn(async () => state.suppressed); + const owner = { orphanBlockers, recoveryActions, stranded: check, dependencies: async (scope: any) => { await check(scope); return { enqueueFailed: 0 }; }, locks: check, watchdogs: async (scope: any) => { await check(scope); return { incomplete: state.incomplete ? 1 : 0 }; }, - schedulingSuppressed: async () => state.suppressed }; + schedulingSuppressed: suppression }; const coordinator = createDeliveryWorkCoordinator({ owner: db, canRun: () => !state.standby, canReconcile: () => !state.standby, onError: vi.fn() }); const worker = registerIssueRecoveryWork(db, coordinator, owner, () => !state.draining && !state.standby); stops.push(() => coordinator.stop()); - return { db, rows, state, check, load, coordinator, worker, + return { db, rows, state, check, load, coordinator, worker, orphanBlockers, recoveryActions, suppression, notify: () => db.transaction((tx: any) => notifyIssueRecovery(tx, { companyId: "selected" })), settle: () => vi.advanceTimersByTimeAsync(1_000) }; } @@ -45,6 +50,51 @@ describe("issue recovery obligation lifecycle", () => { await vi.advanceTimersByTimeAsync(20 * 60_000); expect(f.load).toHaveBeenCalledTimes(loaded); expect(f.check).not.toHaveBeenCalled(); }); + it("reads scheduling suppression once per pass instead of once per issue", async () => { + const f = fixture(); + f.rows.issues.push(...Array.from({ length: 25 }, (_, index) => ({ id: `issue-${index}`, companyId: "selected" }))); + await f.worker.ready; + expect(f.check).toHaveBeenCalledTimes(75); + expect(f.suppression).toHaveBeenCalledTimes(6); + expect(f.coordinator.nextWakeAt()).toBeNull(); + }); + it("backs off a failed suppression read without probing an empty or failed owner repeatedly", async () => { + const f = fixture(); f.suppression.mockRejectedValueOnce(new Error("settings unavailable")); + await f.worker.ready; const reads = f.suppression.mock.calls.length; + expect(f.coordinator.nextWakeAt()).toBe(1_030_000); + await vi.advanceTimersByTimeAsync(29_999); expect(f.suppression).toHaveBeenCalledTimes(reads); + await vi.advanceTimersByTimeAsync(1); expect(f.suppression).toHaveBeenCalledTimes(reads + 1); + expect(f.coordinator.nextWakeAt()).toBeNull(); + }); + it("normalizes a wide scope with cyclic SQL objects before publishing work hints", async () => { + const f = fixture(); await f.worker.ready; + const wide = { companyId: "selected", endpointId: issueRecoveryActions.id, table: issueRecoveryActions }; + await f.db.transaction((tx: object) => notifyIssueRecovery(tx, wide)); await f.settle(); + expect(f.coordinator.nextWakeAt()).toBeNull(); + const loads = f.load.mock.calls.length; + await vi.advanceTimersByTimeAsync(20 * 60_000); expect(f.load).toHaveBeenCalledTimes(loads); + }); + it("runs orphan blocker and action duties without assignment entries", async () => { + const f = fixture(); f.rows.blockers.push({ id: "selected", companyId: "selected" }); + f.rows.actions.push({ id: "selected", companyId: "selected" }); await f.worker.ready; + expect(f.check).not.toHaveBeenCalled(); + expect(f.orphanBlockers).toHaveBeenCalledExactlyOnceWith({ companyId: "selected" }); + expect(f.recoveryActions).toHaveBeenCalledExactlyOnceWith({ companyId: "selected" }); + expect(f.coordinator.nextWakeAt()).toBeNull(); + }); + it("coalesces repeated company reconstruction without delaying an existing lease deadline", async () => { + const f = fixture(); f.rows.issues.push({ id: "issue", companyId: "selected" }); + f.state.deadline = 1_015_000; await f.worker.ready; + await f.notify(); await f.settle(); const loaded = f.load.mock.calls.length, checked = f.check.mock.calls.length; + for (let index = 0; index < 100; index++) { await f.notify(); await vi.advanceTimersByTimeAsync(10); } + expect(f.load).toHaveBeenCalledTimes(loaded); expect(f.check).toHaveBeenCalledTimes(checked); + await vi.advanceTimersByTimeAsync(13_000); + expect(f.state.deadline).toBe(1_015_000); + expect(f.coordinator.nextWakeAt()).toBe(1_031_000); + await vi.advanceTimersByTimeAsync(16_000); + expect(f.load.mock.calls.length).toBeGreaterThan(loaded); + expect(f.coordinator.nextWakeAt()).toBeNull(); + }); it("retains exact grace and lease deadlines without early owner probes", async () => { const f = fixture(); f.rows.issues.push({ id: "issue", companyId: "selected" }); f.rows.watchdogs.push({ id: "watchdog", companyId: "selected" }); f.state.deadline = 1_015_000; @@ -85,7 +135,7 @@ describe("issue recovery obligation lifecycle", () => { const f = fixture(); f.rows.issues.push({ id: "issue", companyId: "selected" }); await f.worker.ready; let release!: () => void; const held = new Promise(resolve => { release = resolve; }); f.check.mockImplementationOnce(() => held); - await f.notify(); await f.settle(); await f.notify(); release(); await f.settle(); await f.settle(); + await f.notify(); await f.settle(); await f.notify(); release(); await f.settle(); await vi.advanceTimersByTimeAsync(30_000); expect(f.coordinator.nextWakeAt()).toBeNull(); expect(f.check.mock.calls.length).toBeGreaterThan(3); }); diff --git a/server/src/services/delivery-work-notifications.ts b/server/src/services/delivery-work-notifications.ts index 1d9ac79f520..8a9fdbe5686 100644 --- a/server/src/services/delivery-work-notifications.ts +++ b/server/src/services/delivery-work-notifications.ts @@ -2,6 +2,8 @@ import { signalDatabaseWork, subscribeDatabaseWork } from "@paperclipai/db"; /** Topics for existing durable queues; register intent before their writes. */ export const DELIVERY_QUEUES = { + orphanBlockers: "orphan-blockers", + activeRecoveryActions: "active-recovery-actions", strandedAssignments: "stranded-assignments", resolvedDependencies: "resolved-dependencies", taskWatchdogs: "task-watchdogs", diff --git a/server/src/services/heartbeat.ts b/server/src/services/heartbeat.ts index 42dc9928ad5..068532357ea 100644 --- a/server/src/services/heartbeat.ts +++ b/server/src/services/heartbeat.ts @@ -4245,7 +4245,9 @@ export function heartbeatService( registerIssueStateRecoveryWork: (coordinator: Parameters[1], canRun: () => boolean) => registerIssueRecoveryWork(db, coordinator, { - stranded: scope => reconcileStrandedAssignedIssues({ ...scope, skipExternalConversationSettlement: true }), + stranded: scope => reconcileStrandedAssignedIssues({ ...scope, skipExternalConversationSettlement: true, skipRelatedRecovery: true }), + orphanBlockers: recovery.reconcileUnassignedBlockingIssues, + recoveryActions: recovery.reconcileActiveRecoveryActions, dependencies: reconcileResolvedDependencyWakes, locks: sweepStaleIssueLocks, watchdogs: reconcileTaskWatchdogs, diff --git a/server/src/services/issue-recovery-notifications.ts b/server/src/services/issue-recovery-notifications.ts index db04ca801ba..3c974228d86 100644 --- a/server/src/services/issue-recovery-notifications.ts +++ b/server/src/services/issue-recovery-notifications.ts @@ -3,8 +3,8 @@ import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notificatio export type IssueRecoveryScope = { companyId: string }; export async function notifyIssueRecovery(transaction: object, scope?: IssueRecoveryScope) { - const hint = scope ? JSON.stringify(scope) : "*"; - for (const queue of [DELIVERY_QUEUES.strandedAssignments, DELIVERY_QUEUES.resolvedDependencies, + const hint = scope ? JSON.stringify({ companyId: scope.companyId }) : "*"; + for (const queue of [DELIVERY_QUEUES.orphanBlockers, DELIVERY_QUEUES.activeRecoveryActions, DELIVERY_QUEUES.strandedAssignments, DELIVERY_QUEUES.resolvedDependencies, DELIVERY_QUEUES.taskWatchdogs, DELIVERY_QUEUES.staleIssueLocks]) { await notifyDeliveryWork(transaction, queue, hint); } diff --git a/server/src/services/issues.ts b/server/src/services/issues.ts index 5a569f4ce1e..34234b01c98 100644 --- a/server/src/services/issues.ts +++ b/server/src/services/issues.ts @@ -9677,27 +9677,30 @@ export function issueService(db: Db) { }); if (blockParentUntilDone) { - const existingBlockers = await db - .select({ blockerIssueId: issueRelations.issueId }) - .from(issueRelations) - .where( - and( - eq(issueRelations.companyId, parent.companyId), - eq(issueRelations.relatedIssueId, parent.id), - eq(issueRelations.type, "blocks"), - ), + await db.transaction(async tx => { + const existingBlockers = await tx + .select({ blockerIssueId: issueRelations.issueId }) + .from(issueRelations) + .where( + and( + eq(issueRelations.companyId, parent.companyId), + eq(issueRelations.relatedIssueId, parent.id), + eq(issueRelations.type, "blocks"), + ), + ); + await syncBlockedByIssueIds( + parent.id, + parent.companyId, + [ + ...new Set([ + ...existingBlockers.map((row) => row.blockerIssueId), + child.id, + ]), + ], + { agentId: actorAgentId ?? null, userId: actorUserId ?? null }, + tx, ); - await syncBlockedByIssueIds( - parent.id, - parent.companyId, - [ - ...new Set([ - ...existingBlockers.map((row) => row.blockerIssueId), - child.id, - ]), - ], - { agentId: actorAgentId ?? null, userId: actorUserId ?? null }, - ); + }); [child] = await withIssueRelationSummaries( parent.companyId, [child], diff --git a/server/src/services/native-runtime/native-workspace-finalizer.test.ts b/server/src/services/native-runtime/native-workspace-finalizer.test.ts index 59270a2bb65..53063f345e9 100644 --- a/server/src/services/native-runtime/native-workspace-finalizer.test.ts +++ b/server/src/services/native-runtime/native-workspace-finalizer.test.ts @@ -131,7 +131,7 @@ describe("native workspace finalization recovery", () => { const external = resolveExternalTestDatabaseUrl("PAPERCLIP_NATIVE_WORKSPACE_FINALIZER_TEST_DATABASE_URL"); if (!external) temporary = await startEmbeddedPostgresTestDatabase("paperclip-native-workspace-finalizer-"); connectionString = external ?? temporary!.connectionString; - db = createDb(connectionString); + db = createDb(connectionString, { maxConnections: 6 }); await db.insert(companies).values([ { id: companyId, name: "Native workspace owner", issuePrefix: "NWO" }, @@ -452,7 +452,7 @@ describe("native workspace finalization recovery", () => { const seeded = await seedRun({ executionWorkspaceId: randomUUID(), title: "Live copyback" }); const live = await withNativeWorkspaceFinalizationOwnership({ db, companyId, runId: seeded.runId }, async (ownership) => { await ownership.assertHeld(); - expect(await resumeNativeWorkspaceFinalization({ db: createDb(connectionString), runId: seeded.runId })).toBeNull(); + expect(await resumeNativeWorkspaceFinalization({ db: createDb(connectionString, { maxConnections: 6 }), runId: seeded.runId })).toBeNull(); expect(await db.select().from(workspaceOperations).where(eq(workspaceOperations.heartbeatRunId, seeded.runId))).toEqual([]); return "copied"; }); @@ -476,7 +476,7 @@ describe("native workspace finalization recovery", () => { try { await entered; await db.execute(sql`select pg_terminate_backend(pid) from pg_locks where locktype = 'advisory' and database = (select oid from pg_database where datname = current_database()) and pid <> pg_backend_pid()`); - const second = await withNativeWorkspaceFinalizationOwnership({ db: createDb(connectionString), companyId, runId: seeded.runId }, async () => "must not run"); + const second = await withNativeWorkspaceFinalizationOwnership({ db: createDb(connectionString, { maxConnections: 6 }), companyId, runId: seeded.runId }, async () => "must not run"); expect(second).toEqual({ acquired: false, reason: "busy" }); const [held] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, seeded.runId)); expect(held.runnerProfileJson?.nativeWorkspaceFinalizationOwner).toBeTruthy(); @@ -489,7 +489,7 @@ describe("native workspace finalization recovery", () => { it("recovers its own joined copyback receipt after the application pool disconnects during cleanup", async () => { const seeded = await seedRun({ executionWorkspaceId: randomUUID(), title: "Reconnect joined copyback" }); - const interruptedDb = createDb(connectionString); + const interruptedDb = createDb(connectionString, { maxConnections: 6 }); let joined = false; await expect(withNativeWorkspaceFinalizationOwnership({ db: interruptedDb, companyId, runId: seeded.runId }, async () => { await fs.writeFile(path.join(workspaceRoot, "joined-copyback.txt"), "durable work"); @@ -507,7 +507,7 @@ describe("native workspace finalization recovery", () => { .toEqual({ acquired: false, reason: "owner_unverified" }); } await db.update(heartbeatRuns).set({ runnerProfileJson: beforeReconnect.runnerProfileJson }).where(eq(heartbeatRuns.id, seeded.runId)); - expect(await withNativeWorkspaceFinalizationOwnership({ db: createDb(connectionString), companyId, runId: seeded.runId }, async () => "resumed")) + expect(await withNativeWorkspaceFinalizationOwnership({ db: createDb(connectionString, { maxConnections: 6 }), companyId, runId: seeded.runId }, async () => "resumed")) .toEqual({ acquired: true, value: "resumed" }); }); diff --git a/server/src/services/recovery/service.ts b/server/src/services/recovery/service.ts index 7888b3e23fb..4b2ec5c0672 100644 --- a/server/src/services/recovery/service.ts +++ b/server/src/services/recovery/service.ts @@ -2203,7 +2203,7 @@ export function recoveryService( return Boolean(budgetBlock); } - async function reconcileUnassignedBlockingIssues() { + async function reconcileUnassignedBlockingIssues(opts?: { companyId?: string }) { const candidates = await db .select({ id: issues.id, @@ -2217,6 +2217,8 @@ export function recoveryService( .where( and( eq(issueRelations.type, "blocks"), + eq(issueRelations.companyId, issues.companyId), + opts?.companyId ? eq(issues.companyId, opts.companyId) : undefined, inArray(issues.status, ["todo", "blocked"]), isNull(issues.assigneeAgentId), isNull(issues.assigneeUserId), @@ -3501,7 +3503,7 @@ export function recoveryService( return Boolean(run || wake); } - async function reconcileActiveRecoveryActions() { + async function reconcileActiveRecoveryActions(opts?: { companyId?: string }) { const rows = await db .select({ action: issueRecoveryActions, @@ -3515,7 +3517,8 @@ export function recoveryService( eq(issues.companyId, issueRecoveryActions.companyId), ), ) - .where(inArray(issueRecoveryActions.status, ["active", "escalated"])); + .where(and(inArray(issueRecoveryActions.status, ["active", "escalated"]), + opts?.companyId ? eq(issueRecoveryActions.companyId, opts.companyId) : undefined)); const result = { requeued: 0, @@ -4532,6 +4535,7 @@ export function recoveryService( onDeferredDeadline?: (at: number) => void; issueCreatedAtGte?: Date | null; skipExternalConversationSettlement?: boolean; + skipRelatedRecovery?: boolean; }) { const candidates = await db .select(recoveryIssueFields) @@ -5780,16 +5784,18 @@ export function recoveryService( } } - const orphanBlockerRecovery = await reconcileUnassignedBlockingIssues(); - result.orphanBlockersAssigned = orphanBlockerRecovery.assigned; - result.skipped += orphanBlockerRecovery.skipped; - result.issueIds.push(...orphanBlockerRecovery.issueIds); + if (!opts?.skipRelatedRecovery) { + const orphanBlockerRecovery = await reconcileUnassignedBlockingIssues(opts); + result.orphanBlockersAssigned = orphanBlockerRecovery.assigned; + result.skipped += orphanBlockerRecovery.skipped; + result.issueIds.push(...orphanBlockerRecovery.issueIds); - const activeRecovery = await reconcileActiveRecoveryActions(); - result.continuationRequeued += activeRecovery.requeued; - result.escalated += activeRecovery.escalated; - result.skipped += activeRecovery.skipped; - result.issueIds.push(...activeRecovery.issueIds); + const activeRecovery = await reconcileActiveRecoveryActions(opts); + result.continuationRequeued += activeRecovery.requeued; + result.escalated += activeRecovery.escalated; + result.skipped += activeRecovery.skipped; + result.issueIds.push(...activeRecovery.issueIds); + } result.issueIds = [...new Set(result.issueIds)]; return result; @@ -6563,6 +6569,8 @@ export function recoveryService( recordWatchdogDecision, scanSilentActiveRuns, reconcileStrandedAssignedIssues, + reconcileUnassignedBlockingIssues, + reconcileActiveRecoveryActions, reconcileLegacyContinuation, legacyRepairDispatchBlock, sweepStaleIssueLocks, diff --git a/server/src/services/recovery/work.ts b/server/src/services/recovery/work.ts index 93679e80423..8bf7b75c7f4 100644 --- a/server/src/services/recovery/work.ts +++ b/server/src/services/recovery/work.ts @@ -1,6 +1,6 @@ import { registerTaskWatchdogWork } from "../task-watchdog-work.js"; -import { and, eq, inArray, isNotNull, isNull, or } from "drizzle-orm"; -import { issues, type Db } from "@paperclipai/db"; +import { and, asc, eq, gt, inArray, isNotNull, isNull, or, sql } from "drizzle-orm"; +import { issueRecoveryActions, issueRelations, issues, type Db } from "@paperclipai/db"; import type { createDeliveryWorkCoordinator } from "../delivery-work-coordinator.js"; import { DELIVERY_QUEUES, type DeliveryQueue } from "../delivery-work-notifications.js"; import type { IssueRecoveryScope } from "../issue-recovery-notifications.js"; @@ -8,12 +8,15 @@ import { registerScopedDeadlineWork } from "../scoped-deadline-work.js"; type Entry = { id: string; companyId: string; dueAt: number }; type Scope = { companyId: string; issueId: string; onDeferredDeadline: (at: number) => void }; +const PAGE_SIZE = 100; export function registerIssueRecoveryWork( db: Db, coordinator: ReturnType, owner: { stranded(scope: Scope): Promise; + orphanBlockers(scope: { companyId: string }): Promise; + recoveryActions(scope: { companyId: string }): Promise; dependencies(scope: Scope): Promise<{ enqueueFailed: number }>; locks(scope: Scope): Promise; watchdogs(scope: { companyId: string; watchdogId: string; onDeferredDeadline: (at: number) => void }): Promise<{ incomplete: number }>; @@ -22,20 +25,32 @@ export function registerIssueRecoveryWork( canRun: () => boolean, ) { function register(queue: DeliveryQueue, statuses: string[] | null, check: (scope: Scope) => Promise) { + const cursors = new Map(); + let enabled = false; return registerScopedDeadlineWork(coordinator, { queue, + reconstructionIntervalMs: 30_000, + beforePass: async () => { enabled = !(await owner.schedulingSuppressed()); }, decodeScope: hint => hint === "*" ? undefined : JSON.parse(hint) as IssueRecoveryScope, matches: (entry, scope) => !scope || entry.companyId === scope.companyId, load: async scope => { + const key = scope?.companyId ?? "*"; + if (!enabled) { cursors.delete(key); return []; } + const cursor = cursors.get(key); const rows = await db.select({ id: issues.id, companyId: issues.companyId }).from(issues).where(and( scope ? eq(issues.companyId, scope.companyId) : undefined, + cursor ? gt(issues.id, cursor) : undefined, statuses ? inArray(issues.status, statuses) : or(isNotNull(issues.checkoutRunId), isNotNull(issues.executionRunId)), statuses ? isNull(issues.hiddenAt) : undefined, - )); - return rows.map(row => ({ ...row, dueAt: Date.now() })); + statuses ? isNull(issues.assigneeUserId) : undefined, + statuses ? or(isNotNull(issues.assigneeAgentId), eq(issues.status, "in_review")) : undefined, + )).orderBy(asc(issues.id)).limit(PAGE_SIZE); + const complete = rows.length < PAGE_SIZE; + if (complete) cursors.delete(key); else cursors.set(key, rows.at(-1)!.id); + return { rows: rows.map(row => ({ ...row, dueAt: Date.now() })), complete }; }, check: async entry => { - if (await owner.schedulingSuppressed()) return null; + if (!enabled) return null; let next: number | null = null; await check({ companyId: entry.companyId, issueId: entry.id, onDeferredDeadline: at => { next = next === null ? at : Math.min(next, at); } }); @@ -43,7 +58,45 @@ export function registerIssueRecoveryWork( }, }, canRun); } + function registerCompany(queue: DeliveryQueue, kind: "blockers" | "actions", check: (scope: { companyId: string }) => Promise) { + const cursors = new Map(); + let enabled = false; + return registerScopedDeadlineWork(coordinator, { + queue, + reconstructionIntervalMs: 30_000, + beforePass: async () => { enabled = !(await owner.schedulingSuppressed()); }, + decodeScope: hint => hint === "*" ? undefined : JSON.parse(hint) as IssueRecoveryScope, + matches: (entry, scope) => !scope || entry.companyId === scope.companyId, + load: async scope => { + const key = scope?.companyId ?? "*"; + if (!enabled) { cursors.delete(key); return []; } + const cursor = cursors.get(key); + const rows = kind === "actions" + ? await db.selectDistinct({ id: issueRecoveryActions.companyId, companyId: issueRecoveryActions.companyId }) + .from(issueRecoveryActions).where(and(inArray(issueRecoveryActions.status, ["active", "escalated"]), + scope ? eq(issueRecoveryActions.companyId, scope.companyId) : undefined, + cursor ? gt(issueRecoveryActions.companyId, cursor) : undefined)) + .orderBy(asc(issueRecoveryActions.companyId)).limit(PAGE_SIZE) + : await db.selectDistinct({ id: issues.companyId, companyId: issues.companyId }).from(issues).where(and( + inArray(issues.status, ["todo", "blocked"]), isNull(issues.assigneeAgentId), isNull(issues.assigneeUserId), + isNotNull(issues.createdByAgentId), + scope ? eq(issues.companyId, scope.companyId) : undefined, + cursor ? gt(issues.companyId, cursor) : undefined, + sql`exists (select 1 from ${issueRelations} r inner join ${issues} dependent + on dependent.id = r.related_issue_id and dependent.company_id = r.company_id + where r.company_id = ${issues.companyId} and r.issue_id = ${issues.id} + and r.type = 'blocks' and dependent.status not in ('done', 'cancelled'))`, + )).orderBy(asc(issues.companyId)).limit(PAGE_SIZE); + const complete = rows.length < PAGE_SIZE; + if (complete) cursors.delete(key); else cursors.set(key, rows.at(-1)!.id); + return { rows: rows.map(row => ({ ...row, dueAt: Date.now() })), complete }; + }, + check: async entry => { if (enabled) await check({ companyId: entry.companyId }); return null; }, + }, canRun); + } const assignments = register(DELIVERY_QUEUES.strandedAssignments, ["todo", "in_progress", "in_review"], owner.stranded); + const orphanBlockers = registerCompany(DELIVERY_QUEUES.orphanBlockers, "blockers", owner.orphanBlockers); + const recoveryActions = registerCompany(DELIVERY_QUEUES.activeRecoveryActions, "actions", owner.recoveryActions); const dependencies = register(DELIVERY_QUEUES.resolvedDependencies, ["blocked"], async scope => { const result = await owner.dependencies(scope); if (result.enqueueFailed > 0) throw new Error("Resolved dependency wake remains undelivered"); @@ -52,7 +105,8 @@ export function registerIssueRecoveryWork( const watchdogs = registerTaskWatchdogWork(db, coordinator, { reconcile: owner.watchdogs, enabled: async () => !(await owner.schedulingSuppressed()), }, canRun); - return { ready: Promise.all([assignments.ready, dependencies.ready, locks.ready, watchdogs.ready]), - wake() { assignments.invalidate(); dependencies.invalidate(); locks.invalidate(); watchdogs.invalidate(); }, - async stop() { await Promise.all([assignments.stop(), dependencies.stop(), locks.stop(), watchdogs.stop()]); } }; + const workers = [assignments, orphanBlockers, recoveryActions, dependencies, locks, watchdogs]; + return { ready: Promise.all(workers.map(worker => worker.ready)), + wake() { for (const worker of workers) worker.invalidate(); }, + async stop() { await Promise.all(workers.map(worker => worker.stop())); } }; } diff --git a/server/src/services/scoped-deadline-work.ts b/server/src/services/scoped-deadline-work.ts index b46841e2a91..dbb9fe3df95 100644 --- a/server/src/services/scoped-deadline-work.ts +++ b/server/src/services/scoped-deadline-work.ts @@ -12,6 +12,8 @@ export function registerScopedDeadlineWork( queue: DeliveryQueue; batchSize?: number; minimumPassIntervalMs?: number; + reconstructionIntervalMs?: number; + beforePass?(): Promise; decodeScope(hint: string): Scope | undefined; matches(entry: Entry, scope?: Scope): boolean; load(scope?: Scope, signal?: AbortSignal): Promise; @@ -25,6 +27,7 @@ export function registerScopedDeadlineWork( const entries = new Map(); const failures = new Map(); const scopeFailures = new Map(); + const reconstructAfter = new Map(); let earliest = 0; let lastCompany: string | null = null; function failed(map: typeof failures, id: string) { @@ -34,13 +37,14 @@ export function registerScopedDeadlineWork( function dirtyScopes() { return [...dirty].filter(hint => hint !== "*" || !scopeFailures.has(hint)) .filter(hint => !scans.has(hint) || scopeFailures.has(hint)) + .filter(hint => (reconstructAfter.get(hint) ?? 0) <= Date.now() || scopeFailures.has(hint)) .map(hint => { try { return options.decodeScope(hint); } catch { return undefined; } }); } function next() { if (!canRun()) return dirty.size > 0 || entries.size > 0 ? Date.now() + 30_000 : null; let nextAt: number | null = null; const take = (at: number) => { nextAt = nextAt === null ? at : Math.min(nextAt, at); }; - for (const hint of dirty) take(Math.max(earliest, scopeFailures.get(hint)?.retryAt ?? 0)); + for (const hint of dirty) take(Math.max(earliest, reconstructAfter.get(hint) ?? 0, scopeFailures.get(hint)?.retryAt ?? 0)); const unavailable = dirtyScopes(); for (const entry of entries.values()) if (!unavailable.some(scope => options.matches(entry, scope))) take(Math.max(earliest, entry.dueAt, failures.get(entry.id)?.retryAt ?? 0)); @@ -54,9 +58,18 @@ export function registerScopedDeadlineWork( } }, run: async signal => { if (!canRun() || Date.now() < earliest) return; + if ((options.reconstructionIntervalMs ?? 0) > 0) for (const [hint, until] of reconstructAfter) { + if (until <= Date.now()) reconstructAfter.delete(hint); + } + const ready = [...dirty].filter(hint => Math.max(reconstructAfter.get(hint) ?? 0, scopeFailures.get(hint)?.retryAt ?? 0) <= Date.now()); + if (ready.length === 0 && ![...entries.values()].some(entry => Math.max(entry.dueAt, failures.get(entry.id)?.retryAt ?? 0) <= Date.now())) return; earliest = Date.now() + (options.minimumPassIntervalMs ?? 1000); + try { await options.beforePass?.(); } catch (error) { + for (const hint of ready) failed(scopeFailures, hint); + for (const entry of entries.values()) if (entry.dueAt <= Date.now()) failed(failures, entry.id); + throw error; + } const errors: unknown[] = []; - const ready = [...dirty].filter(hint => (scopeFailures.get(hint)?.retryAt ?? 0) <= Date.now()); const requested = ready.includes("*") ? ["*", ...ready.filter(hint => hint !== "*")].slice(0, 25) : ready.slice(0, 25); @@ -81,6 +94,9 @@ export function registerScopedDeadlineWork( entries.delete(entry.id); failures.delete(entry.id); } scans.delete(hint); + if ((options.reconstructionIntervalMs ?? 0) > 0) { + reconstructAfter.set(hint, Date.now() + options.reconstructionIntervalMs!); + } if ((versions.get(hint) ?? 0) !== scan.version) dirty.add(hint); else versions.delete(hint); if (!scope) scopeFailures.clear(); else scopeFailures.delete(hint); } catch (error) { diff --git a/server/src/services/task-watchdog-work.ts b/server/src/services/task-watchdog-work.ts index cde0a7eca43..98794472da2 100644 --- a/server/src/services/task-watchdog-work.ts +++ b/server/src/services/task-watchdog-work.ts @@ -1,4 +1,4 @@ -import { and, eq } from "drizzle-orm"; +import { and, asc, eq, gt } from "drizzle-orm"; import { issueWatchdogs, type Db } from "@paperclipai/db"; import type { createDeliveryWorkCoordinator } from "./delivery-work-coordinator.js"; import { DELIVERY_QUEUES } from "./delivery-work-notifications.js"; @@ -16,16 +16,29 @@ export function registerTaskWatchdogWork( }, canRun: () => boolean, ) { + const cursors = new Map(); + let enabled = false; return registerScopedDeadlineWork(coordinator, { queue: DELIVERY_QUEUES.taskWatchdogs, + reconstructionIntervalMs: 30_000, + beforePass: async () => { enabled = await owner.enabled(); }, decodeScope: hint => hint === "*" ? undefined : JSON.parse(hint) as IssueRecoveryScope, matches: (entry, scope) => !scope || entry.companyId === scope.companyId, - load: async scope => (await db.select({ id: issueWatchdogs.id, companyId: issueWatchdogs.companyId }) - .from(issueWatchdogs).where(and(eq(issueWatchdogs.status, "active"), - scope ? eq(issueWatchdogs.companyId, scope.companyId) : undefined))) - .map(row => ({ ...row, dueAt: Date.now() })), + load: async scope => { + const key = scope?.companyId ?? "*"; + if (!enabled) { cursors.delete(key); return []; } + const cursor = cursors.get(key); + const rows = await db.select({ id: issueWatchdogs.id, companyId: issueWatchdogs.companyId }) + .from(issueWatchdogs).where(and(eq(issueWatchdogs.status, "active"), + scope ? eq(issueWatchdogs.companyId, scope.companyId) : undefined, + cursor ? gt(issueWatchdogs.id, cursor) : undefined)) + .orderBy(asc(issueWatchdogs.id)).limit(100); + const complete = rows.length < 100; + if (complete) cursors.delete(key); else cursors.set(key, rows.at(-1)!.id); + return { rows: rows.map(row => ({ ...row, dueAt: Date.now() })), complete }; + }, check: async entry => { - if (!(await owner.enabled())) return null; + if (!enabled) return null; let next: number | null = null; const result = await owner.reconcile({ companyId: entry.companyId, watchdogId: entry.id, onDeferredDeadline: at => { next = next === null ? at : Math.min(next, at); } }); From 32044a63e138e7b6a38ec801878d0db65cce6bbe Mon Sep 17 00:00:00 2001 From: Dotta Date: Fri, 30 Oct 2026 19:00:51 -0500 Subject: [PATCH 55/70] fix: scope status-card deletion notifications to its company --- doc/plans/2026-10-30-status-cards-sleep.md | 6 ++++ .../status-card-evaluation-work.test.ts | 32 +++++++++++++++++++ server/src/services/status-cards.ts | 6 ++-- 3 files changed, 42 insertions(+), 2 deletions(-) diff --git a/doc/plans/2026-10-30-status-cards-sleep.md b/doc/plans/2026-10-30-status-cards-sleep.md index 5b98962698b..61d451caed5 100644 --- a/doc/plans/2026-10-30-status-cards-sleep.md +++ b/doc/plans/2026-10-30-status-cards-sleep.md @@ -135,3 +135,9 @@ scheduled work after settlement. Real issue/comment producer acceptance measures due/next queries and actual searches under 100 writes, preserves both companies' hints, dispatches another company's saved deadline inside the batching window, and checks SQL quietness after authorized results. + +Card deletion resolves the card's owning company and emits only that company's +deadline-reconstruction hint. Every card mutation requires a company ID; only +instance-level settings retain wildcard ingress. Deleting an already-missing +card emits no hint. The real two-company regression proves deletion does not +re-run unchanged reactive searches or wake either summarizer, then stays quiet. diff --git a/server/src/__tests__/status-card-evaluation-work.test.ts b/server/src/__tests__/status-card-evaluation-work.test.ts index 6bb466dea89..e4588b16be1 100644 --- a/server/src/__tests__/status-card-evaluation-work.test.ts +++ b/server/src/__tests__/status-card-evaluation-work.test.ts @@ -144,6 +144,38 @@ const support = external ? { supported: true } : await getEmbeddedPostgresTestSu expect(errors).toEqual([]); }); + it("keeps card deletion scoped to its company and a missing deletion quiet", async () => { + const removed = await card(); + await card(); + const ownCompany = companyId, ownAgent = agentId; + companyId = randomUUID(); agentId = randomUUID(); + await db.insert(companies).values({ id: companyId, name: "Other", issuePrefix: `O${companyId.slice(0, 6)}` }); + await db.insert(agents).values({ id: agentId, companyId, name: "Other summarizer", role: "general", status: "idle", adapterType: "process", adapterConfig: {} }); + const other = await card(); + companyId = ownCompany; agentId = ownAgent; + const factory = companySearch.companySearchService; + const searchedCompanies: string[] = []; + vi.spyOn(companySearch, "companySearchService").mockImplementation(owner => { + const search = factory(owner), actual = search.search; + search.search = async (...args) => { searchedCompanies.push(args[0]); return actual(...args); }; + return search; + }); + await start(); await quiet(); searchedCompanies.length = 0; + const cards = statusCardService(db); + expect(await cards.remove(removed.id)).toMatchObject({ id: removed.id, companyId: ownCompany }); + await settled(); + // Deleting a card only reconstructs deadlines; it is not an issue change + // that warrants re-running any unchanged reactive card's saved search. + expect(searchedCompanies).toEqual([]); + expect(await cards.getById(other.id)).toMatchObject({ nextEvalAt: null, generatingIssueId: null, pendingChangeCount: 0 }); + expect(wakeup).not.toHaveBeenCalled(); + searchedCompanies.length = 0; + expect(await cards.remove(removed.id)).toBeNull(); + await quiet(); + expect(searchedCompanies).toEqual([]); + expect(errors).toEqual([]); + }); + it("coalesces one hundred committed source writes without delaying another company's saved deadline", async () => { const reactive = await card(); const originalCompany = companyId, originalAgent = agentId; diff --git a/server/src/services/status-cards.ts b/server/src/services/status-cards.ts index f9e6372ea2e..d4660227c28 100644 --- a/server/src/services/status-cards.ts +++ b/server/src/services/status-cards.ts @@ -158,7 +158,7 @@ export function statusCardService( const issuesSvc = () => deps.issuesSvc ?? issueService(db); const searchSvc = companySearchService(db); type Transaction = Parameters[0]>[0]; - function mutate(companyId: string | undefined, write: (tx: Transaction) => Promise) { + function mutate(companyId: string, write: (tx: Transaction) => Promise) { return db.transaction(async tx => { await notifyStatusCardEvaluation(tx, companyId); return write(tx); @@ -335,7 +335,9 @@ export function statusCardService( } async function remove(id: string) { - return mutate(undefined, tx => tx.delete(statusCards).where(eq(statusCards.id, id)).returning().then((rows) => rows[0] ?? null)); + const card = await getById(id); + if (!card) return null; + return mutate(card.companyId, tx => tx.delete(statusCards).where(eq(statusCards.id, id)).returning().then((rows) => rows[0] ?? null)); } async function listUpdates(cardId: string) { From e41bde214638004b51752091938834691eb1c616 Mon Sep 17 00:00:00 2001 From: Dotta Date: Fri, 30 Oct 2026 19:01:12 -0500 Subject: [PATCH 56/70] fix(server): bound graph queries and scope issue retention work Chunk graph recovery, agent and workspace inputs; prove a70000-node PostgreSQL graph stays below parameter limits. Route ordinary issue writes through identity-scoped dependency invalidation for old/new parents, blocker ancestors, run links and liveness origins. Correlate all issue-related attention sources, preserve unrelated deadlines, and canonicalize notification keys. Exercise actual held/burst producers, visibility, relation and incident transitions. Co-Authored-By: Paperclip --- .../2026-10-30-attention-retention-sleep.md | 2 +- .../attention-retention-work.test.ts | 111 ++++++++++++++- server/src/services/attention-retention.ts | 31 +++-- .../issue-review-state-notifications.ts | 2 - server/src/services/issues.ts | 127 +++++++++--------- ...ative-status-reassessment-notifications.ts | 4 +- .../services/retention-work-notifications.ts | 48 ++++++- 7 files changed, 237 insertions(+), 88 deletions(-) diff --git a/doc/plans/2026-10-30-attention-retention-sleep.md b/doc/plans/2026-10-30-attention-retention-sleep.md index 463fbe94188..24212b4b98a 100644 --- a/doc/plans/2026-10-30-attention-retention-sleep.md +++ b/doc/plans/2026-10-30-attention-retention-sleep.md @@ -25,7 +25,7 @@ The feed and worker share confirmation collapse, newer-run probes, issue visibil ## Existing mutation boundaries reactivate work -Committed hints identify a company, source kind, and source ID where those values are known. Retention's own writes use exact source hints; blocker retention changes invalidate the company/source because multiple blocked roots can project one terminal subject. Reconstruction pages and due batches remain bounded. Review attention and interaction invokability are computed in company batches per candidate page; a full 100-candidate review page uses at most 20 statements and 100 confirmations use at most six. Decisions, recovery source links, runs, eligible agents, issue visibility, newer-run probes and blocker graphs also load in company batches per page. A 100-source decision or recovery page uses at most three statements; 100 failed runs use at most six; 100 blocker roots use at most 30 graph/source statements plus retention-state reads for their emitted entries. These evidence maps exist only for one load. The blocker owner shares fetched node and edge facts while tracking depth, node count, expanded nodes and live-work reachability independently for each root. An unrelated graph reaching its limit cannot change a shallow root’s terminal subject or deadline. Archive checks still lock and freshly re-read sources, terminal subjects and related eligibility. Ordinary execution, decision and budget mutations pass their affected company; exact decision sources are included. +Committed hints identify a company, source kind, source ID or changed issue where those values are known. Routine issue writers pass identity to the retention domain helper before their mutation. The helper captures old and new parent paths, reverse blocker ancestors, run-linked issues and liveness-origin references; issue-scoped source queries then select reviews, blockers, interactions, decisions, recovery actions and failed runs by their issue identity. Budget incidents and agent errors remain tied to their own source changes. Ordinary issue writes do not reconstruct the unrelated company feed. Queue keys explicitly pick supported identity fields. Retention's own writes use exact source hints; blocker retention changes invalidate the company/source because multiple blocked roots can project one terminal subject. Reconstruction pages and due batches remain bounded. Review attention and interaction invokability are computed in company batches per candidate page; a full 100-candidate review page uses at most 20 statements and 100 confirmations use at most six. Decisions, recovery source links, runs, eligible agents, issue visibility, newer-run probes and blocker graphs also load in company batches per page. A 100-source decision or recovery page uses at most three statements; 100 failed runs use at most six; 100 blocker roots use at most 30 graph/source statements plus retention-state reads for their emitted entries. These evidence maps exist only for one load. The blocker owner shares fetched node and edge facts while tracking depth, node count, expanded nodes and live-work reachability independently for each root. An unrelated graph reaching its limit cannot change a shallow root’s terminal subject or deadline. All later graph queries, including recovery source IDs, agent IDs and workspace-finalize inputs, use bounded chunks. A SQL-generated 70,000-node graph proves PostgreSQL parameter limits are respected at the driver boundary. Archive checks still lock and freshly re-read sources, terminal subjects and related eligibility. Ordinary execution, decision and budget mutations pass their affected company; exact decision sources are included. Producer coverage includes decision creation and changes; interaction creation, resolution, expiry and supersession; issue creation, updates, blocker relations, checkout, adoption and release; agent hire, configuration, lifecycle, error clearing and deletion; budget incidents and company pause/resume; exhausted-run events and retry-run creation; native status, review, finalization and reconstruction; external conversation transitions; and imported blocker relations. Existing service transactions retain outer commit ownership. Savepoint release does not publish work before outer commit. diff --git a/server/src/__tests__/attention-retention-work.test.ts b/server/src/__tests__/attention-retention-work.test.ts index 29a82f866a5..b1e7949a8b7 100644 --- a/server/src/__tests__/attention-retention-work.test.ts +++ b/server/src/__tests__/attention-retention-work.test.ts @@ -1,7 +1,7 @@ import { randomUUID } from "node:crypto"; import { and, eq, sql } from "drizzle-orm"; import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; -import { authUsers, companyMemberships, agents, approvals, budgetIncidents, budgetPolicies, companies, createDb, applyPendingMigrations, decisionArchiveNotificationOutbox, +import { subscribeDatabaseWork, authUsers, companyMemberships, agents, approvals, budgetIncidents, budgetPolicies, companies, createDb, applyPendingMigrations, decisionArchiveNotificationOutbox, decisionRetention, decisions, costEvents, heartbeatRunEvents, heartbeatRuns, issueApprovals, issueRecoveryActions, issueRelations, issueThreadInteractions, issues, type Db } from "@paperclipai/db"; import { attentionRetentionService, registerAttentionRetentionWork } from "../services/attention-retention.js"; @@ -13,6 +13,7 @@ import { companyService } from "../services/companies.js"; import { createDeliveryWorkCoordinator } from "../services/delivery-work-coordinator.js"; import { decisionRetentionService, DEFAULT_DECISION_ARCHIVE_DAYS } from "../services/decision-retention.js"; import { appendHeartbeatRunEvent } from "../services/heartbeat-run-events.js"; +import { buildIssueGraphLivenessIncidentKey, RECOVERY_ORIGIN_KINDS } from "../services/recovery/origins.js"; import { issueThreadInteractionService } from "../services/issue-thread-interactions.js"; import { createAgentLifecycle } from "../services/agent-lifecycle.js"; import { BLOCKER_ATTENTION_MAX_DEPTH, BLOCKER_ATTENTION_MAX_NODES, issueService } from "../services/issues.js"; @@ -32,12 +33,13 @@ const delay = DEFAULT_DECISION_ARCHIVE_DAYS * 86_400_000; let coordinator: ReturnType | undefined; const errors: unknown[] = []; const statements: string[] = []; + const parameterCounts: number[] = []; beforeAll(async () => { if (externalUrl) await applyPendingMigrations(externalUrl); else temporary = await startEmbeddedPostgresTestDatabase("attention-retention-"); db = createDb(externalUrl ?? temporary!.connectionString, { maxConnections: 6, idleTimeoutSeconds: 0, maxLifetimeSeconds: 0 }); db.$client.options.onnotice = () => {}; - db.$client.options.debug = (_connection, query) => statements.push(query); + db.$client.options.debug = (_connection, query, parameters) => { statements.push(query); parameterCounts.push(parameters.length); }; }, 30_000); beforeEach(async () => { companyId = randomUUID(); agentId = randomUUID(); draining = false; errors.length = 0; @@ -311,6 +313,111 @@ const delay = DEFAULT_DECISION_ARCHIVE_DAYS * 86_400_000; } }); + it("bounds every graph query when independent root trees contain over 65535 unfinished nodes", async () => { + await db.execute(sql`insert into issues (id, company_id, title, status, updated_at) + select gen_random_uuid(), ${companyId}::uuid, 'parameter-root-' || n, 'blocked', ${old().toISOString()}::timestamptz + from generate_series(1, 70) n`); + await db.execute(sql`insert into issues (id, company_id, parent_id, title, status, assignee_agent_id, updated_at) + select gen_random_uuid(), ${companyId}::uuid, root.id, 'parameter-leaf-' || n, 'backlog', ${agentId}::uuid, ${old().toISOString()}::timestamptz + from issues root cross join generate_series(1, 999) n + where root.company_id = ${companyId}::uuid and root.status = 'blocked'`); + const roots = await db.select().from(issues).where(and(eq(issues.companyId, companyId), eq(issues.status, "blocked"))); + parameterCounts.length = 0; + const result = await issueService(db).listBlockerAttention(companyId, roots).catch(error => { + throw new Error(`Large graph failed: ${String(error.cause?.message ?? error.message).slice(0, 300)}`); + }); + expect(result.size).toBe(70); + expect([...result.values()].every(item => item.state === "needs_attention" && item.terminalBlockerIssueId)).toBe(true); + expect(Math.max(...parameterCounts)).toBeLessThanOrEqual(1100); + }, 120_000); + + it("canonicalizes wide circular notification variables to the supported identity fields", async () => { + const sourceId = randomUUID(), issueId = randomUUID(), hints: string[] = []; + const wide = { companyId, sourceKind: "review" as const, sourceId, issueId, ignored: {} as unknown }; + wide.ignored = wide; + const unsubscribe = subscribeDatabaseWork(db, DELIVERY_QUEUES.attentionRetention, (event, values) => { + if (event === "settled") hints.push(...values ?? []); + }); + try { await db.transaction(tx => notifyAttentionRetentionWork(tx, wide)); } finally { unsubscribe(); } + expect(hints).toEqual([JSON.stringify({ companyId, sourceKind: "review", sourceId, issueId })]); + }); + + it("keeps held, burst and sustained ordinary issue writes scoped while an unrelated deadline fires", async () => { + const row = await issueService(db).create(companyId, { title: "Active review", status: "in_review", assigneeUserId: "board", createdByAgentId: agentId }); + const urgent = await insertIssue("in_review", { assigneeUserId: "board", updatedAt: new Date(Date.now() - delay + 10_000) }); + const otherCompany = randomUUID(); + await db.insert(companies).values({ id: otherCompany, name: "Unrelated", issuePrefix: "BURST" }); + const foreign = await db.insert(issues).values(Array.from({ length: 125 }, () => ({ companyId: otherCompany, + title: randomUUID(), status: "in_review", assigneeUserId: "board", updatedAt: old() }))).returning(); + await db.insert(decisionRetention).values(foreign.map(issue => ({ companyId: otherCompany, sourceKind: "review", sourceId: issue.id, sourceActivityAt: issue.updatedAt, keep: true }))); + const service = attentionRetentionService(db), original = service.load, load = vi.spyOn(service, "load"); + await start(service); await settle(); load.mockClear(); + let release!: () => void, entered = false; + const gate = new Promise(resolve => { release = resolve; }); + load.mockImplementationOnce(async (...args) => { const page = await original(...args); entered = true; await gate; return page; }); + statements.length = 0; + await issueService(db).update(row.id, { title: "Held mutation" }); + await vi.waitFor(() => expect(entered).toBe(true), { timeout: 15_000 }); + await Promise.all(Array.from({ length: 5 }, (_, index) => issueService(db).update(row.id, { title: `Burst ${index}` }))); + release(); await settle(); + for (let index = 0; index < 8; index++) { + await issueService(db).update(row.id, { title: `Steady ${index}` }); await vi.advanceTimersByTimeAsync(1000); await settle(); + } + await archived("review", urgent.id); + const routine = load.mock.calls.map(([scope]) => scope).filter(scope => !scope?.sourceId); + expect(routine.length).toBeGreaterThan(0); + expect(routine.every(scope => scope?.companyId === companyId && scope.issueId === row.id)).toBe(true); + const scans = statements.filter(query => query.includes("with source_candidates")); + expect(scans.length).toBeGreaterThan(0); + expect(scans.every(query => !query.includes("budget_incidents") && !query.includes("agent_error_alert"))).toBe(true); + expect(await state("review", row.id)).toBeNull(); + }); + + it("wakes all linked issue sources when an ordinary visibility mutation reveals their origin", async () => { + const origin = await insertIssue("in_review", { hiddenAt: old(), assigneeUserId: "board", responsibleUserId: "board" }); + const [run] = await db.insert(heartbeatRuns).values({ companyId, agentId, status: "failed", contextSnapshot: { issueId: origin.id }, createdAt: old(), finishedAt: old() }).returning(); + await db.insert(heartbeatRunEvents).values({ companyId, agentId, runId: run.id, seq: 1, eventType: "lifecycle", message: "Bounded retry exhausted: visibility" }); + const [decision] = await db.insert(decisions).values({ companyId, originAgentId: agentId, originIssueId: origin.id, originRunId: run.id, + title: "Decision", body: "Choose", options: [], signedSpec: "fixture", targetSnapshots: {}, expiresAt: new Date(Date.now() + 86_400_000), updatedAt: old() }).returning(); + const [recovery] = await db.insert(issueRecoveryActions).values({ companyId, sourceIssueId: origin.id, kind: "execution", ownerType: "board", cause: "fixture", fingerprint: "fixture", nextAction: "Choose", updatedAt: old() }).returning(); + const [interaction] = await db.insert(issueThreadInteractions).values({ companyId, issueId: origin.id, kind: "ask_user_questions", payload: { questions: [] }, updatedAt: old() }).returning(); + await start(); await quiet(); + await issueService(db).update(origin.id, { hiddenAt: null }); + for (const [kind, id] of [["decision", decision.id], ["recovery_action", recovery.id], ["failed_run", run.id], ["issue_thread_interaction", interaction.id]]) await archived(kind, id); + await settle(); expect(await state("review", origin.id)).toBeNull(); expect(coordinator!.nextWakeAt()).not.toBeNull(); + }); + + it("invalidates old and new parents, terminal visibility and repointed blocker relations", async () => { + const oldParent = await insertIssue("blocked", { updatedAt: new Date(), responsibleUserId: "board" }); + const newParent = await insertIssue("blocked", { updatedAt: new Date(), responsibleUserId: "board" }); + const terminal = await insertIssue("backlog", { parentId: oldParent.id, hiddenAt: old(), assigneeAgentId: agentId, responsibleUserId: "board" }); + const service = attentionRetentionService(db), load = vi.spyOn(service, "load"); await start(service); await settle(); load.mockClear(); + await issueService(db).update(terminal.id, { parentId: newParent.id }); await settle(); + const ids = load.mock.calls.map(([scope]) => scope?.issueId); + expect(ids).toEqual(expect.arrayContaining([oldParent.id, newParent.id, terminal.id])); + load.mockClear(); + await issueService(db).update(terminal.id, { hiddenAt: null }); await settle(); + expect(await state("blocker_attention", terminal.id)).toBeNull(); + const visible = await service.load({ companyId, issueId: newParent.id }); + expect(visible.rows!.some(entry => entry.sourceId === terminal.id && entry.dueAt > Date.now())).toBe(true); + expect(load.mock.calls.some(([scope]) => scope?.issueId === newParent.id)).toBe(true); + const next = await insertIssue("backlog", { assigneeAgentId: agentId }); + await issueService(db).update(oldParent.id, { blockedByIssueIds: [next.id] }); await archived("blocker_attention", next.id); + }); + + it("cancels and reactivates a blocker deadline through ordinary liveness incident creation and hiding", async () => { + const root = await insertIssue("blocked"); + const terminal = await insertIssue("backlog", { assigneeAgentId: agentId, updatedAt: new Date(Date.now() - delay + 10_000) }); + await db.insert(issueRelations).values({ companyId, issueId: terminal.id, relatedIssueId: root.id, type: "blocks" }); + await start(); await settle(); expect(coordinator!.nextWakeAt()).not.toBeNull(); + const incident = await issueService(db).create(companyId, { title: "Resolve liveness", status: "todo", assigneeUserId: "board", + createdByUserId: "board", originKind: RECOVERY_ORIGIN_KINDS.issueGraphLivenessEscalation, + originId: buildIssueGraphLivenessIncidentKey({ companyId, issueId: root.id, state: "needs_attention", blockerIssueId: terminal.id }) }); + await quiet(); expect(await state("blocker_attention", terminal.id)).toBeNull(); + await issueService(db).update(incident.id, { hiddenAt: new Date() }); + await archived("blocker_attention", terminal.id); await quiet(); + }); + it("scopes ordinary execution-control writes to their company", async () => { const otherCompany = randomUUID(); await db.insert(companies).values({ id: otherCompany, name: "Other", issuePrefix: "OTHER" }); diff --git a/server/src/services/attention-retention.ts b/server/src/services/attention-retention.ts index c72947e5349..27aedba8c47 100644 --- a/server/src/services/attention-retention.ts +++ b/server/src/services/attention-retention.ts @@ -18,8 +18,8 @@ const archiveDelay = DEFAULT_DECISION_ARCHIVE_DAYS * 86_400_000; const kinds = ["decision", "issue_thread_interaction", "recovery_action", "blocker_attention", "review", "failed_run", "budget_alert", "agent_error_alert"] as const; type Kind = typeof kinds[number]; -type Scope = { companyId?: string; sourceKind?: AttentionSourceKind; sourceId?: string }; -type Candidate = { sourceKind: Kind; sourceId: string; companyId: string; activityAt: string; deadlineAt: number }; +type Scope = { companyId?: string; sourceKind?: AttentionSourceKind; sourceId?: string; issueId?: string }; +type Candidate = { sourceKind: Kind; sourceId: string; companyId: string; activityAt: string; deadlineAt: number; issueId: string | null }; type Entry = Candidate & { id: string; dueAt: number; rootId: string; variant?: "owner" | "terminal" }; type Cursor = { sourceKind: string; sourceId: string }; @@ -42,37 +42,39 @@ const sourceTables: Record = { }; const sources: Record = { - decision: sql`select 'decision' as source_kind, d.id::text as source_id, d.company_id, d.updated_at as activity_at + decision: sql`select 'decision' as source_kind, d.id::text as source_id, d.company_id, d.updated_at as activity_at, d.origin_issue_id::text as issue_id from decisions d where d.status = 'open'`, - issue_thread_interaction: sql`select 'issue_thread_interaction', i.id::text, i.company_id, i.updated_at + issue_thread_interaction: sql`select 'issue_thread_interaction', i.id::text, i.company_id, i.updated_at, i.issue_id::text from issue_thread_interactions i where i.status = 'pending' and i.addressee_user_id is null`, - recovery_action: sql`select 'recovery_action', r.id::text, r.company_id, r.updated_at + recovery_action: sql`select 'recovery_action', r.id::text, r.company_id, r.updated_at, r.source_issue_id::text from issue_recovery_actions r where r.status in ('active', 'escalated') and r.owner_type in (${sql.join(HUMAN_RECOVERY_OWNER_TYPES.map(owner => sql`${owner}`), sql`, `)})`, - blocker_attention: sql`select 'blocker_attention', i.id::text, i.company_id, i.updated_at + blocker_attention: sql`select 'blocker_attention', i.id::text, i.company_id, i.updated_at, i.id::text from issues i where i.status = 'blocked' and i.hidden_at is null and i.harness_kind is null`, - review: sql`select 'review', i.id::text, i.company_id, i.updated_at + review: sql`select 'review', i.id::text, i.company_id, i.updated_at, i.id::text from issues i where i.status = 'in_review' and i.hidden_at is null and i.harness_kind is null`, - failed_run: sql`select 'failed_run', r.id::text, r.company_id, coalesce(r.finished_at, r.updated_at, r.created_at) + failed_run: sql`select 'failed_run', r.id::text, r.company_id, coalesce(r.finished_at, r.updated_at, r.created_at), coalesce(r.context_snapshot->>'issueId', r.context_snapshot->>'taskId') from heartbeat_runs r where r.status in ('failed', 'timed_out') and exists ( select 1 from heartbeat_run_events e where e.run_id = r.id and e.company_id = r.company_id and e.event_type = 'lifecycle' and e.message like ${`${RETRY_EXHAUSTION_MESSAGE_PREFIX}%`})`, - budget_alert: sql`select 'budget_alert', b.id::text, b.company_id, b.updated_at from budget_incidents b + budget_alert: sql`select 'budget_alert', b.id::text, b.company_id, b.updated_at, null::text from budget_incidents b where b.status = 'open' and (b.threshold_type = 'hard' or (b.amount_limit > 0 and round(b.amount_observed::numeric / b.amount_limit * 10000) / 100 >= ${ATTENTION_BUDGET_WARNING_PERCENT}))`, - agent_error_alert: sql`select 'agent_error_alert', a.id::text, a.company_id, a.updated_at from agents a where a.status = ${ATTENTION_AGENT_ERROR_STATUS}`, + agent_error_alert: sql`select 'agent_error_alert', a.id::text, a.company_id, a.updated_at, null::text from agents a where a.status = ${ATTENTION_AGENT_ERROR_STATUS}`, }; export function attentionRetentionService(db: Db) { const cursors = new Map(); async function candidates(database: Db, scope: Scope = {}, cursor?: Cursor, sourceId?: string): Promise { - const selected = kinds.filter(kind => !scope.sourceKind || scope.sourceKind === kind); + const selected = kinds.filter(kind => (!scope.sourceKind || scope.sourceKind === kind) + && (!scope.issueId || !["budget_alert", "agent_error_alert"].includes(kind))); if (!selected.length) return []; - const rows = await database.execute(sql`with source_candidates(source_kind, source_id, company_id, activity_at) as (${sql.join(selected.map(kind => sources[kind]), sql` union all `)}) + const rows = await database.execute(sql`with source_candidates(source_kind, source_id, company_id, activity_at, issue_id) as (${sql.join(selected.map(kind => sources[kind]), sql` union all `)}) select s.source_kind as "sourceKind", s.source_id as "sourceId", s.company_id as "companyId", - s.activity_at::text as "activityAt", ceil(extract(epoch from s.activity_at) * 1000 + ${archiveDelay}) as "deadlineAt" + s.issue_id as "issueId", s.activity_at::text as "activityAt", ceil(extract(epoch from s.activity_at) * 1000 + ${archiveDelay}) as "deadlineAt" from source_candidates s inner join companies c on c.id = s.company_id left join decision_retention r on r.company_id = s.company_id and r.source_kind = s.source_kind and r.source_id = s.source_id where c.status = 'active' + and (${scope.issueId ?? null}::text is null or s.issue_id = ${scope.issueId ?? null}) and (${scope.companyId ?? null}::uuid is null or s.company_id = ${scope.companyId ?? null}::uuid) and (${sourceId ?? scope.sourceId ?? null}::text is null or s.source_id = ${sourceId ?? scope.sourceId ?? null}) and (s.source_kind = 'blocker_attention' or ((r.id is null or r.keep = false) and r.archived_at is null)) @@ -302,7 +304,8 @@ export function registerAttentionRetentionWork(coordinator: ReturnType hint === "*" ? undefined : JSON.parse(hint) as Scope, matches: (entry, scope) => (!scope?.companyId || scope.companyId === entry.companyId) && (!scope?.sourceKind || scope.sourceKind === entry.sourceKind) - && (!scope?.sourceId || scope.sourceId === entry.sourceId), + && (!scope?.sourceId || scope.sourceId === entry.sourceId) + && (!scope?.issueId || scope.issueId === entry.issueId), load: service.load, check: service.check, }, canRun); } diff --git a/server/src/services/issue-review-state-notifications.ts b/server/src/services/issue-review-state-notifications.ts index 353114a9bf0..679b7e37475 100644 --- a/server/src/services/issue-review-state-notifications.ts +++ b/server/src/services/issue-review-state-notifications.ts @@ -1,11 +1,9 @@ -import { notifyAttentionRetentionWork } from "./retention-work-notifications.js"; import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js"; import { notifyExternalConversationSettlement } from "./external-conversation-settlement-notifications.js"; import { notifyNativeReviewResponse } from "./native-review-response-notifications.js"; import { notifyNativeStatusReassessment } from "./native-status-reassessment-notifications.js"; export async function notifyIssueReviewState(tx: object, scope: { companyId: string; issueId?: string; runId?: string; agentId?: string }) { - await notifyAttentionRetentionWork(tx, { companyId: scope.companyId }); await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors); await notifyExternalConversationSettlement(tx, scope); await notifyNativeReviewResponse(tx, scope); diff --git a/server/src/services/issues.ts b/server/src/services/issues.ts index fd1fd9ba819..c91a325ad39 100644 --- a/server/src/services/issues.ts +++ b/server/src/services/issues.ts @@ -1,4 +1,4 @@ -import { notifyAttentionRetentionWork } from "./retention-work-notifications.js"; +import { notifyIssueAttentionRetentionWork } from "./retention-work-notifications.js"; import { notifyExternalConversationSettlement } from "./external-conversation-settlement-notifications.js"; import { notifyNativeRecoveryNotices } from "./native-recovery-notice-notifications.js"; import { notifyNativeRunProjection } from "./native-run-projection-notifications.js"; @@ -2538,33 +2538,37 @@ async function listPendingFinalizeBlockerIssueIds( ), ); - if (lockSources) { + const rows: Array<{ issueId: string | null; executionWorkspaceId: string | null; + phase: string; status: string; startedAt: Date }> = []; + for (const chunk of chunkList(executionWorkspaceIds, ISSUE_LIST_RELATED_QUERY_CHUNK_SIZE)) { + if (lockSources) { - await dbOrTx.select({ id: executionWorkspaces.id }).from(executionWorkspaces) - .where(and(eq(executionWorkspaces.companyId, companyId), inArray(executionWorkspaces.id, executionWorkspaceIds))) - .for("update", { noWait: true }); - } - const query = dbOrTx - .select({ - issueId: workspaceOperations.issueId, - executionWorkspaceId: workspaceOperations.executionWorkspaceId, - phase: workspaceOperations.phase, - status: workspaceOperations.status, - startedAt: workspaceOperations.startedAt, - }) - .from(workspaceOperations) - .where( - and( - eq(workspaceOperations.companyId, companyId), - inArray( - workspaceOperations.executionWorkspaceId, - executionWorkspaceIds, + await dbOrTx.select({ id: executionWorkspaces.id }).from(executionWorkspaces) + .where(and(eq(executionWorkspaces.companyId, companyId), inArray(executionWorkspaces.id, chunk))) + .for("update", { noWait: true }); + } + const query = dbOrTx + .select({ + issueId: workspaceOperations.issueId, + executionWorkspaceId: workspaceOperations.executionWorkspaceId, + phase: workspaceOperations.phase, + status: workspaceOperations.status, + startedAt: workspaceOperations.startedAt, + }) + .from(workspaceOperations) + .where( + and( + eq(workspaceOperations.companyId, companyId), + inArray( + workspaceOperations.executionWorkspaceId, + chunk, + ), ), - ), - ); + ); - const rows = await (lockSources ? query.for("share", { noWait: true }) : query); + rows.push(...await (lockSources ? query.for("share", { noWait: true }) : query)); + } const latestAttributedByBlockerWorkspace = new Map< string, @@ -4119,23 +4123,26 @@ async function listIssueBlockerAttentionMap( ownerType: string; ownerAgentId: string | null; ownerUserId: string | null; - }> = await dbOrTx - .select({ - id: issueRecoveryActions.id, - sourceIssueId: issueRecoveryActions.sourceIssueId, - status: issueRecoveryActions.status, - ownerType: issueRecoveryActions.ownerType, - ownerAgentId: issueRecoveryActions.ownerAgentId, - ownerUserId: issueRecoveryActions.ownerUserId, - }) - .from(issueRecoveryActions) - .where( - and( - eq(issueRecoveryActions.companyId, companyId), - inArray(issueRecoveryActions.status, ["active", "escalated"]), - inArray(issueRecoveryActions.sourceIssueId, explicitWaitCandidateIds), - ), - ); + }> = []; + for (const chunk of chunkList(explicitWaitCandidateIds, ISSUE_LIST_RELATED_QUERY_CHUNK_SIZE)) { + recoveryActionRows.push(...await dbOrTx + .select({ + id: issueRecoveryActions.id, + sourceIssueId: issueRecoveryActions.sourceIssueId, + status: issueRecoveryActions.status, + ownerType: issueRecoveryActions.ownerType, + ownerAgentId: issueRecoveryActions.ownerAgentId, + ownerUserId: issueRecoveryActions.ownerUserId, + }) + .from(issueRecoveryActions) + .where( + and( + eq(issueRecoveryActions.companyId, companyId), + inArray(issueRecoveryActions.status, ["active", "escalated"]), + inArray(issueRecoveryActions.sourceIssueId, chunk), + ), + )); + } const recoveryActionIds = recoveryActionRows.map((row) => row.id); const liveRecoveryActionIds = new Set(); for (const chunk of chunkList( @@ -4198,22 +4205,11 @@ async function listIssueBlockerAttentionMap( } } - const agentRows: IssueBlockerAttentionAgentRow[] = - agentIds.size > 0 - ? await dbOrTx - .select({ - id: agents.id, - companyId: agents.companyId, - status: agents.status, - }) - .from(agents) - .where( - and( - eq(agents.companyId, companyId), - inArray(agents.id, [...agentIds]), - ), - ) - : []; + const agentRows: IssueBlockerAttentionAgentRow[] = []; + for (const chunk of chunkList([...agentIds], ISSUE_LIST_RELATED_QUERY_CHUNK_SIZE)) { + agentRows.push(...await dbOrTx.select({ id: agents.id, companyId: agents.companyId, status: agents.status }) + .from(agents).where(and(eq(agents.companyId, companyId), inArray(agents.id, chunk)))); + } const agentsById = new Map(agentRows.map((agent) => [agent.id, agent])); type PathClassification = { @@ -7641,7 +7637,7 @@ export function issueService(db: Db) { await assertNoBlockingCycles(companyId, issueId, deduped, dbOrTx); } - await notifyAttentionRetentionWork(dbOrTx, { companyId }); + await notifyIssueAttentionRetentionWork(dbOrTx, { companyId, issueId }); await dbOrTx .delete(issueRelations) .where( @@ -10627,7 +10623,8 @@ export function issueService(db: Db) { } if (values.monitorNextCheckAt) await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors); await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers); - await notifyAttentionRetentionWork(tx, { companyId }); + values.id ??= randomUUID(); + await notifyIssueAttentionRetentionWork(tx, { companyId, issueId: values.id, parentId: values.parentId, originKind: values.originKind, originId: values.originId }); const [issue] = await tx.insert(issues).values(values).returning(); await attachOwnedDraftImages(tx, issue, issue.description, { userId: issueData.createdByUserId, agentId: issueData.createdByAgentId }); if (issue.visibility === "private" && !inheritedPrivateSubtree && issueData.createdByAgentId) { @@ -11513,7 +11510,7 @@ export function issueService(db: Db) { || issueData.parentId !== undefined || issueData.executionWorkspaceId !== undefined) { await notifyTerminalWorkspaceWork(tx, `issue:${id}`); } - await notifyAttentionRetentionWork(tx, { companyId: receiptExisting.companyId }); + await notifyIssueAttentionRetentionWork(tx, { companyId: receiptExisting.companyId, issueId: id, parentId: patch.parentId, originKind: patch.originKind, originId: patch.originId }); if (patch.status !== undefined || patch.assigneeAgentId !== undefined || patch.assigneeUserId !== undefined || patch.hiddenAt !== undefined || patch.conversationAgentId !== undefined) { await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers); @@ -11975,7 +11972,7 @@ export function issueService(db: Db) { await requestRestorePointDeletion(tx, { sourceIssueId: id }); const [owner] = await tx.select({ companyId: issues.companyId }).from(issues).where(eq(issues.id, id)); if (owner) { - await notifyAttentionRetentionWork(tx, { companyId: owner.companyId }); + await notifyIssueAttentionRetentionWork(tx, { companyId: owner.companyId, issueId: id }); await notifyExternalConversationSettlement(tx, { companyId: owner.companyId }); await notifyApprovalWork(tx, owner.companyId); } @@ -12108,7 +12105,7 @@ export function issueService(db: Db) { ) : isNull(issues.executionRunId); const updated = await db.transaction(async tx => { - await notifyAttentionRetentionWork(tx, { companyId: issueCompany.companyId }); + await notifyIssueAttentionRetentionWork(tx, { companyId: issueCompany.companyId, issueId: id }); await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors); await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers); return tx @@ -12238,7 +12235,7 @@ export function issueService(db: Db) { } const staleExecutionRunId = current.executionRunId; const adopted = await db.transaction(async tx => { - await notifyAttentionRetentionWork(tx, { companyId: issueCompany.companyId }); + await notifyIssueAttentionRetentionWork(tx, { companyId: issueCompany.companyId, issueId: id }); await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors); await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers); return tx @@ -12494,7 +12491,7 @@ export function issueService(db: Db) { existing.status === "in_progress" ? "todo" : existing.status; await notifyTerminalWorkspaceWork(tx, `issue:${id}`); await notifyNativeStatusReassessment(tx, { companyId: existing.companyId, issueId: id }); - await notifyAttentionRetentionWork(tx, { companyId: existing.companyId }); + await notifyIssueAttentionRetentionWork(tx, { companyId: existing.companyId, issueId: id }); await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors); await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers); const updated = await tx @@ -12547,7 +12544,7 @@ export function issueService(db: Db) { patch.assigneeAgentId = null; } - await notifyAttentionRetentionWork(tx, { companyId: existing.companyId }); + await notifyIssueAttentionRetentionWork(tx, { companyId: existing.companyId, issueId: id }); await notifyTerminalWorkspaceWork(tx, `issue:${id}`); if (existing.checkoutRunId || existing.executionRunId || options.clearAssignee) { await notifyExternalConversationSettlement(tx, { companyId: existing.companyId, issueId: id }); diff --git a/server/src/services/native-status-reassessment-notifications.ts b/server/src/services/native-status-reassessment-notifications.ts index 3a5ec8b6c37..30c1bbd098b 100644 --- a/server/src/services/native-status-reassessment-notifications.ts +++ b/server/src/services/native-status-reassessment-notifications.ts @@ -1,10 +1,10 @@ -import { notifyAttentionRetentionWork } from "./retention-work-notifications.js"; +import { notifyIssueAttentionRetentionWork } from "./retention-work-notifications.js"; import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js"; export type NativeStatusReassessmentScope = { companyId: string; issueId?: string; runId?: string; agentId?: string }; export async function notifyNativeStatusReassessment(tx: object, scope: NativeStatusReassessmentScope) { - await notifyAttentionRetentionWork(tx, { companyId: scope.companyId }); + await notifyIssueAttentionRetentionWork(tx, { companyId: scope.companyId, issueId: scope.issueId, runId: scope.runId }); return notifyDeliveryWork(tx, DELIVERY_QUEUES.nativeStatusReassessment, JSON.stringify({ companyId: scope.companyId, issueId: scope.issueId, runId: scope.runId, agentId: scope.agentId })); } diff --git a/server/src/services/retention-work-notifications.ts b/server/src/services/retention-work-notifications.ts index 0331f53f258..cee2486f54c 100644 --- a/server/src/services/retention-work-notifications.ts +++ b/server/src/services/retention-work-notifications.ts @@ -1,10 +1,54 @@ import { notifyApprovalWork } from "./approval-work-notifications.js"; +import { RECOVERY_ORIGIN_KINDS, parseIssueGraphLivenessIncidentKey } from "./recovery/origins.js"; +import { sql } from "drizzle-orm"; import type { Db } from "@paperclipai/db"; import type { AttentionSourceKind } from "@paperclipai/shared"; import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js"; -export async function notifyAttentionRetentionWork(transaction: object, scope: { companyId?: string; sourceKind?: AttentionSourceKind; sourceId?: string } = {}) { - await notifyDeliveryWork(transaction, DELIVERY_QUEUES.attentionRetention, JSON.stringify(scope)); +export async function notifyAttentionRetentionWork(transaction: object, scope: { companyId?: string; sourceKind?: AttentionSourceKind; issueId?: string; sourceId?: string } = {}) { + await notifyDeliveryWork(transaction, DELIVERY_QUEUES.attentionRetention, JSON.stringify({ + companyId: scope.companyId, sourceKind: scope.sourceKind, sourceId: scope.sourceId, issueId: scope.issueId, + })); +} + +export async function notifyIssueAttentionRetentionWork(transaction: object, + scope: { companyId: string; issueId?: string; parentId?: string | null; runId?: string; originKind?: string | null; originId?: string | null }) { + const database = transaction as Db; + const seeds = [scope.issueId, scope.parentId].filter((id): id is string => Boolean(id)); + if (!seeds.length && scope.runId) { + const rows = await database.execute(sql`select issue_id::text as id from heartbeat_runs + where company_id = ${scope.companyId}::uuid and id = ${scope.runId}::uuid and issue_id is not null + union select native_issue_id::text from heartbeat_runs + where company_id = ${scope.companyId}::uuid and id = ${scope.runId}::uuid and native_issue_id is not null + union select coalesce(context_snapshot->>'issueId', context_snapshot->>'taskId') from heartbeat_runs + where company_id = ${scope.companyId}::uuid and id = ${scope.runId}::uuid + and coalesce(context_snapshot->>'issueId', context_snapshot->>'taskId') is not null`); + for (const row of rows) seeds.push(row.id as string); + } + if (!seeds.length) return notifyAttentionRetentionWork(transaction, { companyId: scope.companyId }); + const origins = await database.execute(sql`select id::text as id, origin_kind as "originKind", origin_id as "originId" from issues + where company_id = ${scope.companyId}::uuid + and id in (${sql.join([...new Set(seeds)].map(id => sql`${id}::uuid`), sql`, `)})`); + const current = origins.find(source => source.id === scope.issueId); + origins.push({ originKind: scope.originKind ?? current?.originKind, originId: scope.originId ?? current?.originId }); + for (const source of origins) { + const incident = source.originKind === RECOVERY_ORIGIN_KINDS.issueGraphLivenessEscalation + ? parseIssueGraphLivenessIncidentKey(source.originId as string | null) : null; + if (incident?.companyId === scope.companyId) seeds.push(incident.issueId, incident.leafIssueId); + } + const affected = await database.execute(sql`with recursive affected(id) as ( + select unnest(array[${sql.join([...new Set(seeds)].map(id => sql`${id}::uuid`), sql`, `)}]) + union + select parent.id from affected current cross join lateral ( + select related_issue_id as id from issue_relations where company_id = ${scope.companyId}::uuid + and issue_id = current.id and type = 'blocks' + union select parent_id from issues where company_id = ${scope.companyId}::uuid + and id = current.id and parent_id is not null + ) parent + ) select id::text as "issueId" from affected`); + for (const row of affected) await notifyAttentionRetentionWork(transaction, { + companyId: scope.companyId, issueId: row.issueId as string, + }); } export async function notifyRetentionWork(transaction: object, kind: AttentionSourceKind, scope: { companyId?: string; sourceId?: string } = {}) { From 87ae20ae650de687ea9a2f8ca72e5a8d89050431 Mon Sep 17 00:00:00 2001 From: Devin Foley Date: Fri, 30 Oct 2026 17:04:04 -0700 Subject: [PATCH 57/70] fix(server): reconcile failed sandbox cleanup with signed absence (#18407) Reconcile failed ephemeral Cloud leases through a signed cleanup-only absence observation when local historical inspection is unavailable. Preserve failure outcomes, exact resource bindings, retention and row-version fences; grant no run-continuation authority. All 54 current-head checks/statuses passed. Greptile is 5/5, Medium, with no findings or unresolved threads. The 38 focused tests pass; both new reconciliation cases fail without the fallback. Isolated baseline and branch idle suites each pass all 225 cases. The original broader local teardown timeout is retained in the PR, and the full local run remains in progress. Cloud protocol support is already serving in staging. Exact candidate composition and hosted provider sleep/wake qualification follow this merge. Production has not been changed. Co-Authored-By: Paperclip --- doc/idle-sleep-safety.md | 11 +++ maps/agent-access-map/instance-operations.md | 8 +++ maps/feature-map/instance-operations.md | 9 +++ .../historical-cleanup-verification.test.ts | 54 ++++++++++++++- server/src/services/environment-runtime.ts | 8 ++- .../services/sandbox-absence-proof.test.ts | 52 ++++++++++++-- server/src/services/sandbox-absence-proof.ts | 68 +++++++++++++++---- 7 files changed, 190 insertions(+), 20 deletions(-) diff --git a/doc/idle-sleep-safety.md b/doc/idle-sleep-safety.md index e535edcd569..7b8ef115430 100644 --- a/doc/idle-sleep-safety.md +++ b/doc/idle-sleep-safety.md @@ -101,6 +101,17 @@ The manager permits at most two outstanding inspection RPCs across workers. Caller timeouts retain their slots until the actual worker reply or exit; timeouts cannot accumulate additional provider reads in later sweeps. +If a failed ephemeral lease has no usable local inspection path, a managed +Daytona instance can ask Cloud for a cleanup-only absence receipt. Cloud checks +the stack's historical credential organization and reads the exact allocation +with its manager credential. Only confirmed absence can produce a signed receipt +bound to the stack, company, environment, lease, provider handle and nullable run. +The separate cleanup type and audience cannot authorize run recovery. Older +Cloud servers, unsupported providers, invalid signatures, changed organizations +and failed reads leave the lease unresolved. Existing bounded cleanup batches +and retry deadlines own these reads; no new idle polling is added. An explicit +local provider observation is retained rather than replaced by this fallback. + Host-owned verification metadata binds the company, environment, resource, provider handle, optional run, and exact row version. Concurrent resume, replacement, cleanup, or configuration changes invalidate verification. diff --git a/maps/agent-access-map/instance-operations.md b/maps/agent-access-map/instance-operations.md index 3e21ef72901..f2d4250a4cb 100644 --- a/maps/agent-access-map/instance-operations.md +++ b/maps/agent-access-map/instance-operations.md @@ -62,6 +62,14 @@ authority as company-agent API access. Gaps: Instance start/stop and task workspace start/stop are different capabilities. Local service control is not agent company authority. +Historical Cloud sandbox cleanup remains server-owned. Cleanup-only absence +reads authenticate the tenant host with its server capability; they expose no +new board or agent endpoint. Their signed scope cannot authorize task recovery. +[Runtime and idle-gate regressions](../../server/src/__tests__/historical-cleanup-verification.test.ts) +cover reconciliation, while [receipt checks](../../server/src/services/sandbox-absence-proof.test.ts) +cover signature, identity, freshness and separation from recovery authority. +These checks do not establish a deployed sleep/wake result. + ## `configuration` Capability ID: `instance-operations/configuration`. diff --git a/maps/feature-map/instance-operations.md b/maps/feature-map/instance-operations.md index 7a01c24ebf6..dd43901323e 100644 --- a/maps/feature-map/instance-operations.md +++ b/maps/feature-map/instance-operations.md @@ -61,6 +61,15 @@ served by the admitted application before it finishes. Repeat with a disconnect before headers arrive. Confirm that neither leaves a permanent request hold. An unfinished application write must still prevent sleep after its client leaves. +Automated: [historical cleanup](../../server/src/__tests__/historical-cleanup-verification.test.ts) +exercises the runtime inspection path for failed Cloud sandbox leases, including +verification leases without a run. A signed cleanup-only absence observation +can remove this blocker without changing failed outcomes or granting run +continuation. An unresolved provider resource still prevents idle sleep. +Manual: on an owned Cloud staging workspace, confirm the exact published build, +reconcile failed test allocations, then verify actual provider suspension and +normal hosted entry after wake. Local tests do not prove that hosted journey. + ### `settings-backup` Cloud startup reads nonsecret application choices from a private, versioned diff --git a/server/src/__tests__/historical-cleanup-verification.test.ts b/server/src/__tests__/historical-cleanup-verification.test.ts index 37bc2dd6692..1269c02f1a2 100644 --- a/server/src/__tests__/historical-cleanup-verification.test.ts +++ b/server/src/__tests__/historical-cleanup-verification.test.ts @@ -1,8 +1,9 @@ -import { randomUUID } from "node:crypto"; +import { generateKeyPairSync, randomUUID, sign } from "node:crypto"; import { eq, sql } from "drizzle-orm"; import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest"; import { agents, companies, createDb, environmentCustomImageSetupSessions, environmentLeases, environments, heartbeatRuns } from "@paperclipai/db"; import { historicalCleanupVerificationService } from "../services/historical-cleanup-verification.js"; +import { environmentRuntimeService } from "../services/environment-runtime.js"; import { withoutCleanupVerification } from "../services/cleanup-verification-proof.js"; import { idleWorkSnapshot } from "../services/task-admission.js"; import { readIdleSleepSafety, type IdleSleepDrainStatus } from "../services/idle-sleep-safety.js"; @@ -16,7 +17,7 @@ if (!support.supported) console.warn(`Skipping historical cleanup database tests (support.supported ? describe : describe.skip)("historical cleanup verification", () => { let db: ReturnType, database: Awaited>; beforeAll(async () => { database = await startEmbeddedPostgresTestDatabase("historical-cleanup-"); db = createDb(database.connectionString); }, 60_000); - afterEach(async () => { await db.execute(sql`SET client_min_messages = warning; TRUNCATE companies, plugins, environments CASCADE`); }); + afterEach(async () => { vi.unstubAllEnvs(); vi.unstubAllGlobals(); await db.execute(sql`SET client_min_messages = warning; TRUNCATE companies, plugins, environments CASCADE`); }); afterAll(async () => { await db?.$client.end(); await database?.cleanup(); }); const read = () => readIdleSleepSafety(db, owned, () => now, ownerId, async () => "none", undefined, undefined, true, true); const state = async () => (await read()).backgroundWork; @@ -37,6 +38,55 @@ if (!support.supported) console.warn(`Skipping historical cleanup database tests inspectSetup: async row => ({ providerLeaseId: row.providerLeaseId, state: setupState }), }); } + it.each([false, true])("reconciles a legacy Cloud lease through signed absence and permits idle sleep (ad-hoc %s)", async adHoc => { + const f = await seed(adHoc); + const { privateKey, publicKey } = generateKeyPairSync("ed25519"); + for (const [key, value] of Object.entries({ PAPERCLIP_CLOUD_STACK_ID: "fixture-stack", + PAPERCLIP_CLOUD_API_ORIGIN: "https://cloud.example.test", PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN: "fleet-fixture", + PAPERCLIP_PRIMARY_HOST: "fixture.paperclip.test", PAPERCLIP_AUTH_SECRET: "fixture-secret", + PAPERCLIP_CLOUD_RUNTIME_IDENTITY_JWKS: JSON.stringify({ keys: [{ ...publicKey.export({ format: "jwk" }), kid: "fixture", alg: "EdDSA", use: "sig" }] }) })) vi.stubEnv(key, value); + let absent = false; + const fetchImpl = vi.fn(async (_url: string, init: RequestInit) => { + if (!absent) return Response.json({ absent: false }); + const binding = JSON.parse(init.body as string), seconds = Math.floor(Date.now() / 1000); + expect(binding).toEqual({ purpose: "cleanup", companyId: f.companyId, environmentId: f.environment.id, + leaseId: f.lease.id, heartbeatRunId: adHoc ? null : f.runId, provider: "daytona", providerLeaseId: f.lease.providerLeaseId }); + const header = Buffer.from(JSON.stringify({ alg: "EdDSA", kid: "fixture", typ: "paperclip-sandbox-cleanup-absence+jwt" })).toString("base64url"); + const claims = Buffer.from(JSON.stringify({ v: 1, iss: "paperclip-cloud", aud: "paperclip-sandbox-cleanup-absence/v1", + sub: "fixture-stack", ...binding, absent: true, iat: seconds, exp: seconds + 60 })).toString("base64url"); + return Response.json({ receipt: `${header}.${claims}.${sign(null, Buffer.from(`${header}.${claims}`), privateKey).toString("base64url")}` }); + }); + vi.stubGlobal("fetch", fetchImpl); + const runtime = environmentRuntimeService(db); + const service = historicalCleanupVerificationService(db, { + inspectLease: row => runtime.inspectLeaseResource(row.id), inspectSetup: async () => undefined, + }); + expect(await state()).toBe("present"); + expect(await service.sweep()).toMatchObject({ checked: 1, deferred: 1, verified: 0 }); + expect(await state()).toBe("present"); + absent = true; + expect(await service.sweep()).toMatchObject({ checked: 1, verified: 1, failed: 0 }); + expect(await state()).toBe("none"); + const [current] = await db.select().from(environmentLeases); + expect(current).toMatchObject({ status: "failed", cleanupStatus: "success", heartbeatRunId: adHoc ? null : f.runId }); + expect(current!.metadata).not.toHaveProperty("remoteExecutionTermination"); + expect(await service.sweep()).toMatchObject({ checked: 0 }); + expect(fetchImpl).toHaveBeenCalledTimes(2); + }); + it.each(["active", "stopped", "unknown"] as const)("does not replace an explicit %s provider observation with a Cloud read", async providerState => { + await seed(true); + for (const [key, value] of Object.entries({ PAPERCLIP_CLOUD_STACK_ID: "fixture-stack", + PAPERCLIP_CLOUD_API_ORIGIN: "https://cloud.example.test", PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN: "fleet-fixture", + PAPERCLIP_PRIMARY_HOST: "fixture.paperclip.test", PAPERCLIP_AUTH_SECRET: "fixture-secret" })) vi.stubEnv(key, value); + const fetchImpl = vi.fn(); vi.stubGlobal("fetch", fetchImpl); + const runtime = environmentRuntimeService(db, { drivers: [{ driver: "sandbox", + acquireRunLease: async () => { throw new Error("unexpected acquire"); }, releaseRunLease: async () => null, + inspectLeaseResource: async ({ lease }) => ({ providerLeaseId: lease.providerLeaseId!, state: providerState }), + }] }); + const service = historicalCleanupVerificationService(db, { inspectLease: row => runtime.inspectLeaseResource(row.id), inspectSetup: async () => undefined }); + expect(await service.sweep()).toMatchObject({ checked: 1, verified: 0, deferred: 1 }); + expect(await state()).toBe("present"); expect(fetchImpl).not.toHaveBeenCalled(); + }); it.each([false, true])("accepts exact deletion proof without changing outcomes or granting run authority (ad-hoc %s)", async adHoc => { const f = await seed(adHoc); expect(await state()).toBe("present"); const service = verifier(); expect(await service.sweep()).toMatchObject({ checked: 1, verified: 1, failed: 0 }); diff --git a/server/src/services/environment-runtime.ts b/server/src/services/environment-runtime.ts index 277a5a0254c..33f161457f9 100644 --- a/server/src/services/environment-runtime.ts +++ b/server/src/services/environment-runtime.ts @@ -15,6 +15,7 @@ import { hasStopOnlyCleanup, prepareSandboxStopAndRetain, readStopOnlyCleanup, s import { JsonRpcCallError, readEnvironmentAcquisitionDiagnostic, readEnvironmentCreationCleanupError } from "@paperclipai/plugin-sdk"; import { preserveEnvironmentSyncOutErrorDiagnostic } from "./environment-sync-out-error.js"; import { remoteTerminationReceipt } from "./remote-execution-termination.js"; +import { inspectCloudSandboxCleanupAbsence } from "./sandbox-absence-proof.js"; import { hasNativeWorkspaceExportResume, releaseCompletedNativeWorkspaceExportRetention } from "./native-runtime/native-workspace-export-resume.js"; import { createHash, randomUUID } from "node:crypto"; import { and, eq, inArray, ne, or, sql } from "drizzle-orm"; @@ -4021,7 +4022,12 @@ export function environmentRuntimeService( const lease = await environmentsSvc.getLeaseById(leaseId); if (!lease || lease.status !== "failed" || lease.leasePolicy !== "ephemeral" || !lease.providerLeaseId) return undefined; const environment = lease.environmentId ? await environmentsSvc.getById(lease.environmentId) : null; - return await getDriver(getLeaseDriverKey(lease, environment))?.inspectLeaseResource?.({ environment, lease }); + // Legacy Cloud leases can lack a credential version that permits a + // provider-local observation. Cloud's manager verifies the historical + // organization and signs only exact absence; no replacement key or + // cleanup label is treated as proof. Explicit provider answers win. + return await getDriver(getLeaseDriverKey(lease, environment))?.inspectLeaseResource?.({ environment, lease }) + ?? await inspectCloudSandboxCleanupAbsence(lease); }, async retryPendingSandboxTeardown(input: { diff --git a/server/src/services/sandbox-absence-proof.test.ts b/server/src/services/sandbox-absence-proof.test.ts index d145425137e..82a7a3cfbba 100644 --- a/server/src/services/sandbox-absence-proof.test.ts +++ b/server/src/services/sandbox-absence-proof.test.ts @@ -1,6 +1,7 @@ import { createHmac, generateKeyPairSync, randomUUID, sign } from "node:crypto"; import { describe, expect, it, vi } from "vitest"; -import { requestSandboxAbsenceProof, verifySandboxAbsenceReceipt } from "./sandbox-absence-proof.js"; +import { inspectCloudSandboxCleanupAbsence, requestSandboxAbsenceProof, verifySandboxAbsenceReceipt, + verifySandboxCleanupAbsenceReceipt } from "./sandbox-absence-proof.js"; const { privateKey, publicKey } = generateKeyPairSync("ed25519"); const binding = { companyId: randomUUID(), issueId: randomUUID(), runId: randomUUID(), leaseId: randomUUID(), @@ -9,10 +10,10 @@ const env = { PAPERCLIP_CLOUD_STACK_ID: "fixture-stack", PAPERCLIP_CLOUD_API_ORI PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN: "fleet-token", PAPERCLIP_PRIMARY_HOST: "fixture.paperclip.test", PAPERCLIP_AUTH_SECRET: "fixture-secret", PAPERCLIP_CLOUD_RUNTIME_IDENTITY_JWKS: JSON.stringify({ keys: [{ ...publicKey.export({ format: "jwk" }), kid: "fixture", alg: "EdDSA", use: "sig" }] }) }; const now = new Date("2099-01-01T00:00:00Z"), seconds = now.getTime() / 1000; -function receipt(overrides: Record = {}) { - const header = Buffer.from(JSON.stringify({ alg: "EdDSA", kid: "fixture", typ: "paperclip-sandbox-absence+jwt" })).toString("base64url"); - const payload = Buffer.from(JSON.stringify({ v: 1, iss: "paperclip-cloud", aud: "paperclip-sandbox-absence/v1", sub: "fixture-stack", - ...binding, absent: true, iat: seconds, exp: seconds + 60, ...overrides })).toString("base64url"); +function receipt(overrides: Record = {}, purpose = "sandbox-absence", source: Record = binding) { + const header = Buffer.from(JSON.stringify({ alg: "EdDSA", kid: "fixture", typ: `paperclip-${purpose}+jwt` })).toString("base64url"); + const payload = Buffer.from(JSON.stringify({ v: 1, iss: "paperclip-cloud", aud: `paperclip-${purpose}/v1`, sub: "fixture-stack", + ...source, absent: true, iat: seconds, exp: seconds + 60, ...overrides })).toString("base64url"); return `${header}.${payload}.${sign(null, Buffer.from(`${header}.${payload}`), privateKey).toString("base64url")}`; } describe("sandbox absence proof", () => { @@ -45,3 +46,44 @@ describe("sandbox absence proof", () => { expect(fetchImpl).not.toHaveBeenCalled(); }); }); + +describe("cleanup-only sandbox absence", () => { + const cleanup = { purpose: "cleanup" as const, companyId: binding.companyId, environmentId: randomUUID(), + leaseId: binding.leaseId, heartbeatRunId: null as string | null, provider: binding.provider, providerLeaseId: binding.providerLeaseId }; + const lease = { id: cleanup.leaseId, ...cleanup }; + const signed = (overrides: Record = {}) => receipt(overrides, "sandbox-cleanup-absence", cleanup); + it.each([null, binding.runId])("accepts exact absence with nullable run %s without issuing a run receipt", async heartbeatRunId => { + const source = { ...cleanup, heartbeatRunId }; + const fetchImpl = vi.fn().mockResolvedValue(Response.json({ receipt: receipt({}, "sandbox-cleanup-absence", source) })); + expect(await inspectCloudSandboxCleanupAbsence({ ...lease, heartbeatRunId }, { env, now, fetchImpl })) + .toEqual({ providerLeaseId: cleanup.providerLeaseId, state: "destroyed" }); + const [url, init] = fetchImpl.mock.calls[0]; + expect(url).toBe("https://cloud.example.test/v1/tenant/sandbox-absence"); + expect(JSON.parse(init.body)).toEqual(source); + expect(init.headers.authorization).not.toContain(env.PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN); + expect(init.redirect).toBe("error"); + }); + it("requires every binding, current stack, signature and freshness, and rejects recovery receipts", () => { + expect(verifySandboxCleanupAbsenceReceipt(signed(), cleanup, env, now)).toMatchObject(cleanup); + for (const overrides of [ + ...Object.keys(cleanup).map(key => ({ [key]: randomUUID() })), + { sub: "foreign-stack" }, { absent: false }, { aud: "paperclip-sandbox-absence/v1" }, { iss: "other" }, + { iat: seconds + 31 }, { exp: seconds }, { exp: seconds + 61 }, + ]) expect(verifySandboxCleanupAbsenceReceipt(signed(overrides), cleanup, env, now)).toBeNull(); + expect(verifySandboxCleanupAbsenceReceipt(signed().slice(0, -8) + "tampered", cleanup, env, now)).toBeNull(); + expect(verifySandboxCleanupAbsenceReceipt(receipt(), cleanup, env, now)).toBeNull(); + expect(verifySandboxAbsenceReceipt(signed(), binding, env, now)).toBeNull(); + }); + it("keeps old Clouds, outages, invalid replies and unconfigured instances inconclusive", async () => { + for (const answer of [{ absent: false }, { absent: true }, { receipt: "forged" }, { receipt: signed({ heartbeatRunId: binding.runId }) }]) + expect(await inspectCloudSandboxCleanupAbsence(lease, { env, now, fetchImpl: vi.fn().mockResolvedValue(Response.json(answer)) })).toBeUndefined(); + for (const status of [400, 401, 403, 503]) expect(await inspectCloudSandboxCleanupAbsence(lease, + { env, now, fetchImpl: vi.fn().mockResolvedValue(new Response(null, { status })) })).toBeUndefined(); + expect(await inspectCloudSandboxCleanupAbsence(lease, { env, now, fetchImpl: vi.fn().mockRejectedValue(new Error("offline")) })).toBeUndefined(); + const fetchImpl = vi.fn(); + expect(await inspectCloudSandboxCleanupAbsence(lease, { env: {}, now, fetchImpl })).toBeUndefined(); + for (const change of [{ provider: "other" }, { environmentId: null }, { providerLeaseId: "invalid" }]) + expect(await inspectCloudSandboxCleanupAbsence({ ...lease, ...change }, { env, now, fetchImpl })).toBeUndefined(); + expect(fetchImpl).not.toHaveBeenCalled(); + }); +}); diff --git a/server/src/services/sandbox-absence-proof.ts b/server/src/services/sandbox-absence-proof.ts index 8ae852688ed..ae6044dd0af 100644 --- a/server/src/services/sandbox-absence-proof.ts +++ b/server/src/services/sandbox-absence-proof.ts @@ -7,8 +7,18 @@ const bindingSchema = z.object({ companyId: z.string().uuid(), issueId: z.string().uuid(), runId: z.string().uuid(), leaseId: z.string().uuid(), ownerToken: z.string().uuid(), provider: z.literal("daytona"), providerLeaseId: z.string().uuid(), -}); +}).strict(); +const cleanupBindingSchema = z.object({ + purpose: z.literal("cleanup"), companyId: z.string().uuid(), environmentId: z.string().uuid(), + leaseId: z.string().uuid(), heartbeatRunId: z.string().uuid().nullable(), + provider: z.literal("daytona"), providerLeaseId: z.string().uuid(), +}).strict(); export type SandboxAbsenceBinding = z.infer; +export type SandboxCleanupAbsenceBinding = z.infer; +export type SandboxCleanupAbsenceProof = SandboxCleanupAbsenceBinding & { + v: 1; iss: "paperclip-cloud"; aud: "paperclip-sandbox-cleanup-absence/v1"; sub: string; + absent: true; iat: number; exp: number; +}; export type SandboxAbsenceProof = SandboxAbsenceBinding & { v: 1; iss: "paperclip-cloud"; aud: "paperclip-sandbox-absence/v1"; sub: string; absent: true; iat: number; exp: number; @@ -16,19 +26,30 @@ export type SandboxAbsenceProof = SandboxAbsenceBinding & { export function verifySandboxAbsenceReceipt(receipt: string, binding: SandboxAbsenceBinding, env: NodeJS.ProcessEnv = process.env, now = new Date()): SandboxAbsenceProof | null { + const parsed = bindingSchema.safeParse(binding); + return parsed.success ? verifyReceipt(receipt, parsed.data, "sandbox-absence", env, now) as SandboxAbsenceProof | null : null; +} + +export function verifySandboxCleanupAbsenceReceipt(receipt: string, binding: SandboxCleanupAbsenceBinding, + env: NodeJS.ProcessEnv = process.env, now = new Date()): SandboxCleanupAbsenceProof | null { + const parsed = cleanupBindingSchema.safeParse(binding); + return parsed.success ? verifyReceipt(receipt, parsed.data, "sandbox-cleanup-absence", env, now) as SandboxCleanupAbsenceProof | null : null; +} + +function verifyReceipt(receipt: string, binding: Record, purpose: string, + env: NodeJS.ProcessEnv, now: Date): Record | null { try { - const parsed = bindingSchema.safeParse(binding); - if (receipt.length > 16_000 || !parsed.success) return null; + if (receipt.length > 16_000) return null; const parts = receipt.split("."); if (parts.length !== 3 || parts.some(part => !/^[A-Za-z0-9_-]+$/.test(part))) return null; const header = JSON.parse(Buffer.from(parts[0], "base64url").toString("utf8")); - if (header.alg !== "EdDSA" || header.typ !== "paperclip-sandbox-absence+jwt" || typeof header.kid !== "string") return null; + if (header.alg !== "EdDSA" || header.typ !== `paperclip-${purpose}+jwt` || typeof header.kid !== "string") return null; if (!verify(null, Buffer.from(`${parts[0]}.${parts[1]}`, "ascii"), cloudRuntimePublicKeyForKid(env, header.kid), Buffer.from(parts[2], "base64url"))) return null; - const proof = JSON.parse(Buffer.from(parts[1], "base64url").toString("utf8")) as SandboxAbsenceProof; + const proof = JSON.parse(Buffer.from(parts[1], "base64url").toString("utf8")); const seconds = Math.floor(now.getTime() / 1000); - if (proof.v !== 1 || proof.iss !== "paperclip-cloud" || proof.aud !== "paperclip-sandbox-absence/v1" + if (proof.v !== 1 || proof.iss !== "paperclip-cloud" || proof.aud !== `paperclip-${purpose}/v1` || !env.PAPERCLIP_CLOUD_STACK_ID || proof.sub !== env.PAPERCLIP_CLOUD_STACK_ID || proof.absent !== true - || Object.keys(parsed.data).some(key => proof[key as keyof SandboxAbsenceBinding] !== parsed.data[key as keyof SandboxAbsenceBinding]) + || Object.keys(binding).some(key => proof[key] !== binding[key]) || !Number.isInteger(proof.iat) || !Number.isInteger(proof.exp) || proof.iat > seconds + 30 || proof.exp <= seconds || proof.exp <= proof.iat || proof.exp - proof.iat > 60) return null; return proof; @@ -38,11 +59,35 @@ export function verifySandboxAbsenceReceipt(receipt: string, binding: SandboxAbs export async function requestSandboxAbsenceProof(binding: SandboxAbsenceBinding, options: { env?: NodeJS.ProcessEnv; fetchImpl?: typeof fetch; now?: Date; } = {}): Promise<{ proof: SandboxAbsenceProof; receipt: string } | null> { + const parsed = bindingSchema.safeParse(binding); + if (!parsed.success) return null; + const receipt = await requestReceipt(parsed.data, options); + if (!receipt) return null; + const proof = verifySandboxAbsenceReceipt(receipt, parsed.data, options.env, options.now); + return proof ? { proof, receipt } : null; +} + +/** Cleanup-only observation: never grants continuation authority to a run. */ +export async function inspectCloudSandboxCleanupAbsence(lease: { + id: string; companyId: string; environmentId: string | null; heartbeatRunId: string | null; + provider: string | null; providerLeaseId: string | null; +}, options: { env?: NodeJS.ProcessEnv; fetchImpl?: typeof fetch; now?: Date } = {}) { + const parsed = cleanupBindingSchema.safeParse({ purpose: "cleanup", companyId: lease.companyId, + environmentId: lease.environmentId, leaseId: lease.id, heartbeatRunId: lease.heartbeatRunId, + provider: lease.provider, providerLeaseId: lease.providerLeaseId }); + if (!parsed.success) return undefined; + const receipt = await requestReceipt(parsed.data, options); + if (!receipt || !verifySandboxCleanupAbsenceReceipt(receipt, parsed.data, options.env, options.now)) return undefined; + return { providerLeaseId: parsed.data.providerLeaseId, state: "destroyed" as const }; +} + +async function requestReceipt(binding: Record, options: { + env?: NodeJS.ProcessEnv; fetchImpl?: typeof fetch; +}): Promise { const env = options.env ?? process.env; const context = getCloudStackContext(env); const secret = env.PAPERCLIP_AUTH_SECRET ?? env.BETTER_AUTH_SECRET; - const parsed = bindingSchema.safeParse(binding); - if (!context?.stackId || !context.primaryHost || !context.cloudOrigin || !secret?.trim() || !parsed.success) return null; + if (!context?.stackId || !context.primaryHost || !context.cloudOrigin || !secret?.trim()) return null; try { const origin = new URL(context.cloudOrigin); const host = new URL(`https://${context.primaryHost}`); @@ -51,12 +96,11 @@ export async function requestSandboxAbsenceProof(binding: SandboxAbsenceBinding, const response = await (options.fetchImpl ?? fetch)(`${origin.origin}/v1/tenant/sandbox-absence`, { method: "POST", redirect: "error", signal: AbortSignal.timeout(10_000), headers: { authorization: `Bearer ${capability}`, "x-paperclip-cloud-tenant-host": host.hostname, "content-type": "application/json" }, - body: JSON.stringify(parsed.data), + body: JSON.stringify(binding), }); if (!response.ok) return null; const payload = await response.json() as { receipt?: unknown }; if (typeof payload.receipt !== "string") return null; - const proof = verifySandboxAbsenceReceipt(payload.receipt, binding, env, options.now); - return proof ? { proof, receipt: payload.receipt } : null; + return payload.receipt; } catch { return null; } } From 493e229d93ba7798e80152f4549e2e9177ce50ae Mon Sep 17 00:00:00 2001 From: Nicky Leach Date: Fri, 30 Oct 2026 17:07:32 -0700 Subject: [PATCH 58/70] perf(runner): cache verified native executable digests (#18390) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Thinking Path > - Paperclip manages AI agents and their task execution. > - Runner checks qualified native programs before it starts a provider. > - Installation checks and command opening can hash the same program again. > - A new sidecar process repeats this work on a later warm task. > - This pull request stores the computed digest in a private filesystem cache. > - Every cache hit still checks the independently selected trusted digest and current file metadata. > - An unchanged program can start without another full native file read. ## Linked Issues or Issue Description **What existing behavior does this improve?** Repeated **qualified ACPX** native executable admission in Runner. Claude, ACPX-Codex, and Grok use this verifier in both Core-owned and plugin-backed launches. The ordinary native Codex app-server launcher is a different path and is not changed. A retained provider that opens no new native command has no saved hash work here. The measured Claude task path does not require a comment backlog. **Current behavior** Installation verification and final command opening each read and hash the native executable. The qualified Linux x64 Claude file is 241,667,256 bytes. A new verification and command lease reads 483,334,512 native bytes. Another Node process repeats both reads. **Proposed behavior** Store the actual computed SHA-256 digest by device, inode, size, nanosecond modification time, and nanosecond change time. Reuse it across Runner and sidecar processes. Compare each hit with the current build-owned digest. Hash the file on a miss or changed identity. This replaces the earlier Claude-only deferral design in this same PR. **Reason and benefit** The first verification and command lease reads the native file once. Later processes read none of its native bytes while its identity stays the same. Related: #16372 established qualified executable admission. This change is independent of other preparation PRs. It does not change provider pins or profile versions. ## What Changed - Add a shared filesystem digest cache under the absolute OS temporary root at `paperclip-runner-/executable-digests-v1`. - Use private directories and files, versioned hash-only filenames, bounded JSON entries, and atomic temporary-file replacement. Cache files contain a version and the computed digest. They contain no credentials or source paths. - Fall back to full hashing when the cache is missing, unreadable, malformed, or unusable. Keep current expected-digest selection outside the cache. - Keep cache data outside task homes. Neither `HOME` nor task cwd selects the path. The OS temporary-root selection honors absolute platform `TMPDIR`, `TMP`, or `TEMP` overrides. A relative root disables caching. Hosts must keep that root outside task workspaces. Temp cleanup causes a later rehash. - Keep the no-follow executable open, regular-file and size checks, execute permission, descriptor ownership, before/after identity checks, and preparation-to-launch identity comparison. Cache publication occurs before the final identity checks. - Use this path for standalone installation checks and final command opening. Remove the earlier pending-digest type and Claude-only launch helper. Runtime host and profile selection keep their original structure. - Test changed content, all fingerprint fields, symlinks, mode changes, inode replacement, mutation during lookup/publication, concurrent cache writers, wrong trusted pins, and cache failures. - Include a deliberate trust-boundary test: a process running as the same OS user can plant a matching cached digest for changed bytes. It opens a test descriptor and executes nothing. In plain language: Runner remembers the hash for the same file. It still checks which file it opened and which hash the build requires. The cache trusts the OS user who can write it. ## Verification From `packages/paperclip-runner`: - `../../node_modules/.bin/vitest run src/drivers/acpx/executable-digest-cache.test.ts src/drivers/acpx/installation-integrity.test.ts src/drivers/acpx/runtime-host.test.ts` — 177 passed, 6 platform-specific skips. - `../../node_modules/.bin/tsc -p tsconfig.json --noEmit` — passed. - `git diff --check` and the added-line secret, PII, private-address, and code-comment scan — passed. The offline comparison uses baseline `45a3a92e6dc52d20923b9aa34a22fee2ec5443f9` and treatment `c4356df12032448de90516ae329d8f939af10984`. Each sample starts a different Node process. Each normal sample runs the actual qualified installation verifier and opens a command lease. Seven samples per case follow one warmup. The cache is removed before each cold case. The next process reuses its entry. Baseline order alternates around the treatment pair. Command digest and package paths match in every pair. | Verifier plus command opening | Baseline | Cold cache | Later process hit | | --- | ---: | ---: | ---: | | Full native reads | 2 | 1 | 0 | | Native bytes read | 483,334,512 | 241,667,256 | 0 | | Local median, ms | 1041.26 | 602.70 | 130.17 | | Local range, ms | 994.20–1133.11 | 580.27–999.83 | 121.96–179.45 | | Median whole child process, ms | 1379.96 | 988.75 | 523.01 | Raw verifier-plus-open samples in ms: - Baseline: 1133.11, 1074.40, 1061.99, 1041.26, 994.20, 1036.12, 1037.70. - Cold: 647.13, 646.33, 999.83, 584.83, 580.27, 583.37, 602.70. - Later process hit: 179.45, 138.37, 134.39, 130.17, 121.96, 122.33, 126.68. Separate direct-opener fault probes use a disposable copy of that same native file: | Change | Native bytes read | Result | Local open, ms | | --- | ---: | --- | ---: | | First copied file | 241,667,256 | Accepted | 526.14 | | Next process, different HOME and task cwd | 0 | Accepted | 24.91 | | Changed timestamps | 241,667,256 | Accepted after hashing | 531.40 | | New inode, identical bytes | 241,667,256 | Accepted after hashing | 536.70 | | Corrupt byte, same size and exactly restored mtime | 241,667,256 | Rejected | Whole process: 841.36 | | Different independent trusted pin | 0 | Rejected | Whole process: 374.76 | The corruption probe keeps `mtimeNs` exactly equal before and after, at `1000000000000000000`. Its changed `ctimeNs` invalidates the entry. Cache expiry is not required. The changed-pin probe rejects the old actual digest without a native read. The cached image is `sha256:99ce80690a0c13328f41fd84156cc73e1ac562eaed3120fd41d04d7acdfb325d`. The Node binary is `/opt/paperclip/runner/libexec/node`, version `v24.20.0`. The artifact is Claude SDK `0.3.286`, with expected digest `sha256:fe503f65c6289d59c23e5b21ae44f03583f997dd33a2cbfc75ab4f96fb8fc73f`. The container used a read-only root, a disposable writable tmpfs, and disabled network. No provider process, model request, credential, or deployment was used. **These elapsed values are diagnostics from Linux x64 CPU emulation on an ARM64 Docker host. They are not production latency or TTFT estimates.** The exact byte counts, cross-process reuse, output equality, and invalidation results are the portable evidence. The saved phase 0 exports have no directly comparable native executable hashing span. Their broad `runner.transport.attach` spans include other work. A production comparison within 20% is therefore unavailable for this seam. The earlier local 919 ms control was also an emulation diagnostic; it is not a production baseline. The initial Docker attempt was denied access to the local socket by the sandbox. The same offline command then ran with local socket permission. An initial fault probe restored mtime through a `Date`, which did not prove exact nanosecond equality. Its raw report is retained as historical; the final probe sets and verifies an exact mtime before and after corruption. The prior Claude-only implementation and its checks and measurements remain historical evidence. They do not qualify this replacement. The superseded home-directory cache head had one browser recovery test timeout in its GitHub workflow. It expected a saved title after session recovery. The failure is retained. No browser code was changed; the current head reruns CI after moving the cache outside task homes. No full local CI suite was run, as requested. GitHub CI supplies full regression coverage. Current head `c4356df12032448de90516ae329d8f939af10984` passed the complete [GitHub workflow](https://github.com/paperclipai/paperclip/actions/runs/41145223331), including all browser shards. Its latest checks have 56 successes and 2 intended skips, including both security scans started when the PR became ready. Standard Greptile reviewed this exact head at 5/5 with no open findings. Apex then reviewed the same head at 5/5 under the documented OS-user trust policy. It reported no new findings. There are no unresolved review threads. The PR is ready for review and remains unmerged. ## Risks - This deliberately weakens protection against the same OS user. That user can write the cache and can plant a chosen digest for a file fingerprint. A cache hit trusts that record; metadata identity is not a fresh content hash. The explicit fault test documents this accepted policy. - A read-only administrator-owned Runner can use this user-writable cache. The cache has no signature, secret, or daemon. It does not require a Launcher or ROM change. - Other users cannot use normal cache permissions to write these entries. Cache errors fall back to hashing. Removing the cache directory restores full hashing. Old fingerprints leave small unused entries until the cache is removed or OS temp cleanup evicts it. Temp eviction costs one later full hash. - A changed file normally rehashes. A digest that differs from the independent current pin still fails admission. Existing command, package, credential, process-lifetime, and final dispatch checks remain. - No schema, public protocol, package dependency, profile version, or provider pin changes. ## Model Used OpenAI Codex, GPT-6 family, with reasoning, code editing, test execution, and tool use. This session does not expose a more precise serving model ID or context-window capacity. ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used with the details available to this session - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above - [x] I have described the issue in-PR following the enhancement template - [x] I have not referenced internal or instance-local Paperclip issues or links - [x] My branch name describes the change and contains no internal ticket ID - [x] The targeted local tests pass; the full local suite was not run as requested - [x] I have added or updated tests where applicable - [x] I have updated relevant documentation - [x] I have considered and documented risks - [x] All Paperclip CI gates are green for the replacement head - [x] Standard Greptile is 5/5 with no open findings for the replacement head - [x] Apex is 5/5 on the current head with no unresolved findings - [x] I will address review findings before requesting merge --------- Co-authored-by: Paperclip --- .../paperclip-runner/docs/durable-recovery.md | 29 ++++ .../acpx/executable-digest-cache.test.ts | 160 ++++++++++++++++++ .../drivers/acpx/installation-integrity.ts | 52 +++--- .../src/executable-digest-cache.ts | 70 ++++++++ 4 files changed, 286 insertions(+), 25 deletions(-) create mode 100644 packages/paperclip-runner/src/drivers/acpx/executable-digest-cache.test.ts create mode 100644 packages/paperclip-runner/src/executable-digest-cache.ts diff --git a/packages/paperclip-runner/docs/durable-recovery.md b/packages/paperclip-runner/docs/durable-recovery.md index 0fbef29f508..f04aab08a23 100644 --- a/packages/paperclip-runner/docs/durable-recovery.md +++ b/packages/paperclip-runner/docs/durable-recovery.md @@ -98,6 +98,35 @@ digest, and refuse the rest. A profile identity does not establish hosted harness qualification. [`rules/acpx-profile-identity.mdc`](../../../rules/acpx-profile-identity.mdc) summarizes these practices for contributors. +Qualified ACPX native executables use Runner's persistent digest cache. The cache +stores a computed SHA-256 digest under a key made from device, inode, size, +nanosecond modification time, and nanosecond change time. Each check compares +that digest with the independently selected build-owned pin. A changed file +identity, missing entry, or unusable cache causes a full descriptor read and +hash. A hit reuses the earlier digest; metadata identity is not a fresh content +hash. Claude, Codex, and Grok use the same native executable admission path. + +Cache entries live in the operating-system temporary directory under +`paperclip-runner-/executable-digests-v1`, outside task workspaces such as +`/home/agent`. Neither `HOME` nor the task working directory selects this path. +Node's `os.tmpdir()` selects the root and honors the platform's absolute +`TMPDIR`, `TMP`, or `TEMP` override. Hosts must keep that root outside task +workspaces; a relative or unusable root disables caching. Temporary-directory +eviction removes the optimization until a later full hash fills it again. The +directory and files are private to that user. Writes use a temporary file and +atomic rename, so later Runner or sidecar processes can reuse complete entries. +Cache errors fall back to hashing. The cache stores no credentials or file paths. + +Every admission still opens the executable without following its final symlink, +checks its regular-file type, bounds, execute permission, and before/after file +identity, and retains the descriptor for launch. Package qualification and the +preparation-to-launch identity check remain. The cache deliberately trusts the +operating-system user: a process with that user's access can plant a digest for +a chosen fingerprint. It is not authenticated with a secret or signature. +An administrator-owned read-only Runner can therefore use a user-writable cache, +but this provides weaker protection against that same user than hashing every +executable on each admission. Removing the cache restores full hashing. + ## External controller checkpoints The optional controller persistence callback stores recovery control state, not diff --git a/packages/paperclip-runner/src/drivers/acpx/executable-digest-cache.test.ts b/packages/paperclip-runner/src/drivers/acpx/executable-digest-cache.test.ts new file mode 100644 index 00000000000..f33eb4fd39f --- /dev/null +++ b/packages/paperclip-runner/src/drivers/acpx/executable-digest-cache.test.ts @@ -0,0 +1,160 @@ +import { createHash } from "node:crypto"; +import { chmod, lstat, mkdir, mkdtemp, readFile, readdir, rename, rm, symlink, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { createExecutableDigestCache, executableDigestCache, type ExecutableFileIdentity } from "../../executable-digest-cache.js"; +import { openVerifiedRuntimeExecutable } from "./installation-integrity.js"; + +const temporary: string[] = []; +const identity: ExecutableFileIdentity = { device: "1", inode: "2", size: "3", modifiedNanoseconds: "4", changedNanoseconds: "5" }; +const digest = `sha256:${"a".repeat(64)}`; +afterEach(async () => { + vi.restoreAllMocks(); + await Promise.all(temporary.splice(0).map(path => rm(path, { recursive: true, force: true }))); +}); + +async function fixture() { + const root = await mkdtemp(join(tmpdir(), "runner-executable-digest-cache-")); + temporary.push(root); + const directory = join(root, "cache"); + const cache = createExecutableDigestCache(directory); + return { root, directory, cache }; +} + +describe("persistent executable digest cache", () => { + it("stores the actual digest in a private, versioned entry and survives a new cache instance", async () => { + const f = await fixture(); + expect(await f.cache.read(identity)).toBeNull(); + await f.cache.write(identity, digest); + expect(await createExecutableDigestCache(f.directory).read(identity)).toBe(digest); + const [name] = await readdir(f.directory); + expect(name).toMatch(/^v1-[a-f0-9]{64}\.json$/); + expect(JSON.parse(await readFile(join(f.directory, name), "utf8"))).toEqual({ version: 1, digest }); + expect((await lstat(f.directory)).mode & 0o777).toBe(0o700); + expect((await lstat(join(f.directory, name))).mode & 0o777).toBe(0o600); + }); + + it.each(Object.keys(identity) as (keyof ExecutableFileIdentity)[])("invalidates a changed %s", async key => { + const f = await fixture(); + await f.cache.write(identity, digest); + expect(await f.cache.read({ ...identity, [key]: "changed" })).toBeNull(); + }); + + it.each(["malformed", "version", "digest", "oversized", "permissions", "symlink"] as const)( + "treats an unusable %s entry as a miss", async kind => { + const f = await fixture(); + await f.cache.write(identity, digest); + const [name] = await readdir(f.directory), file = join(f.directory, name); + if (kind === "malformed") await writeFile(file, "{"); + else if (kind === "version") await writeFile(file, JSON.stringify({ version: 2, digest })); + else if (kind === "digest") await writeFile(file, JSON.stringify({ version: 1, digest: "expected" })); + else if (kind === "oversized") await writeFile(file, "x".repeat(1025)); + else if (kind === "permissions") await chmod(file, 0o644); + else { await rename(file, `${file}.target`); await symlink(`${file}.target`, file); } + expect(await f.cache.read(identity)).toBeNull(); + }, + ); + + it("falls back when the cache directory cannot be used and leaves existing permissions unchanged", async () => { + const f = await fixture(); + await writeFile(f.directory, "not a directory"); + await expect(f.cache.write(identity, digest)).resolves.toBeUndefined(); + expect(await f.cache.read(identity)).toBeNull(); + await rm(f.directory); + await mkdir(f.directory, { mode: 0o755 }); + await f.cache.write(identity, digest); + expect(await f.cache.read(identity)).toBeNull(); + expect((await lstat(f.directory)).mode & 0o777).toBe(0o755); + }); + + it("does not resolve a relative cache directory through the task working directory", async () => { + const cache = createExecutableDigestCache("relative-task-cache"); + await cache.write(identity, digest); + expect(await cache.read(identity)).toBeNull(); + }); + + it("publishes complete entries under concurrent writers without leftover temporary files", async () => { + const f = await fixture(); + await Promise.all(Array.from({ length: 8 }, () => f.cache.write(identity, digest))); + expect(await f.cache.read(identity)).toBe(digest); + expect(await readdir(f.directory)).toHaveLength(1); + }); +}); + +describe.skipIf(!["linux", "darwin"].includes(process.platform))("descriptor admission with cached digests", () => { + async function executable() { + const f = await fixture(), path = join(f.root, "native"), bytes = Buffer.from("qualified native fixture"); + await writeFile(path, bytes, { mode: 0o700 }); + vi.spyOn(executableDigestCache, "read").mockImplementation(f.cache.read); + vi.spyOn(executableDigestCache, "write").mockImplementation(f.cache.write); + return { ...f, path, digest: `sha256:${createHash("sha256").update(bytes).digest("hex")}` }; + } + + it("hashes a miss, reuses the actual digest, and still rejects a different independent pin", async () => { + const f = await executable(); + await (await openVerifiedRuntimeExecutable(f.path, f.digest, "fixture")).handle.close(); + expect(executableDigestCache.write).toHaveBeenCalledOnce(); + await (await openVerifiedRuntimeExecutable(f.path, f.digest, "fixture")).handle.close(); + expect(executableDigestCache.write).toHaveBeenCalledOnce(); + await expect(openVerifiedRuntimeExecutable(f.path, digest, "fixture")).rejects.toThrow("digest mismatch"); + }); + + it("rehashes changed contents and rejects corruption despite a previous valid entry", async () => { + const f = await executable(); + await (await openVerifiedRuntimeExecutable(f.path, f.digest, "fixture")).handle.close(); + await writeFile(f.path, "corrupted native fixture"); + await expect(openVerifiedRuntimeExecutable(f.path, f.digest, "fixture")).rejects.toThrow("digest mismatch"); + }); + + it.each(["permissions", "symlink", "replacement"] as const)("retains admission checks after cached %s changes", async kind => { + const f = await executable(); + await (await openVerifiedRuntimeExecutable(f.path, f.digest, "fixture")).handle.close(); + if (kind === "permissions") { + await chmod(f.path, 0o600); + await expect(openVerifiedRuntimeExecutable(f.path, f.digest, "fixture")).rejects.toThrow("bounded executable file"); + } else if (kind === "symlink") { + await rename(f.path, `${f.path}.original`); await symlink(`${f.path}.original`, f.path); + await expect(openVerifiedRuntimeExecutable(f.path, f.digest, "fixture")).rejects.toThrow("real regular file"); + } else { + await rename(f.path, `${f.path}.original`); await writeFile(f.path, "new invalid executable", { mode: 0o700 }); + await expect(openVerifiedRuntimeExecutable(f.path, f.digest, "fixture")).rejects.toThrow("digest mismatch"); + } + }); + + it("rejects mutation while a cache hit is being read", async () => { + const f = await executable(); + await (await openVerifiedRuntimeExecutable(f.path, f.digest, "fixture")).handle.close(); + vi.mocked(executableDigestCache.read).mockImplementationOnce(async () => { + await writeFile(f.path, "mutation during lookup"); + return f.digest; + }); + await expect(openVerifiedRuntimeExecutable(f.path, f.digest, "fixture")).rejects.toThrow("changed while it was verified"); + }); + + it("checks executable identity after publishing a cold cache entry", async () => { + const f = await executable(); + vi.mocked(executableDigestCache.write).mockImplementationOnce(async (identity, digest) => { + await f.cache.write(identity, digest); + await writeFile(f.path, "mutation during cache publication"); + }); + await expect(openVerifiedRuntimeExecutable(f.path, f.digest, "fixture")).rejects.toThrow("changed while it was verified"); + }); + + it("falls back to hashing when the cache is unavailable", async () => { + const f = await executable(); + await writeFile(f.directory, "unavailable cache"); + await (await openVerifiedRuntimeExecutable(f.path, f.digest, "fixture")).handle.close(); + await writeFile(f.path, "invalid native fixture"); + await expect(openVerifiedRuntimeExecutable(f.path, f.digest, "fixture")).rejects.toThrow("digest mismatch"); + }); + + it("trusts a digest planted by the same operating-system user for current metadata", async () => { + const f = await executable(); + await writeFile(f.path, "content changed by the same user"); + const metadata = await lstat(f.path, { bigint: true }); + await f.cache.write({ device: metadata.dev.toString(), inode: metadata.ino.toString(), size: metadata.size.toString(), + modifiedNanoseconds: metadata.mtimeNs.toString(), changedNanoseconds: metadata.ctimeNs.toString() }, f.digest); + await (await openVerifiedRuntimeExecutable(f.path, f.digest, "fixture")).handle.close(); + }); +}); diff --git a/packages/paperclip-runner/src/drivers/acpx/installation-integrity.ts b/packages/paperclip-runner/src/drivers/acpx/installation-integrity.ts index fbf39064e18..1fab0150997 100644 --- a/packages/paperclip-runner/src/drivers/acpx/installation-integrity.ts +++ b/packages/paperclip-runner/src/drivers/acpx/installation-integrity.ts @@ -2,6 +2,7 @@ import { MAX_ACPX_RUNTIME_EXECUTABLE_BYTES, ACPX_PRIVATE_SNAPSHOT_ENV, createAcp import { createNativeAcpxDistributionSnapshot, NATIVE_ACPX_BOOTSTRAP_NAME, readNativeAcpxDistributionEntries, type NativeAcpxDistributionInput } from "./native-distribution-integrity.js"; export type { NativeAcpxDistributionInput } from "./native-distribution-integrity.js"; import { createHash } from "node:crypto"; +import { executableDigestCache, type ExecutableFileIdentity } from "../../executable-digest-cache.js"; import { spawn as spawnChildProcess, type ChildProcess, @@ -606,13 +607,7 @@ export async function awaitVerifiedAcpxProviderExit( await exitProof; } -interface VerifiedAcpxCommandIdentity { - device: string; - inode: string; - size: string; - modifiedNanoseconds: string; - changedNanoseconds: string; -} +type VerifiedAcpxCommandIdentity = ExecutableFileIdentity; interface VerifiedAcpxRuntimeExecutable { path: string; @@ -1204,7 +1199,7 @@ async function inspectCommand( } } -async function openVerifiedRuntimeExecutable( +export async function openVerifiedRuntimeExecutable( executablePath: string, expectedDigest: string, agent: string, @@ -1246,23 +1241,31 @@ async function openVerifiedRuntimeExecutable( `ACPX ${agent} runtime executable must be a bounded executable file`, ); } - const hash = createHash("sha256"); - const buffer = Buffer.alloc(1024 * 1024); + const cachedDigest = await executableDigestCache.read(fileIdentity(before)); + let digest = cachedDigest; let position = 0; - try { - while (position < Number(before.size)) { - const { bytesRead } = await handle.read( - buffer, - 0, - Math.min(buffer.length, Number(before.size) - position), - position, - ); - if (bytesRead === 0) break; - hash.update(buffer.subarray(0, bytesRead)); - position += bytesRead; + if (digest === null) { + const hash = createHash("sha256"); + const buffer = Buffer.alloc(1024 * 1024); + try { + while (position < Number(before.size)) { + const { bytesRead } = await handle.read( + buffer, + 0, + Math.min(buffer.length, Number(before.size) - position), + position, + ); + if (bytesRead === 0) break; + hash.update(buffer.subarray(0, bytesRead)); + position += bytesRead; + } + } finally { + buffer.fill(0); } - } finally { - buffer.fill(0); + digest = `sha256:${hash.digest("hex")}`; + } + if (cachedDigest === null && position === Number(before.size) && digest === expectedDigest) { + await executableDigestCache.write(fileIdentity(before), digest); } const after = await handle.stat({ bigint: true }); const lexicalAfter = await lstat(executablePath, { bigint: true }).catch( @@ -1271,7 +1274,7 @@ async function openVerifiedRuntimeExecutable( const beforeIdentity = fileIdentity(before); const afterIdentity = fileIdentity(after); if ( - position !== Number(before.size) || + (cachedDigest === null && position !== Number(before.size)) || lexicalAfter === null || lexicalAfter.isSymbolicLink() || !lexicalAfter.isFile() || @@ -1283,7 +1286,6 @@ async function openVerifiedRuntimeExecutable( `ACPX ${agent} runtime executable changed while it was verified`, ); } - const digest = `sha256:${hash.digest("hex")}`; if (digest !== expectedDigest) { throw new Error(`ACPX ${agent} runtime executable digest mismatch`); } diff --git a/packages/paperclip-runner/src/executable-digest-cache.ts b/packages/paperclip-runner/src/executable-digest-cache.ts new file mode 100644 index 00000000000..b50b54d9059 --- /dev/null +++ b/packages/paperclip-runner/src/executable-digest-cache.ts @@ -0,0 +1,70 @@ +import { createHash, randomUUID } from "node:crypto"; +import { constants } from "node:fs"; +import { lstat, mkdir, open, rename, unlink } from "node:fs/promises"; +import { tmpdir, userInfo } from "node:os"; +import { isAbsolute, join } from "node:path"; + +export interface ExecutableFileIdentity { + device: string; + inode: string; + size: string; + modifiedNanoseconds: string; + changedNanoseconds: string; +} + +export function createExecutableDigestCache(directory?: string) { + async function location(identity: ExecutableFileIdentity, create: boolean): Promise { + const user = userInfo(); + const temporaryRoot = tmpdir(); + if (!isAbsolute(directory ?? temporaryRoot)) throw new Error("Executable digest cache directory is unavailable"); + const base = join(temporaryRoot, `paperclip-runner-${user.uid}`); + const directories = directory ? [directory] : [base, join(base, "executable-digests-v1")]; + for (const [index, path] of directories.entries()) { + if (create) await mkdir(path, { mode: 0o700 }).catch(error => { + if ((error as NodeJS.ErrnoException).code !== "EEXIST") throw error; + }); + const metadata = await lstat(path); + if (!metadata.isDirectory() || metadata.isSymbolicLink() || metadata.uid !== user.uid || + (metadata.mode & (index === directories.length - 1 ? 0o077 : 0o022)) !== 0) { + throw new Error("Executable digest cache directory is unavailable"); + } + } + const key = createHash("sha256").update(JSON.stringify([1, identity.device, identity.inode, identity.size, + identity.modifiedNanoseconds, identity.changedNanoseconds])).digest("hex"); + return join(directories.at(-1)!, `v1-${key}.json`); + } + + return { + async read(identity: ExecutableFileIdentity): Promise { + try { + const path = await location(identity, false); + const handle = await open(path, constants.O_RDONLY | constants.O_NOFOLLOW); + try { + const metadata = await handle.stat(); + if (!metadata.isFile() || metadata.uid !== userInfo().uid || (metadata.mode & 0o077) !== 0 || + metadata.size < 1 || metadata.size > 1024) return null; + const entry: unknown = JSON.parse(await handle.readFile("utf8")); + if (typeof entry !== "object" || entry === null || !("version" in entry) || entry.version !== 1 || + !("digest" in entry) || typeof entry.digest !== "string" || !/^sha256:[a-f0-9]{64}$/.test(entry.digest)) return null; + return entry.digest; + } finally { await handle.close(); } + } catch { return null; } + }, + async write(identity: ExecutableFileIdentity, digest: string): Promise { + let temporary: string | undefined; + try { + if (!/^sha256:[a-f0-9]{64}$/.test(digest)) return; + const path = await location(identity, true); + temporary = `${path}.${randomUUID()}.tmp`; + const handle = await open(temporary, constants.O_WRONLY | constants.O_CREAT | constants.O_EXCL | constants.O_NOFOLLOW, 0o600); + try { await handle.writeFile(JSON.stringify({ version: 1, digest })); } + finally { await handle.close(); } + await rename(temporary, path); + } catch { } finally { + if (temporary) await unlink(temporary).catch(() => undefined); + } + }, + }; +} + +export const executableDigestCache = createExecutableDigestCache(); From 4ef018cfe65e96b93c4310e15e479fd4425fad8f Mon Sep 17 00:00:00 2001 From: Dotta Date: Fri, 30 Oct 2026 19:17:14 -0500 Subject: [PATCH 59/70] test: model retention reads in native cleanup transaction fixture --- .../native-runtime/native-session-executor.test.ts | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/server/src/services/native-runtime/native-session-executor.test.ts b/server/src/services/native-runtime/native-session-executor.test.ts index 4dabeda6954..6fb2e52c777 100644 --- a/server/src/services/native-runtime/native-session-executor.test.ts +++ b/server/src/services/native-runtime/native-session-executor.test.ts @@ -4302,6 +4302,17 @@ describe("retained native cleanup activation", () => { releaseCommit = release; }); const db = { + execute: async (query: SQL) => { + // Retention hints read the fixture's run/issue relationship inside the + // same cleanup transaction; this fixture has one issue and no ancestors. + const statement = new PgDialect().sqlToQuery(query); + expect(transactionOpen).toBe(true); + expect(statement.params).toContain(run.companyId); + if (statement.sql.includes("from heartbeat_runs")) return [{ id: run.nativeIssueId }]; + if (statement.sql.includes("origin_kind")) return [{ id: run.nativeIssueId, originKind: null, originId: null }]; + if (statement.sql.includes("with recursive affected")) return [{ issueId: run.nativeIssueId }]; + throw new Error(`Unexpected cleanup fixture query: ${statement.sql}`); + }, select: () => ({ from: (table: unknown) => { const rows = From f201195a92a3f6baa6e4411aa52da39f93f07d83 Mon Sep 17 00:00:00 2001 From: Dotta Date: Fri, 30 Oct 2026 19:20:10 -0500 Subject: [PATCH 60/70] test: await committed recovery and adapter admission outcomes --- .../src/__tests__/chat-channels.integration.test.ts | 12 +++++++++--- .../heartbeat-stale-queue-invalidation.test.ts | 8 ++++++++ 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/server/src/__tests__/chat-channels.integration.test.ts b/server/src/__tests__/chat-channels.integration.test.ts index fcaf642aa84..8b10fe2eec7 100644 --- a/server/src/__tests__/chat-channels.integration.test.ts +++ b/server/src/__tests__/chat-channels.integration.test.ts @@ -16341,9 +16341,15 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => { await tx.update(toolConnections).set({ credentialSecretRefs: connection.credentialSecretRefs.map(ref => ({ ...ref, versionSelector: ref.versionSelector === "1" ? "latest" : "1" })) }).where(eq(toolConnections.id, connection.id)); }); - await expect.poll(async () => ((await f.checkpoint()).value as Record).credentialFingerprint, - { timeout: 5000 }).not.toBe(oldWindow.credentialFingerprint); - expect((await f.checkpoint()).value).toMatchObject({ generation: oldWindow.generation, outcome: "epoch_initialized" }); + // Credential synchronization and epoch initialization commit separately. + // Require the final epoch receipt, not just the intermediate fingerprint. + await expect.poll(async () => { + const value = (await f.checkpoint()).value as Record; + return { ...value, fingerprintChanged: typeof value.credentialFingerprint === "string" + && value.credentialFingerprint !== oldWindow.credentialFingerprint }; + }, { timeout: 5000 }).toMatchObject({ + fingerprintChanged: true, generation: oldWindow.generation, outcome: "epoch_initialized", + }); expect(f.requests).toEqual([]); } finally { await coordinator.stop(); } }); diff --git a/server/src/__tests__/heartbeat-stale-queue-invalidation.test.ts b/server/src/__tests__/heartbeat-stale-queue-invalidation.test.ts index 65191c24d2f..c4f0772ad47 100644 --- a/server/src/__tests__/heartbeat-stale-queue-invalidation.test.ts +++ b/server/src/__tests__/heartbeat-stale-queue-invalidation.test.ts @@ -262,6 +262,14 @@ describeEmbeddedPostgres("heartbeat stale queued-run invalidation", () => { await heartbeat.resumeQueuedRuns(); const [delivered] = await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, wake.id)); expect(delivered.runId).toBeTruthy(); + // The receipt can commit before execution enters liveRunExecutions. + // Observe actual adapter admission before awaiting its drain. + await vi.waitFor(async () => { + const current = await heartbeat.getRun(delivered.runId!); + expect(countExecuteCallsForRun(delivered.runId!), JSON.stringify({ + status: current?.status, errorCode: current?.errorCode, + })).toBe(1); + }, { timeout: 5_000 }); await heartbeat.waitForRunExecutionDrain(delivered.runId!); expect(countExecuteCallsForRun(delivered.runId!)).toBe(1); expect((await heartbeat.getRun(delivered.runId!))?.contextSnapshot).toMatchObject({ wakeCommentIds: [comment.id] }); From ac6c5191ee9047230fa53acd34e2260e96905d48 Mon Sep 17 00:00:00 2001 From: Dotta Date: Fri, 30 Oct 2026 19:22:39 -0500 Subject: [PATCH 61/70] Drive saved execution receipts from committed work and deadlines Co-Authored-By: Paperclip --- .../2026-10-30-deferred-execution-sleep.md | 19 ++ .../deferred-execution-receipt-work.test.ts | 226 ++++++++++++++++++ .../server-startup-feedback-export.test.ts | 5 +- server/src/index.ts | 8 +- .../modules/run-dispatch/adapters/postgres.ts | 2 + .../wake-queue/adapters/postgres.test.ts | 9 +- .../modules/wake-queue/adapters/postgres.ts | 7 + server/src/routes/issues.ts | 2 + server/src/services/agent-configuration.ts | 2 + server/src/services/agent-lifecycle.ts | 2 + server/src/services/budgets.ts | 5 + .../chat-channels/failed-run-retry.ts | 5 + .../chat-channels/task-retry-input.ts | 2 + server/src/services/companies.ts | 3 + .../deferred-execution-notifications.ts | 34 +++ .../services/delivery-work-notifications.ts | 3 + .../execution-control-notifications.ts | 7 +- server/src/services/heartbeat.ts | 9 + .../heartbeat/deferred-receipt-work.ts | 100 ++++++++ server/src/services/heartbeat/queue.ts | 38 ++- server/src/services/heartbeat/recovery.ts | 4 +- server/src/services/heartbeat/run-control.ts | 19 +- server/src/services/instance-settings.ts | 2 + server/src/services/issues.ts | 9 + .../status-decision-committer.ts | 2 + server/src/services/task-retry.ts | 6 +- 26 files changed, 505 insertions(+), 25 deletions(-) create mode 100644 doc/plans/2026-10-30-deferred-execution-sleep.md create mode 100644 server/src/__tests__/deferred-execution-receipt-work.test.ts create mode 100644 server/src/services/deferred-execution-notifications.ts create mode 100644 server/src/services/heartbeat/deferred-receipt-work.ts diff --git a/doc/plans/2026-10-30-deferred-execution-sleep.md b/doc/plans/2026-10-30-deferred-execution-sleep.md new file mode 100644 index 00000000000..0478fe79300 --- /dev/null +++ b/doc/plans/2026-10-30-deferred-execution-sleep.md @@ -0,0 +1,19 @@ +# Deferred execution receipt ownership + +This slice removes the periodic `resumeQueuedRuns` sweep after queue dispatch migrated separately. It also delegates native status wake intents out of the periodic orphan reaper. Existing queue admission, authorization, task recovery, cancellation, resource and controller checks remain the mutation authority. + +| Duty | Durable pending truth | Committing producers | Deadline and exit | +| --- | --- | --- | --- | +| Task Retry | deferred task_retry wake with certified actor/generation | requestTaskRetry through execution control; execution, issue, agent, company and budget eligibility | immediate; current admission links canonical intent to one dispatch/run, supersession or terminal policy retires it; blocked input awaits its actual gate mutation | +| Saved comments and interaction responses | deferred wake carrying saved comment IDs or interaction mutation | wake admission insert; merge only genuinely new comment IDs or interaction identity; issue/lock/assignment/dependency and execution eligibility | immediate after terminal source; original release owner revalidates comment/card authority, process/cleanup evidence and issue claim | +| Execution wait | deferred wake with executionWait | original admission/restored source receipt; actual recovery action, execution and eligibility changes | original updatedAt + 30 seconds, then current owner revalidation; unchanged blocked receipt carries no recurring scan | +| Interrupt retry | deferred wake with queuedCommentInterrupt | authorized route persists interrupt marker; cancellation lifecycle and cleanup/eligibility transitions | original updatedAt + 30 seconds; cancelled legacy run naming exact queue bypasses initial grace as before; owner retains explicit interrupt and stop evidence | +| Cancelled legacy queue | cancelled legacy run result names exact still-deferred queue | run lifecycle terminal/control commits; original interrupt intent | immediate restart reconstruction; original release/admission owner settles or leaves saved input behind its actual blocker | +| Task chat receipt settlement | deferred task_retry with taskRetry.chatActionId | failed chat retry action settlement | immediate settlement-only pass folds linked action result; it cannot restage its own processed-but-deferred chat action. Startup/actual eligibility retains normal re-admission policy | +| Native status wake intent | system/native-status-committer wake queued/claimed without run | status committer transaction; actual execution/issue/agent/company/budget eligibility | immediate queued intent; claimedAt + 60 seconds for crash recovery; error updatedAt + 30 seconds. Existing deterministic dispatcher identity/claims prevent duplicate delivery; original intent becomes linked/coalesced/skipped while a separate blocked dispatch receipt owns saved admission | + +Three scoped owners reconstruct only their actual durable receipt predicates, with keyset pages of 100. Normal signals contain normalized company/request identity. Instance scheduler settings alone may request instance-wide reconstruction. Repeated source changes coalesce reconstruction for 30 seconds; already-known stored deadlines preempt that cooldown. Pending checks read memory only. Shutdown uses the existing coordinator and physical execution tracking; task drain reconciles producer transactions and releases the settled intent hold while retaining work for resume. Warm standby and disabled registration do not execute domain work. + +The generic run-dispatch notifier and broad issue review-state notifier deliberately do not fan out deferred eligibility. Queue bookkeeping may write them while processing its own saved input. Separate eligibility commits and settlement-only chat notifications avoid a self-restaging loop. + +C2 remains separate and mandatory: active output silence deadlines, remaining orphan/process/controller/remote authority and resource debts, legacy shutdown settlements, stranded agent status, and final aggregate chain removal. Retained workspace and migrated native finalization workers remain their existing owners. diff --git a/server/src/__tests__/deferred-execution-receipt-work.test.ts b/server/src/__tests__/deferred-execution-receipt-work.test.ts new file mode 100644 index 00000000000..ba0f13bed58 --- /dev/null +++ b/server/src/__tests__/deferred-execution-receipt-work.test.ts @@ -0,0 +1,226 @@ +import { randomUUID } from "node:crypto"; +import { and, eq, sql } from "drizzle-orm"; +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; +import { agents, agentWakeupRequests, authUsers, chatActions, chatEndpoints, toolApplications, toolConnections, companies, companyMemberships, createDb, heartbeatRuns, issues } from "@paperclipai/db"; +import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js"; +import { startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; +import { createDeliveryWorkCoordinator } from "../services/delivery-work-coordinator.js"; +import { notifyDeferredExecutionEligibility, notifyDeferredExecutionWork, notifyNativeStatusWakeWork, notifyTaskRetrySettlement } from "../services/deferred-execution-notifications.js"; +import { registerDeferredReceiptWork } from "../services/heartbeat/deferred-receipt-work.js"; +import { heartbeatService } from "../services/heartbeat.js"; +import { requestTaskRetry } from "../services/task-retry.js"; +import { issueService } from "../services/issues.js"; +import { idleWorkSnapshot } from "../services/task-admission.js"; + +vi.setConfig({ testTimeout: 90_000 }); +describe("deferred receipt durable dispatch", () => { + let temporary: Awaited> | undefined; + let db: ReturnType; + let heartbeat: ReturnType; + let coordinator: ReturnType; + let drain = false; + const errors: unknown[] = []; + beforeAll(async () => { + const external = resolveExternalTestDatabaseUrl("PAPERCLIP_DEFERRED_RECEIPT_TEST_DATABASE_URL"); + if (!external) temporary = await startEmbeddedPostgresTestDatabase("deferred-receipts-"); + db = createDb(external ?? temporary!.connectionString, { maxConnections: 6, idleTimeoutSeconds: 0, maxLifetimeSeconds: 0 }); + await Promise.all(Array.from({ length: 6 }, () => db.execute(sql`select pg_sleep(0.01)`))); + }); + beforeEach(async () => { + await db.execute(sql`truncate companies cascade`); drain = false; errors.length = 0; + heartbeat = heartbeatService(db); + coordinator = createDeliveryWorkCoordinator({ owner: db, canRun: () => !drain, canReconcile: () => true, + onError: error => errors.push(error) }); + }); + afterEach(async () => { + await coordinator.stop(); await heartbeat.drainActiveRunExecutions(); + vi.useRealTimers(); vi.restoreAllMocks(); expect(errors).toEqual([]); + }); + afterAll(async () => { await db?.$client.end({ timeout: 1 }); await temporary?.cleanup(); }); + async function fixture() { + const companyId = randomUUID(), actorId = "deferred-receipt-user"; + await db.insert(authUsers).values({ id: actorId, name: "Receipt operator", email: `${actorId}@example.test`, + createdAt: new Date(), updatedAt: new Date() }).onConflictDoNothing(); + await db.insert(companies).values({ id: companyId, name: "Deferred dispatch", issuePrefix: `DE${companyId.slice(0, 6)}`, + defaultResponsibleUserId: actorId, requireBoardApprovalForNewAgents: false }); + await db.insert(companyMemberships).values({ companyId, principalId: actorId, principalType: "user", status: "active", membershipRole: "owner" }); + const [agent] = await db.insert(agents).values({ companyId, name: "Receipt process", adapterType: "process", status: "idle", + adapterConfig: { command: process.execPath, args: ["-e", "console.log('receipt-proof')"], cwd: process.cwd() }, + runtimeConfig: { heartbeat: { enabled: true, intervalSec: 0, wakeOnDemand: true } } }).returning(); + const [issue] = await db.insert(issues).values({ companyId, title: "Resume saved task", status: "todo", assigneeAgentId: agent.id }).returning(); + const [source] = await db.insert(heartbeatRuns).values({ companyId, agentId: agent.id, issueId: issue.id, + status: "failed", finishedAt: new Date(), responsibleUserId: actorId, errorCode: "process_failed", + contextSnapshot: { issueId: issue.id }, resultJson: { executionRecovery: { kind: "bootstrap", providerWorkStarted: false } } }).returning(); + return { companyId, agent, issue, source, actorId }; + } + async function quiet() { + await vi.waitFor(async () => { expect(coordinator.nextWakeAt()).toBeNull(); expect(idleWorkSnapshot().active).toBe(0); }, { timeout: 40_000 }); + const reads = vi.spyOn(db, "select"), executes = vi.spyOn(db, "execute"); + const selected = reads.mock.calls.length, executed = executes.mock.calls.length; + if (!vi.isFakeTimers()) vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"] }); + await vi.advanceTimersByTimeAsync(20 * 60_000); + expect(reads).toHaveBeenCalledTimes(selected); expect(executes).toHaveBeenCalledTimes(executed); + } + it("is quiet after empty reconstruction and terminal receipt history", async () => { + const f = await fixture(); + await db.insert(agentWakeupRequests).values({ companyId: f.companyId, agentId: f.agent.id, + source: "on_demand", status: "completed", reason: "task_retry", payload: { issueId: f.issue.id } }); + await heartbeat.registerDeferredReceiptWork(coordinator, () => !drain).ready; await quiet(); + }); + it("dispatches an ordinary committed task Retry into saved process output", async () => { + const f = await fixture(); + await heartbeat.registerDeferredReceiptWork(coordinator, () => !drain).ready; + await heartbeat.registerRunDispatchWork(coordinator, () => !drain).ready; + await heartbeat.registerScheduledRetryCancellationWork(coordinator, () => !drain).ready; + const receipt = await requestTaskRetry({ db, companyId: f.companyId, issueId: f.issue.id, actorId: f.actorId, + observedRunId: f.source.id, idempotencyKey: randomUUID() }); + expect(receipt.accepted).toBe(true); + let runId: string | undefined; + await vi.waitFor(async () => { + const [wake] = await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, receipt.requestId!)); + expect(wake.runId, JSON.stringify(wake)).toBeTruthy(); runId = wake.runId!; + }, { timeout: 15_000 }); + await vi.waitFor(async () => { + await heartbeat.drainActiveRunExecutions(); + const [run] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, runId!)); + expect(run.status, JSON.stringify({ error: run.error, code: run.errorCode })).toBe("succeeded"); + expect(run.stdoutExcerpt).toContain("receipt-proof"); + }, { timeout: 15_000 }); + await issueService(db).update(f.issue.id, { status: "done" }); + await heartbeat.drainActiveRunExecutions(); + const saved = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.status, "scheduled_retry")); + expect(await db.select().from(heartbeatRuns).where(sql`${heartbeatRuns.status} in ('queued', 'running')`)).toHaveLength(0); + vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"] }); + if (saved.length) await vi.advanceTimersByTimeAsync(Math.max(0, Math.max(...saved.map(run => run.scheduledRetryAt!.getTime())) - Date.now()) + 1); + await vi.waitFor(async () => expect(await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.status, "scheduled_retry"))).toHaveLength(0)); + const linked = await db.select().from(agentWakeupRequests).where(and(eq(agentWakeupRequests.reason, "task_retry"), sql`${agentWakeupRequests.runId} is not null`)); + expect(linked).toHaveLength(2); + expect(new Set(linked.map(row => row.runId))).toEqual(new Set([runId])); + expect(linked.map(row => row.idempotencyKey)).toContain(`task-retry-dispatch:${receipt.requestId}`); + expect(linked.find(row => row.id === receipt.requestId)?.status).toBe("coalesced"); + expect(await db.select().from(heartbeatRuns).where(sql`${heartbeatRuns.contextSnapshot}->>'taskRetryRequestId' = ${receipt.requestId}`)).toHaveLength(1); + await quiet(); + }); + it.each(["native claim", "execution wait"])("reconstructs an exact fresh %s deadline without probing early", async kind => { + const f = await fixture(), updatedAt = new Date(), native = kind === "native claim"; + const [wake] = await db.insert(agentWakeupRequests).values({ companyId: f.companyId, agentId: f.agent.id, + source: "automation", status: native ? "claimed" : "deferred_issue_execution", claimedAt: updatedAt, updatedAt, + payload: native ? {} : { executionWait: { reason: "recovery" } }, + requestedByActorType: "system", requestedByActorId: native ? "native-status-committer" : null }).returning(); + vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"] }); + const check = vi.fn(async (scope: { companyId?: string; requestId?: string }) => { + await db.update(agentWakeupRequests).set({ status: "skipped" }).where(eq(agentWakeupRequests.id, scope.requestId!)); + }); + await registerDeferredReceiptWork(db, coordinator, { resume: check, settleTaskChat: vi.fn(), dispatchNative: check, + schedulingSuppressed: async () => false }, () => !drain).ready; + const due = updatedAt.getTime() + (native ? 60_000 : 30_000); + expect(coordinator.nextWakeAt()).toBe(due); expect(check).not.toHaveBeenCalled(); + await vi.advanceTimersByTimeAsync(due - Date.now() - 1); expect(check).not.toHaveBeenCalled(); + await vi.advanceTimersByTimeAsync(1); + await vi.waitFor(() => expect(check).toHaveBeenCalledExactlyOnceWith(expect.objectContaining({ companyId: f.companyId, requestId: wake.id }))); + await quiet(); + }); + it("reconciles the first drained producer and waits for resume without domain SQL", async () => { + const f = await fixture(); drain = true; + const resume = vi.fn(async () => {}), dispatchNative = vi.fn(async () => {}); + await registerDeferredReceiptWork(db, coordinator, { resume, settleTaskChat: vi.fn(), dispatchNative, + schedulingSuppressed: async () => false }, () => !drain).ready; + await db.transaction(async tx => { + await notifyDeferredExecutionWork(tx, { companyId: f.companyId }); + await tx.insert(agentWakeupRequests).values({ companyId: f.companyId, agentId: f.agent.id, source: "on_demand", + status: "deferred_issue_execution", reason: "task_retry" }); + }); + vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"] }); + await vi.advanceTimersByTimeAsync(1000); + expect(resume).not.toHaveBeenCalled(); expect(dispatchNative).not.toHaveBeenCalled(); + await vi.waitFor(() => expect(idleWorkSnapshot().active).toBe(0)); + drain = false; await vi.advanceTimersByTimeAsync(30_000); + await vi.waitFor(() => expect(resume).toHaveBeenCalledTimes(1)); await quiet(); + }); + it.each([ + { name: "saved comment", payload: { _paperclipWakeContext: { wakeCommentIds: ["saved-comment"] } } }, + { name: "interaction", payload: { mutation: "interaction" } }, + { name: "execution wait", payload: { executionWait: { reason: "recovery" } } }, + { name: "interrupt", payload: { queuedCommentInterrupt: { actorId: "operator" } } }, + ])("activates only the committed company's $name receipt after an empty start", async ({ payload }) => { + const f = await fixture(), foreign = await fixture(); + const resume = vi.fn(async () => {}); + await registerDeferredReceiptWork(db, coordinator, { resume, settleTaskChat: vi.fn(), dispatchNative: vi.fn(), + schedulingSuppressed: async () => false }, () => !drain).ready; + const [other] = await db.insert(agentWakeupRequests).values({ companyId: foreign.companyId, agentId: foreign.agent.id, + source: "on_demand", status: "deferred_issue_execution", payload }).returning(); + let requestId: string; + await db.transaction(async tx => { + await tx.transaction(async nested => { + await notifyDeferredExecutionWork(nested, { companyId: f.companyId }); + const [row] = await nested.insert(agentWakeupRequests).values({ companyId: f.companyId, agentId: f.agent.id, + source: "on_demand", status: "deferred_issue_execution", payload, updatedAt: new Date(Date.now() - 31_000) }).returning(); + requestId = row.id; + }); + expect(resume).not.toHaveBeenCalled(); + }); + await vi.waitFor(() => expect(resume).toHaveBeenCalledExactlyOnceWith(expect.objectContaining({ companyId: f.companyId, requestId })), { timeout: 5_000 }); + expect((await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, other.id)))[0].status).toBe("deferred_issue_execution"); + await quiet(); + }); + it("reconstructs a cancelled legacy run's exact deferred queue even without comment payload", async () => { + const f = await fixture(); + const [wake] = await db.insert(agentWakeupRequests).values({ companyId: f.companyId, agentId: f.agent.id, + source: "on_demand", status: "deferred_issue_execution" }).returning(); + await db.update(heartbeatRuns).set({ status: "cancelled", resultJson: { queuedCommentInterruptQueueId: wake.id } }) + .where(eq(heartbeatRuns.id, f.source.id)); + const resume = vi.fn(async () => {}); + await registerDeferredReceiptWork(db, coordinator, { resume, settleTaskChat: vi.fn(), dispatchNative: vi.fn(), + schedulingSuppressed: async () => false }, () => !drain).ready; + expect(resume).toHaveBeenCalledExactlyOnceWith(expect.objectContaining({ companyId: f.companyId, requestId: wake.id })); + await quiet(); + }); + it("settles a processed chat receipt without restaging it until an actual eligibility commit", async () => { + const f = await fixture(); + await heartbeat.registerDeferredReceiptWork(coordinator, () => !drain).ready; + const [application] = await db.insert(toolApplications).values({ companyId: f.companyId, name: "Receipt chat", type: "mcp_http" }).returning(); + const [connection] = await db.insert(toolConnections).values({ companyId: f.companyId, applicationId: application.id, + uid: randomUUID(), name: "Receipt chat", transport: "rest_api" }).returning(); + const [endpoint] = await db.insert(chatEndpoints).values({ companyId: f.companyId, connectionId: connection.id, + provider: "slack", publicId: randomUUID(), assignedAgentId: f.agent.id }).returning(); + const intentId = randomUUID(), actionId = randomUUID(), previous = new Date(Date.now() - 60_000); + await db.insert(agentWakeupRequests).values([ + { id: intentId, companyId: f.companyId, agentId: f.agent.id, source: "on_demand", reason: "task_retry", + status: "deferred_issue_execution", updatedAt: previous, payload: { issueId: f.issue.id, + taskRetry: { schema: "paperclip.task-retry.v1", generationId: intentId, sourceRunId: f.source.id, + actorId: f.actorId, strategy: "evaluate", chatActionId: actionId } } }, + { id: actionId, companyId: f.companyId, agentId: f.agent.id, source: "automation", status: "deferred_issue_execution", + payload: { issueId: f.issue.id, taskRetryRequestId: intentId } }, + ]); + await db.transaction(async tx => { + await notifyTaskRetrySettlement(tx, { companyId: f.companyId, requestId: intentId }); + await tx.insert(chatActions).values({ id: actionId, companyId: f.companyId, endpointId: endpoint.id, + providerActionId: randomUUID(), kind: "failed_run_retry", status: "processed", updatedAt: new Date(), + payload: { issueId: f.issue.id, taskRetryIntentId: intentId, failedRunId: f.source.id } }); + }); + await vi.waitFor(async () => { + const [intent] = await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, intentId)); + expect(intent.updatedAt.getTime()).toBeGreaterThan(previous.getTime()); + }, { timeout: 5_000 }); + expect((await db.select().from(chatActions).where(eq(chatActions.id, actionId)))[0].status).toBe("processed"); + await db.transaction(tx => notifyDeferredExecutionEligibility(tx, { companyId: f.companyId })); + await vi.waitFor(async () => expect((await db.select().from(chatActions).where(eq(chatActions.id, actionId)))[0].status) + .toBe("issued"), { timeout: 5_000 }); + await quiet(); + }); + it("outer rollback and nested savepoint rollback do not dispatch receipt intent", async () => { + const f = await fixture(); const dispatchNative = vi.fn(async () => {}); + await registerDeferredReceiptWork(db, coordinator, { resume: vi.fn(), settleTaskChat: vi.fn(), dispatchNative, + schedulingSuppressed: async () => false }, () => !drain).ready; + await expect(db.transaction(async tx => { + await notifyNativeStatusWakeWork(tx, { companyId: f.companyId }); + throw new Error("outer rollback"); + })).rejects.toThrow("outer rollback"); + await db.transaction(async tx => { + await expect(tx.transaction(async nested => { + await notifyNativeStatusWakeWork(nested, { companyId: f.companyId }); throw new Error("nested rollback"); + })).rejects.toThrow("nested rollback"); + }); + await quiet(); expect(dispatchNative).not.toHaveBeenCalled(); + }); +}); diff --git a/server/src/__tests__/server-startup-feedback-export.test.ts b/server/src/__tests__/server-startup-feedback-export.test.ts index a26d94f50d4..55d261f91c0 100644 --- a/server/src/__tests__/server-startup-feedback-export.test.ts +++ b/server/src/__tests__/server-startup-feedback-export.test.ts @@ -135,6 +135,7 @@ const { reconcileHotRestartAdoption: vi.fn(async () => ({ mode: "none" })), reapOrphanedRuns: vi.fn(async () => ({ reaped: 0, runIds: [] })), registerIssueStateRecoveryWork: vi.fn((_coordinator: unknown, _canRun: () => boolean) => ({ ready: Promise.resolve() })), + registerDeferredReceiptWork: vi.fn((_coordinator: unknown, _canRun: () => boolean) => ({ ready: Promise.resolve() })), registerRunDispatchWork: vi.fn((_coordinator: unknown, _canRun: () => boolean) => ({ ready: Promise.resolve() })), registerScheduledRetryCancellationWork: vi.fn((_coordinator: unknown, _canRun: () => boolean) => ({ ready: Promise.resolve() })), registerStoppedInstructionCollectionWork: vi.fn((_coordinator: unknown, _canRun: () => boolean) => ({ ready: Promise.resolve() })), @@ -930,6 +931,8 @@ describe("startServer feedback export wiring", () => { await startServer(); await settleStartupHousekeeping(); expect(heartbeatServiceMock.registerIssueStateRecoveryWork).toHaveBeenCalledTimes(enabled ? 1 : 0); expect(heartbeatServiceMock.registerRunDispatchWork).toHaveBeenCalledTimes(enabled ? 1 : 0); + expect(heartbeatServiceMock.registerDeferredReceiptWork).toHaveBeenCalledTimes(enabled ? 1 : 0); + expect(heartbeatServiceMock.resumeQueuedRuns).not.toHaveBeenCalled(); if (enabled) { const [coordinator, admitted] = heartbeatServiceMock.registerRunDispatchWork.mock.calls[0]; const app = await createAppMock.mock.results.at(-1)!.value; @@ -1341,7 +1344,7 @@ describe("startServer feedback export wiring", () => { intervalCallback?.(); await new Promise((resolve) => setImmediate(resolve)); expect(heartbeatServiceMock.sweepStaleIssueLocks).not.toHaveBeenCalled(); - expect(heartbeatServiceMock.reapOrphanedRuns).toHaveBeenLastCalledWith({ staleThresholdMs: 300_000, skipScheduledRetryCancellation: true, skipCapturedInstructionCommit: true, skipInstructionDirectoryCleanup: true, skipStoppedInstructionCollection: true, skipCancelledWorkspacePreservation: true, skipObsoletePolicyReviews: true, skipAutomaticCompletionReviews: true, skipUnsafeWorkspaceExports: true, skipNativePendingEffects: true, skipNativeResponsePresentation: true, skipNativeReviewResponse: true, skipNativeStatusReassessment: true, skipRetainedNativeSessionCleanup: true, skipNativeWorkspaceFinalization: true, skipNativeResultCommit: true, skipNativeRunProjection: true, skipNativeRecoveryNotices: true, skipExternalConversationSettlement: true, skipExternalAgentSettlement: true, skipNativeEnvironmentSettlement: true }); + expect(heartbeatServiceMock.reapOrphanedRuns).toHaveBeenLastCalledWith({ staleThresholdMs: 300_000, skipNativeStatusWakeIntents: true, skipScheduledRetryCancellation: true, skipCapturedInstructionCommit: true, skipInstructionDirectoryCleanup: true, skipStoppedInstructionCollection: true, skipCancelledWorkspacePreservation: true, skipObsoletePolicyReviews: true, skipAutomaticCompletionReviews: true, skipUnsafeWorkspaceExports: true, skipNativePendingEffects: true, skipNativeResponsePresentation: true, skipNativeReviewResponse: true, skipNativeStatusReassessment: true, skipRetainedNativeSessionCleanup: true, skipNativeWorkspaceFinalization: true, skipNativeResultCommit: true, skipNativeRunProjection: true, skipNativeRecoveryNotices: true, skipExternalConversationSettlement: true, skipExternalAgentSettlement: true, skipNativeEnvironmentSettlement: true }); expect(heartbeatServiceMock.reconcileCostAccounting).toHaveBeenCalledTimes(1); expect(heartbeatServiceMock.promoteDueScheduledRetries).not.toHaveBeenCalled(); expect(retiredDetector).not.toHaveBeenCalled(); diff --git a/server/src/index.ts b/server/src/index.ts index 0ebe5c3332c..fffac2a7972 100644 --- a/server/src/index.ts +++ b/server/src/index.ts @@ -1774,7 +1774,6 @@ async function startServerWithDatabaseTeardown( } } - await timeStartupRecovery("queued_runs", () => heartbeat.resumeQueuedRuns()); const recoveredGoalActions = await timeStartupRecovery("goal_actions", () => heartbeat.recoverPendingSessionGoalActions()); if ( recoveredGoalActions.enqueued > 0 || @@ -1806,6 +1805,8 @@ async function startServerWithDatabaseTeardown( if (config.heartbeatSchedulerEnabled) await heartbeat.registerIssueStateRecoveryWork(deliveryWork, () => !isIdleTaskDrainActive() && !isWarmStandby() && !heartbeatSchedulerStopped).ready; + if (config.heartbeatSchedulerEnabled) await heartbeat.registerDeferredReceiptWork(deliveryWork, + () => !isIdleTaskDrainActive() && !isWarmStandby() && !heartbeatSchedulerStopped).ready; if (config.heartbeatSchedulerEnabled) await heartbeat.registerRunDispatchWork(deliveryWork, () => !isIdleTaskDrainActive() && !isWarmStandby() && !heartbeatSchedulerStopped).ready; if (config.heartbeatSchedulerEnabled) await heartbeat.registerScheduledRetryCancellationWork(deliveryWork, @@ -2009,10 +2010,7 @@ async function startServerWithDatabaseTeardown( // Periodically reap orphaned runs (5-min staleness threshold) and make sure // persisted queued work is still being driven forward. runPeriodicRecovery(() => heartbeat - .reapOrphanedRuns({ staleThresholdMs: 5 * 60 * 1000, skipScheduledRetryCancellation: true, skipCapturedInstructionCommit: true, skipInstructionDirectoryCleanup: true, skipStoppedInstructionCollection: true, skipCancelledWorkspacePreservation: true, skipObsoletePolicyReviews: true, skipAutomaticCompletionReviews: true, skipUnsafeWorkspaceExports: true, skipNativePendingEffects: true, skipNativeResponsePresentation: true, skipNativeReviewResponse: true, skipNativeStatusReassessment: true, skipRetainedNativeSessionCleanup: true, skipNativeWorkspaceFinalization: true, skipNativeResultCommit: true, skipNativeRunProjection: true, skipNativeRecoveryNotices: true, skipExternalConversationSettlement: true, skipExternalAgentSettlement: true, skipNativeEnvironmentSettlement: true }) - .then(async () => { - await heartbeat.resumeQueuedRuns(); - }) + .reapOrphanedRuns({ staleThresholdMs: 5 * 60 * 1000, skipNativeStatusWakeIntents: true, skipScheduledRetryCancellation: true, skipCapturedInstructionCommit: true, skipInstructionDirectoryCleanup: true, skipStoppedInstructionCollection: true, skipCancelledWorkspacePreservation: true, skipObsoletePolicyReviews: true, skipAutomaticCompletionReviews: true, skipUnsafeWorkspaceExports: true, skipNativePendingEffects: true, skipNativeResponsePresentation: true, skipNativeReviewResponse: true, skipNativeStatusReassessment: true, skipRetainedNativeSessionCleanup: true, skipNativeWorkspaceFinalization: true, skipNativeResultCommit: true, skipNativeRunProjection: true, skipNativeRecoveryNotices: true, skipExternalConversationSettlement: true, skipExternalAgentSettlement: true, skipNativeEnvironmentSettlement: true }) .then(async () => { const scanned = await heartbeat.scanSilentActiveRuns(); if (scanned.created > 0 || scanned.escalated > 0) { diff --git a/server/src/modules/run-dispatch/adapters/postgres.ts b/server/src/modules/run-dispatch/adapters/postgres.ts index b05f94e1a4b..5acbfcf6884 100644 --- a/server/src/modules/run-dispatch/adapters/postgres.ts +++ b/server/src/modules/run-dispatch/adapters/postgres.ts @@ -1,3 +1,4 @@ +import { notifyDeferredExecutionWork } from "../../../services/deferred-execution-notifications.js"; import { hasRequiredWorkspaceRecovery } from "../../../services/workspace-restore-recovery-state.js"; import { readNativeWorkspaceSyncReference } from "../../../services/native-runtime/native-workspace-sync.js"; import { isPreDispatchExecutionWaitVerified } from "../../../services/pre-dispatch-execution-wait.js"; @@ -996,6 +997,7 @@ export function createPostgresRunDispatchAdapter( recordedCommentId && authors.get(recordedCommentId) === actorId ? [recordedCommentId] : []; })); if (commentIds.some(id => !recordedCommentIds.has(id))) return null; + await notifyDeferredExecutionWork(tx, { companyId: run.companyId, requestId: wake.id }); const [deferred] = await tx.update(agentWakeupRequests).set({ status: "deferred_issue_execution", runId: null, claimedAt: null, finishedAt: null, error: null, payload: withQueuedCommentIdsInWakePayload({ ...payload, _paperclipWakeContext: context, diff --git a/server/src/modules/wake-queue/adapters/postgres.test.ts b/server/src/modules/wake-queue/adapters/postgres.test.ts index 068a21f2420..09fbb1749ac 100644 --- a/server/src/modules/wake-queue/adapters/postgres.test.ts +++ b/server/src/modules/wake-queue/adapters/postgres.test.ts @@ -1,3 +1,4 @@ +import { resolveExternalTestDatabaseUrl } from "../../../__tests__/helpers/external-test-database.js"; import { DELIVERY_QUEUES } from "../../../services/delivery-work-notifications.js"; import { instanceSettingsService } from "../../../services/instance-settings.js"; import { randomBytes, randomUUID } from "node:crypto"; @@ -37,7 +38,8 @@ import type { TransactionScope } from "../application/ports.js"; // a foreign-company row is invisible to a read, and a deferred-status // compare-and-set that affects no row leaves no other trace. The decision // branching itself is proven against plain facts in `domain/policy.test.ts`. -const embeddedPostgresSupport = await getEmbeddedPostgresTestSupport(); +const external = resolveExternalTestDatabaseUrl("PAPERCLIP_WAKE_QUEUE_TEST_DATABASE_URL"); +const embeddedPostgresSupport = external ? { supported: true, reason: undefined } : await getEmbeddedPostgresTestSupport(); const describeEmbeddedPostgres = embeddedPostgresSupport.supported ? describe : describe.skip; if (!embeddedPostgresSupport.supported) { @@ -57,8 +59,8 @@ describeEmbeddedPostgres("wake-queue postgres adapter", () => { }; beforeAll(async () => { - tempDb = await startEmbeddedPostgresTestDatabase("paperclip-wake-queue-postgres-adapter-"); - db = createDb(tempDb.connectionString); + if (!external) tempDb = await startEmbeddedPostgresTestDatabase("paperclip-wake-queue-postgres-adapter-"); + db = createDb(external ?? tempDb!.connectionString, { maxConnections: 6 }); }, 20_000); afterEach(async () => { @@ -78,6 +80,7 @@ describeEmbeddedPostgres("wake-queue postgres adapter", () => { }); afterAll(async () => { + await db?.$client.end({ timeout: 1 }); await tempDb?.cleanup(); }); diff --git a/server/src/modules/wake-queue/adapters/postgres.ts b/server/src/modules/wake-queue/adapters/postgres.ts index 1df4420615f..e7a5f12f3f2 100644 --- a/server/src/modules/wake-queue/adapters/postgres.ts +++ b/server/src/modules/wake-queue/adapters/postgres.ts @@ -1,3 +1,4 @@ +import { notifyDeferredExecutionWork } from "../../../services/deferred-execution-notifications.js"; import { notifyRunDispatch } from "../../../services/run-dispatch-notifications.js"; import { notifyExternalConversationSettlement } from "../../../services/external-conversation-settlement-notifications.js"; import { notifyNativeStatusReassessment } from "../../../services/native-status-reassessment-notifications.js"; @@ -1010,6 +1011,11 @@ export function createWakeAdmissionWriter(): WakeAdmissionWriter { )).for("update"); if (!prior) throw new Error("wake-queue: the deferred wake to merge into was not found for this company"); const priorPayload = parseObject(prior.payload), issueId = readNonEmptyString(priorPayload.issueId); + const previousComments = new Set(queuedCommentIdsFromWakePayload(priorPayload)); + const newComments = queuedCommentIdsFromWakePayload(input.mergedPayload).some(id => !previousComments.has(id)); + const newInteraction = readNonEmptyString(input.mergedPayload.interactionId); + if (newComments || newInteraction && newInteraction !== readNonEmptyString(priorPayload.interactionId)) + await notifyDeferredExecutionWork(tx, { companyId: input.companyId, requestId: prior.id }); const priorCommentId = readNonEmptyString(priorPayload.commentId), priorActor = readNonEmptyString(prior.requestedByActorId); const [priorComment] = prior.requestedByActorType === "user" && priorActor && issueId && isUuidLike(issueId) && priorCommentId && isUuidLike(priorCommentId) ? await tx.select({ id: issueComments.id, body: issueComments.body }).from(issueComments).where(and( @@ -1081,6 +1087,7 @@ export function createWakeAdmissionWriter(): WakeAdmissionWriter { async insertNewDeferredWake(scope, input) { const tx = requireAdmissionTx(scope, input.companyId); + await notifyDeferredExecutionWork(tx, { companyId: input.companyId }); await tx.insert(agentWakeupRequests).values({ ...input.durableReceipt, companyId: input.companyId, diff --git a/server/src/routes/issues.ts b/server/src/routes/issues.ts index cf86556ce5e..efdf363975d 100644 --- a/server/src/routes/issues.ts +++ b/server/src/routes/issues.ts @@ -1,3 +1,4 @@ +import { notifyDeferredExecutionWork } from "../services/deferred-execution-notifications.js"; import { notifyNativeRecoveryNotices } from "../services/native-recovery-notice-notifications.js"; import { notifyNativeStatusReassessment } from "../services/native-status-reassessment-notifications.js"; import { notifyUnsafeWorkspaceExportRecovery } from "../services/unsafe-workspace-export-notifications.js"; @@ -16674,6 +16675,7 @@ export function issueRoutes( } // The click is durable fresh user intent, including when the message // predates a failed run's stop. Keep its content and original attribution. + await notifyDeferredExecutionWork(tx, { companyId: issue.companyId, requestId: locked.wake.id }); await tx.update(agentWakeupRequests).set({ payload: { ...readObject(locked.wake.payload), queuedCommentInterrupt: { actorId: actor.actorId, requestedAt: new Date().toISOString(), diff --git a/server/src/services/agent-configuration.ts b/server/src/services/agent-configuration.ts index dc10efaa082..2d30da27cb7 100644 --- a/server/src/services/agent-configuration.ts +++ b/server/src/services/agent-configuration.ts @@ -1,3 +1,4 @@ +import { notifyDeferredExecutionEligibility } from "./deferred-execution-notifications.js"; import { notifyRunDispatch } from "./run-dispatch-notifications.js"; import { notifySkillVersionPinChange } from "./skill-version-retention-notifications.js"; import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js"; @@ -85,6 +86,7 @@ export async function prepareAgentConfiguration(db: Db, existing: AgentConfigura ); } await notifyRunDispatch(db, { companyId: existing.companyId }); + await notifyDeferredExecutionEligibility(db, { companyId: existing.companyId }); if (normalizedPatch.runtimeConfig !== undefined || normalizedPatch.reportsTo !== undefined) { await notifyDeliveryWork(db, DELIVERY_QUEUES.agentTimers); } diff --git a/server/src/services/agent-lifecycle.ts b/server/src/services/agent-lifecycle.ts index 59357fcc5de..8bb46882256 100644 --- a/server/src/services/agent-lifecycle.ts +++ b/server/src/services/agent-lifecycle.ts @@ -1,3 +1,4 @@ +import { notifyDeferredExecutionEligibility } from "./deferred-execution-notifications.js"; import { notifyRunDispatch } from "./run-dispatch-notifications.js"; import { notifyNativeReviewResponse } from "./native-review-response-notifications.js"; import { isAgentStatusInvokable } from "@paperclipai/shared"; @@ -41,6 +42,7 @@ export function createAgentLifecycleEffects(_hooks: BudgetServiceHooks = {}): li async beforeStatusChange(tx, companyId, agentId, before, after) { if (before !== after) { await notifyRunDispatch(tx, { companyId }); + await notifyDeferredExecutionEligibility(tx, { companyId }); await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers); } if (!isAgentStatusInvokable(before) && isAgentStatusInvokable(after)) { diff --git a/server/src/services/budgets.ts b/server/src/services/budgets.ts index 186253b19bb..985197be3cd 100644 --- a/server/src/services/budgets.ts +++ b/server/src/services/budgets.ts @@ -1,3 +1,4 @@ +import { notifyDeferredExecutionEligibility } from "./deferred-execution-notifications.js"; import { notifyRunDispatch } from "./run-dispatch-notifications.js"; import { notifyApprovalWork } from "./approval-work-notifications.js"; import { notifyCompanyRetentionWork } from "./retention-work-notifications.js"; @@ -289,6 +290,7 @@ export function budgetServiceInTransaction(db: Db, publications: ActivityPublica if (policy.scopeType === "project") { await notifyRunDispatch(db, { companyId: policy.companyId }); + await notifyDeferredExecutionEligibility(db, { companyId: policy.companyId }); await db .update(projects) .set({ @@ -303,6 +305,7 @@ export function budgetServiceInTransaction(db: Db, publications: ActivityPublica const scope = await resolveScopeRecord(db, "company", policy.scopeId); if (scope.paused) return; await notifyRunDispatch(db, { companyId: policy.companyId }); + await notifyDeferredExecutionEligibility(db, { companyId: policy.companyId }); await notifyCompanyRetentionWork(db, policy.companyId); await notifyDeliveryWork(db, DELIVERY_QUEUES.agentTimers); await notifyDeliveryWork(db, DELIVERY_QUEUES.routineSchedules); @@ -350,6 +353,7 @@ export function budgetServiceInTransaction(db: Db, publications: ActivityPublica if (policy.scopeType === "project") { await notifyRunDispatch(db, { companyId: policy.companyId }); + await notifyDeferredExecutionEligibility(db, { companyId: policy.companyId }); await db .update(projects) .set({ @@ -365,6 +369,7 @@ export function budgetServiceInTransaction(db: Db, publications: ActivityPublica const scope = await resolveScopeRecord(db, "company", policy.scopeId); if (scope.pauseReason !== "budget") return; await notifyRunDispatch(db, { companyId: policy.companyId }); + await notifyDeferredExecutionEligibility(db, { companyId: policy.companyId }); await notifyCompanyRetentionWork(db, policy.companyId); await notifyDeliveryWork(db, DELIVERY_QUEUES.agentTimers); await notifyDeliveryWork(db, DELIVERY_QUEUES.routineSchedules); diff --git a/server/src/services/chat-channels/failed-run-retry.ts b/server/src/services/chat-channels/failed-run-retry.ts index c324a6f3c34..f8b8c3ebeab 100644 --- a/server/src/services/chat-channels/failed-run-retry.ts +++ b/server/src/services/chat-channels/failed-run-retry.ts @@ -1,3 +1,4 @@ +import { notifyDeferredExecutionWork, notifyTaskRetrySettlement } from "../deferred-execution-notifications.js"; import { notifyNativeResponsePresentation } from "../native-response-presentation-notifications.js"; import { notifyChatInboundWakeNotices } from "../chat-inbound-wake-notice-notifications.js"; import { createFailedRetryNotices } from "./failed-retry-notices.js"; @@ -1424,6 +1425,7 @@ export function createChatFailedRunRetry(db: Db, options: ChatFailedRunRetryOpti // Source authorization above checked every external receipt and its owner. // Reset only this certified unstarted batch under the task's admission lock. const skippedIds = source.taskRetryInput?.skippedActionIds ?? []; + await notifyDeferredExecutionWork(tx, { companyId: input.companyId }); await tx.update(agentWakeupRequests).set({ status: "deferred_issue_execution", runId: null, finishedAt: null, payload: sql`${agentWakeupRequests.payload} - 'coalescedIntoWakeupRequestId'`, updatedAt: new Date(), }).where(and(eq(agentWakeupRequests.companyId, input.companyId), eq(agentWakeupRequests.agentId, input.agentId), @@ -1784,6 +1786,9 @@ export function createChatFailedRunRetry(db: Db, options: ChatFailedRunRetryOpti const attemptCount = Number(candidate.result?.attemptCount ?? 0) + 1; const settle = async (status: string, result: Record) => { const [current] = await db.transaction(async (tx) => { + if (typeof claimed.payload.taskRetryIntentId === "string") await notifyTaskRetrySettlement(tx, { + companyId: claimed.companyId, requestId: claimed.payload.taskRetryIntentId, + }); await notifyChatActionWork(tx, "failed_run_retry"); if (status === "processed") await notifyNativeResponsePresentation(tx, { companyId: claimed.companyId, issueId: String(claimed.payload.issueId), diff --git a/server/src/services/chat-channels/task-retry-input.ts b/server/src/services/chat-channels/task-retry-input.ts index 71743f42ad4..8257a8ec079 100644 --- a/server/src/services/chat-channels/task-retry-input.ts +++ b/server/src/services/chat-channels/task-retry-input.ts @@ -1,3 +1,4 @@ +import { notifyDeferredExecutionWork } from "../deferred-execution-notifications.js"; import { z } from "zod"; import { and, asc, eq, gt, inArray, isNull, or, sql } from "drizzle-orm"; import { agentWakeupRequests, chatActions, chatConversations, chatEndpoints, heartbeatRuns, issues, type Db } from "@paperclipai/db"; @@ -91,6 +92,7 @@ export async function releaseUnadmittedTaskRetryInput(db: Db, input: { companyId )).for("update"); if (receipt?.runId) return; await notifyChatInboundWakeNotices(tx, { companyId: input.companyId, issueId: issue.id }); + await notifyDeferredExecutionWork(tx, { companyId: input.companyId }); await tx.update(agentWakeupRequests).set({ status: "deferred_issue_execution", finishedAt: null, payload: sql`${agentWakeupRequests.payload} - 'coalescedIntoWakeupRequestId'`, updatedAt: new Date(), }).where(and(eq(agentWakeupRequests.companyId, input.companyId), eq(agentWakeupRequests.status, "coalesced"), diff --git a/server/src/services/companies.ts b/server/src/services/companies.ts index a9a8f59998c..f0522ee4ddb 100644 --- a/server/src/services/companies.ts +++ b/server/src/services/companies.ts @@ -1,3 +1,4 @@ +import { notifyDeferredExecutionEligibility } from "./deferred-execution-notifications.js"; import { notifyRunDispatch } from "./run-dispatch-notifications.js"; import { notifyChatInboundWakeNotices } from "./chat-inbound-wake-notice-notifications.js"; import { notifyCompanyRetentionWork } from "./retention-work-notifications.js"; @@ -334,6 +335,7 @@ export function companyService(db: Db, _budgetHooks: BudgetServiceHooks = {}) { if (companyPatch.status !== undefined && companyPatch.status !== existing.status) { await notifyRunDispatch(tx, { companyId: id }); + await notifyDeferredExecutionEligibility(tx, { companyId: id }); await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers); await notifyDeliveryWork(tx, DELIVERY_QUEUES.skillSourceSync); await notifyDeliveryWork(tx, DELIVERY_QUEUES.routineSchedules); @@ -491,6 +493,7 @@ export function companyService(db: Db, _budgetHooks: BudgetServiceHooks = {}) { if (!wasAlreadyArchived) { await notifyRunDispatch(tx, { companyId: id }); + await notifyDeferredExecutionEligibility(tx, { companyId: id }); await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers); await notifyDeliveryWork(tx, DELIVERY_QUEUES.skillSourceSync); await notifyDeliveryWork(tx, DELIVERY_QUEUES.routineSchedules); diff --git a/server/src/services/deferred-execution-notifications.ts b/server/src/services/deferred-execution-notifications.ts new file mode 100644 index 00000000000..b82cea725a5 --- /dev/null +++ b/server/src/services/deferred-execution-notifications.ts @@ -0,0 +1,34 @@ +import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js"; + +export type DeferredExecutionScope = { companyId: string; requestId?: string }; +export type DeferredReceiptRecoveryOptions = { + companyId?: string; + requestId?: string; + onError?: (error: unknown) => void; +}; + +function hint(scope: DeferredExecutionScope) { + return JSON.stringify({ companyId: scope.companyId, requestId: scope.requestId }); +} + +export async function notifyDeferredExecutionWork(tx: object, scope: DeferredExecutionScope) { + await notifyDeliveryWork(tx, DELIVERY_QUEUES.deferredExecutionReceipts, hint(scope)); +} + +export async function notifyNativeStatusWakeWork(tx: object, scope: DeferredExecutionScope) { + await notifyDeliveryWork(tx, DELIVERY_QUEUES.nativeStatusWakeIntents, hint(scope)); +} + +export async function notifyTaskRetrySettlement(tx: object, scope: DeferredExecutionScope) { + await notifyDeliveryWork(tx, DELIVERY_QUEUES.taskRetryReceiptSettlement, hint(scope)); +} + +export async function notifyDeferredExecutionEligibility(tx: object, scope?: { companyId: string }) { + if (scope) { + await notifyDeferredExecutionWork(tx, scope); + await notifyNativeStatusWakeWork(tx, scope); + } else { + await notifyDeliveryWork(tx, DELIVERY_QUEUES.deferredExecutionReceipts, "*"); + await notifyDeliveryWork(tx, DELIVERY_QUEUES.nativeStatusWakeIntents, "*"); + } +} diff --git a/server/src/services/delivery-work-notifications.ts b/server/src/services/delivery-work-notifications.ts index 8a9fdbe5686..79f0f5d1d3d 100644 --- a/server/src/services/delivery-work-notifications.ts +++ b/server/src/services/delivery-work-notifications.ts @@ -2,6 +2,9 @@ import { signalDatabaseWork, subscribeDatabaseWork } from "@paperclipai/db"; /** Topics for existing durable queues; register intent before their writes. */ export const DELIVERY_QUEUES = { + deferredExecutionReceipts: "deferred-execution-receipts", + taskRetryReceiptSettlement: "task-retry-receipt-settlement", + nativeStatusWakeIntents: "native-status-wake-intents", orphanBlockers: "orphan-blockers", activeRecoveryActions: "active-recovery-actions", strandedAssignments: "stranded-assignments", diff --git a/server/src/services/execution-control-notifications.ts b/server/src/services/execution-control-notifications.ts index 1839176769f..7d9530f5197 100644 --- a/server/src/services/execution-control-notifications.ts +++ b/server/src/services/execution-control-notifications.ts @@ -1,3 +1,4 @@ +import { notifyDeferredExecutionEligibility } from "./deferred-execution-notifications.js"; import { notifyRunDispatch } from "./run-dispatch-notifications.js"; import type { Db } from "@paperclipai/db"; import { DELIVERY_QUEUES, notifyDeliveryWork, type DeliveryQueue } from "./delivery-work-notifications.js"; @@ -5,7 +6,10 @@ import { DELIVERY_QUEUES, notifyDeliveryWork, type DeliveryQueue } from "./deliv export type ExecutionWorkScope = { companyId: string; agentId?: string; runId?: string }; export async function notifyExecutionRunWork(transaction: object, scope?: ExecutionWorkScope) { - if (scope) await notifyRunDispatch(transaction, scope); + if (scope) { + await notifyRunDispatch(transaction, scope); + await notifyDeferredExecutionEligibility(transaction, scope); + } for (const queue of [DELIVERY_QUEUES.executionStatus, DELIVERY_QUEUES.executionFinalization, DELIVERY_QUEUES.executionReplacement, DELIVERY_QUEUES.executionDisposition, DELIVERY_QUEUES.issueMonitors]) { await notifyDeliveryWork(transaction, queue); @@ -20,6 +24,7 @@ export function writeExecutionControl( ): Promise { return db.transaction(async transaction => { await notifyRunDispatch(transaction, scope); + if (scope) await notifyDeferredExecutionEligibility(transaction, scope); await notifyDeliveryWork(transaction, DELIVERY_QUEUES.issueMonitors); for (const topic of typeof queue === "string" ? [queue] : queue) await notifyDeliveryWork(transaction, topic); return write(transaction as unknown as Db); diff --git a/server/src/services/heartbeat.ts b/server/src/services/heartbeat.ts index 068532357ea..63b836a566f 100644 --- a/server/src/services/heartbeat.ts +++ b/server/src/services/heartbeat.ts @@ -1,3 +1,5 @@ +import { registerDeferredReceiptWork } from "./heartbeat/deferred-receipt-work.js"; +import { reconcileTaskChatRetryReceipts } from "./task-retry.js"; import { registerIssueRecoveryWork } from "./recovery/work.js"; import { registerRunDispatchWork } from "./heartbeat/dispatch-work.js"; import { externalConversationSettlementWork } from "./external-conversation-settlement-work.js"; @@ -4243,6 +4245,13 @@ export function heartbeatService( computeTaskDrain, applyTaskDrain, + registerDeferredReceiptWork: (coordinator: Parameters[1], canRun: () => boolean) => + registerDeferredReceiptWork(db, coordinator, { + resume: scope => heartbeatQueue.resumeDeferredExecutionReceipts(scope), + settleTaskChat: scope => reconcileTaskChatRetryReceipts(db, { ...scope, settleOnly: true }), + dispatchNative: dispatchPendingNativeStatusWakeups, + schedulingSuppressed: async () => (await getSchedulingSuppression(true)).suppressed, + }, () => canRun() && readTaskDrain(new Date()) === null), registerIssueStateRecoveryWork: (coordinator: Parameters[1], canRun: () => boolean) => registerIssueRecoveryWork(db, coordinator, { stranded: scope => reconcileStrandedAssignedIssues({ ...scope, skipExternalConversationSettlement: true, skipRelatedRecovery: true }), diff --git a/server/src/services/heartbeat/deferred-receipt-work.ts b/server/src/services/heartbeat/deferred-receipt-work.ts new file mode 100644 index 00000000000..fddbce3be8d --- /dev/null +++ b/server/src/services/heartbeat/deferred-receipt-work.ts @@ -0,0 +1,100 @@ +import { and, asc, eq, gt, inArray, isNull, or, sql } from "drizzle-orm"; +import { agentWakeupRequests, type Db } from "@paperclipai/db"; +import type { createDeliveryWorkCoordinator } from "../delivery-work-coordinator.js"; +import { DELIVERY_QUEUES } from "../delivery-work-notifications.js"; +import type { DeferredExecutionScope, DeferredReceiptRecoveryOptions } from "../deferred-execution-notifications.js"; +import { registerScopedDeadlineWork } from "../scoped-deadline-work.js"; + +type Entry = { id: string; companyId: string; dueAt: number }; +type NativeIntentOptions = DeferredExecutionScope & { + onDeferredDeadline(at: number): void; + onError(error: unknown): void; +}; + +export function registerDeferredReceiptWork( + db: Db, + coordinator: ReturnType, + owner: { + resume(scope: DeferredReceiptRecoveryOptions): Promise; + settleTaskChat(scope: DeferredExecutionScope): Promise; + dispatchNative(scope: NativeIntentOptions): Promise; + schedulingSuppressed(): Promise; + }, + canRun: () => boolean, +) { + function register(kind: "deferred" | "task_chat" | "native") { + const cursors = new Map(); + let enabled = false; + const queue = kind === "deferred" ? DELIVERY_QUEUES.deferredExecutionReceipts + : kind === "task_chat" ? DELIVERY_QUEUES.taskRetryReceiptSettlement : DELIVERY_QUEUES.nativeStatusWakeIntents; + return registerScopedDeadlineWork(coordinator, { + queue, + reconstructionIntervalMs: 30_000, + beforePass: async () => { enabled = !(await owner.schedulingSuppressed()); }, + decodeScope: hint => hint === "*" ? undefined : JSON.parse(hint), + matches: (entry, scope) => !scope || entry.companyId === scope.companyId && (!scope.requestId || entry.id === scope.requestId), + load: async scope => { + const key = scope ? JSON.stringify({ companyId: scope.companyId, requestId: scope.requestId }) : "*"; + if (!enabled) { cursors.delete(key); return []; } + const cursor = cursors.get(key); + const rows = await db.select({ id: agentWakeupRequests.id, companyId: agentWakeupRequests.companyId, + status: agentWakeupRequests.status, updatedAt: agentWakeupRequests.updatedAt, + claimedAt: agentWakeupRequests.claimedAt, error: agentWakeupRequests.error, + delayed: sql`${agentWakeupRequests.payload}->'queuedCommentInterrupt' is not null + or ${agentWakeupRequests.payload}->'executionWait' is not null`, + interruptedLegacy: sql`exists (select 1 from heartbeat_runs r + where r.company_id = ${agentWakeupRequests.companyId} and r.agent_id = ${agentWakeupRequests.agentId} + and r.runtime_mode = 'legacy' and r.status = 'cancelled' + and r.result_json->>'queuedCommentInterruptQueueId' = ${agentWakeupRequests.id}::text)`, + }).from(agentWakeupRequests).where(and( + scope ? eq(agentWakeupRequests.companyId, scope.companyId) : undefined, + scope?.requestId ? eq(agentWakeupRequests.id, scope.requestId) : undefined, + cursor ? gt(agentWakeupRequests.id, cursor) : undefined, + kind === "native" ? and( + eq(agentWakeupRequests.requestedByActorType, "system"), + eq(agentWakeupRequests.requestedByActorId, "native-status-committer"), + inArray(agentWakeupRequests.status, ["queued", "claimed"]), isNull(agentWakeupRequests.runId), + ) : and(eq(agentWakeupRequests.status, "deferred_issue_execution"), + kind === "task_chat" ? and(eq(agentWakeupRequests.reason, "task_retry"), + sql`${agentWakeupRequests.payload}#>>'{taskRetry,chatActionId}' is not null`) : or( + eq(agentWakeupRequests.reason, "task_retry"), + sql`${agentWakeupRequests.payload}->'queuedCommentInterrupt' is not null`, + sql`${agentWakeupRequests.payload}->'executionWait' is not null`, + sql`jsonb_typeof(${agentWakeupRequests.payload}#>'{_paperclipWakeContext,wakeCommentIds}') = 'array' + and ${agentWakeupRequests.payload}#>'{_paperclipWakeContext,wakeCommentIds}' <> '[]'::jsonb`, + sql`${agentWakeupRequests.payload}->>'mutation' = 'interaction'`, + sql`exists (select 1 from heartbeat_runs r where r.company_id = ${agentWakeupRequests.companyId} + and r.runtime_mode = 'legacy' and r.status = 'cancelled' + and r.result_json->>'queuedCommentInterruptQueueId' = ${agentWakeupRequests.id}::text)`, + )), + )).orderBy(asc(agentWakeupRequests.id)).limit(100); + const complete = rows.length < 100; + if (complete) cursors.delete(key); else cursors.set(key, rows.at(-1)!.id); + return { rows: rows.map(row => ({ id: row.id, companyId: row.companyId, + dueAt: kind === "native" && row.status === "claimed" && row.claimedAt ? row.claimedAt.getTime() + 60_000 + : kind === "native" && row.error ? row.updatedAt.getTime() + 30_000 + : kind === "deferred" && row.delayed && !row.interruptedLegacy ? row.updatedAt.getTime() + 30_000 + : Date.now(), + })), complete }; + }, + check: async entry => { + if (!enabled) return null; + const errors: unknown[] = []; + let next: number | null = null; + const scope = { companyId: entry.companyId, requestId: entry.id }; + if (kind === "native") await owner.dispatchNative({ ...scope, + onDeferredDeadline: at => { next = next === null ? at : Math.min(next, at); }, + onError: error => { errors.push(error); }, + }); + else if (kind === "task_chat") await owner.settleTaskChat(scope); + else await owner.resume({ ...scope, onError: error => { errors.push(error); } }); + if (errors.length) throw new AggregateError(errors, `Deferred receipt ${entry.id} could not settle`); + return next === null ? null : { ...entry, dueAt: next }; + }, + }, canRun); + } + const workers = [register("deferred"), register("task_chat"), register("native")]; + return { ready: Promise.all(workers.map(worker => worker.ready)), + wake() { for (const worker of workers) worker.invalidate(); }, + async stop() { await Promise.all(workers.map(worker => worker.stop())); } }; +} diff --git a/server/src/services/heartbeat/queue.ts b/server/src/services/heartbeat/queue.ts index 64ca0b84ffd..ef3ecfeb7ac 100644 --- a/server/src/services/heartbeat/queue.ts +++ b/server/src/services/heartbeat/queue.ts @@ -1,3 +1,5 @@ +import { notifyDeferredExecutionWork } from "../deferred-execution-notifications.js"; +import type { DeferredReceiptRecoveryOptions } from "../deferred-execution-notifications.js"; import { notifyRunDispatch } from "../run-dispatch-notifications.js"; import { DELIVERY_QUEUES, notifyDeliveryWork } from "../delivery-work-notifications.js"; import { notifyExternalAgentSettlement } from "../../modules/external-agents/index.js"; @@ -222,7 +224,7 @@ export interface HeartbeatQueueDependencies extends Pick void; cancelRunInternal: (runId: string, reason?: string, options?: { errorCode?: string }) => Promise; cancelActiveForAgentInternal: (agentId: string, reason: string) => Promise; - resumeExecutionWaitComments: () => Promise; + resumeExecutionWaitComments: (scope?: DeferredReceiptRecoveryOptions) => Promise; resumeQueuedCommentInterrupt: (companyId: string, queueId: string) => Promise; resumeSavedLegacyComments: (companyId: string, queueId: string) => Promise; formatIssueIdentifierLink: (identifier: string | null, fallback: string) => string; @@ -1851,13 +1853,15 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende return cancelled; } - async function resumeQueuedRuns() { + async function resumeDeferredExecutionReceipts(scope: DeferredReceiptRecoveryOptions = {}) { if ((await getSchedulingSuppression()).suppressed) return; - await resumeExecutionWaitComments(); + await resumeExecutionWaitComments(scope); const cutoff = await getWorktreeExecutionCutoff(); const pendingInterrupts = await db.select({ id: agentWakeupRequests.id, companyId: agentWakeupRequests.companyId }) .from(agentWakeupRequests).innerJoin(companies, eq(companies.id, agentWakeupRequests.companyId)) - .where(and(eq(agentWakeupRequests.status, "deferred_issue_execution"), + .where(and(scope.companyId ? eq(agentWakeupRequests.companyId, scope.companyId) : undefined, + scope.requestId ? eq(agentWakeupRequests.id, scope.requestId) : undefined, + eq(agentWakeupRequests.status, "deferred_issue_execution"), eq(companies.status, "active"), sql`${agentWakeupRequests.payload}->'queuedCommentInterrupt' is not null`, lte(agentWakeupRequests.updatedAt, new Date(Date.now() - 30_000)), @@ -1868,6 +1872,7 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende eq(agentWakeupRequests.id, wake.id), eq(agentWakeupRequests.status, "deferred_issue_execution"), )); await resumeQueuedCommentInterrupt(wake.companyId, wake.id).catch(err => { + scope.onError?.(err); logger.warn({ err, queueId: wake.id }, "failed to resume interrupted comment queue"); }); } @@ -1881,7 +1886,9 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende or(eq(issues.assigneeAgentId, agentWakeupRequests.agentId), sql`${agentWakeupRequests.payload} #>> '{_paperclipWakeContext,wakeReason}' = 'issue_comment_mentioned'`))) .innerJoin(companies, and(eq(companies.id, issues.companyId), eq(companies.status, "active"))) - .where(and(eq(agentWakeupRequests.status, "deferred_issue_execution"), + .where(and(scope.companyId ? eq(agentWakeupRequests.companyId, scope.companyId) : undefined, + scope.requestId ? eq(agentWakeupRequests.id, scope.requestId) : undefined, + eq(agentWakeupRequests.status, "deferred_issue_execution"), isNull(issues.executionRunId), or(and( sql`jsonb_typeof(${agentWakeupRequests.payload} #> '{_paperclipWakeContext,wakeCommentIds}') = 'array'`, @@ -1920,7 +1927,8 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende if (latest.runtimeMode !== "legacy" && latest.runtimeMode !== "native") continue; if (wake.idempotencyKey?.startsWith("chat-inbound:")) { await resumeDeferredChatWake(db, { companyId: wake.companyId, agentId: wake.agentId, - issueId: String(wake.payload?.issueId), requestId: wake.id }).catch(() => { + issueId: String(wake.payload?.issueId), requestId: wake.id }).catch(err => { + scope.onError?.(err); logger.warn({ requestId: wake.id }, "saved chat source could not resume after recovery"); }); continue; @@ -1932,6 +1940,7 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende latest.errorCode === "execution_reconciliation_required"; if ((latest.status !== "cancelled" || cancelledAdmission) && await getExecutionBlocker(db, wake.companyId, String(wake.payload?.issueId))) { await resumeSavedLegacyComments(wake.companyId, wake.id).catch(err => { + scope.onError?.(err); logger.warn({ err, queueId: wake.id }, "failed to deliver saved legacy comment after recovery stopped"); }); } @@ -1939,6 +1948,7 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende await releaseIssueExecutionAndPromote(latest, { suppressImmediateRecovery: true, resumeStrandedQueue: true }).then(() => { if (latest.runtimeMode === "native") releasedNativeSources.add(releaseSource); }).catch(err => { + scope.onError?.(err); logger.warn({ err, queueId: wake.id }, "failed to promote stranded comments"); }); } @@ -1956,6 +1966,8 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende )) .innerJoin(companies, eq(companies.id, heartbeatRuns.companyId)) .where(and( + scope.companyId ? eq(agentWakeupRequests.companyId, scope.companyId) : undefined, + scope.requestId ? eq(agentWakeupRequests.id, scope.requestId) : undefined, eq(agentWakeupRequests.status, "deferred_issue_execution"), eq(heartbeatRuns.status, "cancelled"), eq(heartbeatRuns.runtimeMode, "legacy"), @@ -1964,10 +1976,15 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende )); for (const run of interruptedQueues) { await releaseIssueExecutionAndPromote(run, { suppressImmediateRecovery: true }).catch((err) => { + scope.onError?.(err); logger.error({ err, runId: run.id }, "failed to retry interrupted comment queue"); }); } + } + + async function resumeQueuedRuns() { + await resumeDeferredExecutionReceipts(); await resumeQueuedRunsAfterDrain(); } @@ -3413,6 +3430,7 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende if (durableRequest || wakeCommentId || hasInteractionContinuationWakeContext(enrichedContextSnapshot) || readNonEmptyString(enrichedContextSnapshot.nativeStatusWakeIntentId)) { + await notifyDeferredExecutionWork(tx, { companyId: agent.companyId }); await tx.insert(agentWakeupRequests).values({ ...durableReceiptFields, companyId: agent.companyId, agentId, source, triggerDetail, reason, @@ -4030,6 +4048,7 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende })); } if (followUp) { + if (!followUp.consumed) await notifyDeferredExecutionWork(tx, { companyId: agent.companyId }); const assignmentId = followUp.assignmentId; const pending = followUp.consumed ? undefined : pendingFollowUps.find(wake => wake.payload?.[followUpPayloadKey] === assignmentId && @@ -4768,6 +4787,9 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende companyId?: string; limit?: number; staleClaimMs?: number; + requestId?: string; + onDeferredDeadline?: (at: number) => void; + onError?: (error: unknown) => void; } = {}, ) { const now = new Date(); @@ -4780,6 +4802,7 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende input.companyId ? eq(agentWakeupRequests.companyId, input.companyId) : undefined, + input.requestId ? eq(agentWakeupRequests.id, input.requestId) : undefined, eq(agentWakeupRequests.requestedByActorType, "system"), eq(agentWakeupRequests.requestedByActorId, "native-status-committer"), inArray(agentWakeupRequests.status, ["queued", "claimed"]), @@ -4856,6 +4879,7 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende candidate.claimedAt && now.getTime() - candidate.claimedAt.getTime() < staleClaimMs ) { + input.onDeferredDeadline?.(candidate.claimedAt.getTime() + staleClaimMs); deferred += 1; continue; } @@ -5030,6 +5054,7 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende if (wakeRun) dispatched += 1; else deferred += 1; } catch (error) { + input.onError?.(error); await db .update(agentWakeupRequests) .set({ @@ -5073,6 +5098,7 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende withChatControlRecoveryGate, trackWakeup, resumeQueuedRuns, + resumeDeferredExecutionReceipts, resumeQueuedRunsAfterDrain, parseHeartbeatPolicy, nextTimerHeartbeatAt, diff --git a/server/src/services/heartbeat/recovery.ts b/server/src/services/heartbeat/recovery.ts index 3b180d2679d..e506f0f7e62 100644 --- a/server/src/services/heartbeat/recovery.ts +++ b/server/src/services/heartbeat/recovery.ts @@ -1979,7 +1979,7 @@ export function createHeartbeatRecovery(db: Db, dependencies: HeartbeatRecoveryD let settledProcessesDiscovered = false; - async function reapOrphanedRuns(opts?: { staleThresholdMs?: number; skipScheduledRetryCancellation?: boolean; skipCapturedInstructionCommit?: boolean; skipInstructionDirectoryCleanup?: boolean; skipStoppedInstructionCollection?: boolean; skipCancelledWorkspacePreservation?: boolean; skipObsoletePolicyReviews?: boolean; skipAutomaticCompletionReviews?: boolean; skipUnsafeWorkspaceExports?: boolean; skipNativePendingEffects?: boolean; skipNativeResponsePresentation?: boolean; skipNativeReviewResponse?: boolean; skipNativeStatusReassessment?: boolean; skipRetainedNativeSessionCleanup?: boolean; skipNativeWorkspaceFinalization?: boolean; skipNativeResultCommit?: boolean; skipNativeRunProjection?: boolean; skipNativeRecoveryNotices?: boolean; skipExternalConversationSettlement?: boolean; skipExternalAgentSettlement?: boolean; skipNativeEnvironmentSettlement?: boolean }) { + async function reapOrphanedRuns(opts?: { staleThresholdMs?: number; skipNativeStatusWakeIntents?: boolean; skipScheduledRetryCancellation?: boolean; skipCapturedInstructionCommit?: boolean; skipInstructionDirectoryCleanup?: boolean; skipStoppedInstructionCollection?: boolean; skipCancelledWorkspacePreservation?: boolean; skipObsoletePolicyReviews?: boolean; skipAutomaticCompletionReviews?: boolean; skipUnsafeWorkspaceExports?: boolean; skipNativePendingEffects?: boolean; skipNativeResponsePresentation?: boolean; skipNativeReviewResponse?: boolean; skipNativeStatusReassessment?: boolean; skipRetainedNativeSessionCleanup?: boolean; skipNativeWorkspaceFinalization?: boolean; skipNativeResultCommit?: boolean; skipNativeRunProjection?: boolean; skipNativeRecoveryNotices?: boolean; skipExternalConversationSettlement?: boolean; skipExternalAgentSettlement?: boolean; skipNativeEnvironmentSettlement?: boolean }) { if (!opts?.skipCancelledWorkspacePreservation) await preserveCancelledNativeWorkspaces(db); const staleThresholdMs = opts?.staleThresholdMs ?? 0; const now = new Date(); @@ -2023,7 +2023,7 @@ export function createHeartbeatRecovery(db: Db, dependencies: HeartbeatRecoveryD }); if (!opts?.skipRetainedNativeSessionCleanup) scheduleRetainedNativeSessionCleanup(); await reconcileRetainedWorkspaces(db, environmentRuntime, resumeRemoteStopComments); - await dispatchPendingNativeStatusWakeups().catch((error) => { + if (!opts?.skipNativeStatusWakeIntents) await dispatchPendingNativeStatusWakeups().catch((error) => { logger.warn( { err: error }, "failed to dispatch persisted native status wake intents before orphan reaping", diff --git a/server/src/services/heartbeat/run-control.ts b/server/src/services/heartbeat/run-control.ts index 820f1e50c61..691309a1d76 100644 --- a/server/src/services/heartbeat/run-control.ts +++ b/server/src/services/heartbeat/run-control.ts @@ -1,3 +1,4 @@ +import type { DeferredReceiptRecoveryOptions } from "../deferred-execution-notifications.js"; import { observeStoppedWorkspaceRecovery } from "../workspace-recovery.js"; import { notifyScheduledRetryCancellation } from "../scheduled-retry-cancellation-notifications.js"; import { notifyChatInboundWakeNotices } from "../chat-inbound-wake-notice-notifications.js"; @@ -777,21 +778,25 @@ export function createHeartbeatRunControl(db: Db, dependencies: HeartbeatRunCont idempotencyKey: `task-retry-dispatch:${wake.id}` }, wake.id); } - async function resumeExecutionWaitComments() { + async function resumeExecutionWaitComments(scope: DeferredReceiptRecoveryOptions = {}) { if ((await getSchedulingSuppression()).suppressed) return; - await reconcileTaskChatRetryReceipts(db); + await reconcileTaskChatRetryReceipts(db, scope); const retries = await db.select({ id: agentWakeupRequests.id, companyId: agentWakeupRequests.companyId }) .from(agentWakeupRequests).innerJoin(issues, and(eq(issues.companyId, agentWakeupRequests.companyId), sql`${issues.id}::text = ${agentWakeupRequests.payload}->>'issueId'`, eq(issues.assigneeAgentId, agentWakeupRequests.agentId))) .innerJoin(companies, and(eq(companies.id, issues.companyId), eq(companies.status, "active"))) - .where(and(eq(agentWakeupRequests.reason, "task_retry"), eq(agentWakeupRequests.status, "deferred_issue_execution"), + .where(and(scope.companyId ? eq(agentWakeupRequests.companyId, scope.companyId) : undefined, + scope.requestId ? eq(agentWakeupRequests.id, scope.requestId) : undefined, + eq(agentWakeupRequests.reason, "task_retry"), eq(agentWakeupRequests.status, "deferred_issue_execution"), sql`${agentWakeupRequests.payload}->'taskRetry'->>'chatActionId' is null`, notInArray(issues.status, ["done", "cancelled"]))) .orderBy(asc(agentWakeupRequests.updatedAt)).limit(50); for (const retry of retries) { await db.update(agentWakeupRequests).set({ updatedAt: new Date() }).where(eq(agentWakeupRequests.id, retry.id)); - await resumeTaskRetry(retry.companyId, retry.id).catch(err => logger.warn({ err, requestId: retry.id }, "Task retry dispatch deferred")); + await resumeTaskRetry(retry.companyId, retry.id).catch(err => { + scope.onError?.(err); logger.warn({ err, requestId: retry.id }, "Task retry dispatch deferred"); + }); } const waits = await db.select({ wake: agentWakeupRequests }) .from(agentWakeupRequests) @@ -799,7 +804,9 @@ export function createHeartbeatRunControl(db: Db, dependencies: HeartbeatRunCont sql`${issues.id}::text = ${agentWakeupRequests.payload}->>'issueId'`, eq(issues.assigneeAgentId, agentWakeupRequests.agentId))) .innerJoin(companies, and(eq(companies.id, issues.companyId), eq(companies.status, "active"))) - .where(and(exists(db.select({ id: issueRecoveryActions.id }).from(issueRecoveryActions).where(and( + .where(and(scope.companyId ? eq(agentWakeupRequests.companyId, scope.companyId) : undefined, + scope.requestId ? eq(agentWakeupRequests.id, scope.requestId) : undefined, + exists(db.select({ id: issueRecoveryActions.id }).from(issueRecoveryActions).where(and( eq(issueRecoveryActions.companyId, issues.companyId), eq(issueRecoveryActions.sourceIssueId, issues.id), executionBlockerPredicate(), ))), eq(agentWakeupRequests.status, "deferred_issue_execution"), @@ -831,11 +838,13 @@ export function createHeartbeatRunControl(db: Db, dependencies: HeartbeatRunCont if (!run || run.companyId !== wake.companyId || run.agentId !== wake.agentId) continue; if (canContinueCancelledRun(run)) { await resumeSavedComments(wake.companyId, wake.id).catch(err => { + scope.onError?.(err); logger.warn({ err, runId: run.id }, "failed to resume saved input after provider cancellation"); }); continue; } await resumeRemoteStopComments(run, wake.id).catch(err => { + scope.onError?.(err); logger.warn({ err, runId: run.id }, "failed to resume saved execution-wait message"); }); } diff --git a/server/src/services/instance-settings.ts b/server/src/services/instance-settings.ts index 174c39d6125..6c5f948d303 100644 --- a/server/src/services/instance-settings.ts +++ b/server/src/services/instance-settings.ts @@ -1,3 +1,4 @@ +import { notifyDeferredExecutionEligibility } from "./deferred-execution-notifications.js"; import { notifyRunDispatch } from "./run-dispatch-notifications.js"; import { notifySkillVersionRetentionReconstruction } from "./skill-version-retention-notifications.js"; import { notifyNativeEnvironmentSettlement } from "./native-environment-settlement-notifications.js"; @@ -632,6 +633,7 @@ export function instanceSettingsService(db: Db, options: InstanceSettingsService if (patch.enableWorktreeRunExecution !== undefined) await notifyScheduledRetryCancellation(tx); if (patch.enableWorktreeRunExecution !== undefined) { await notifyRunDispatch(tx); + await notifyDeferredExecutionEligibility(tx); await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers); await notifyExternalAgentSettlement(tx); await notifyInstructionCommit(tx); diff --git a/server/src/services/issues.ts b/server/src/services/issues.ts index 34234b01c98..fa47af138b9 100644 --- a/server/src/services/issues.ts +++ b/server/src/services/issues.ts @@ -1,3 +1,4 @@ +import { notifyDeferredExecutionEligibility } from "./deferred-execution-notifications.js"; import { notifyIssueRecovery } from "./issue-recovery-notifications.js"; import { notifyRunDispatch } from "./run-dispatch-notifications.js"; import { notifyExternalConversationSettlement } from "./external-conversation-settlement-notifications.js"; @@ -7629,6 +7630,7 @@ export function issueService(db: Db) { } await notifyIssueRecovery(dbOrTx, { companyId }); + await notifyDeferredExecutionEligibility(dbOrTx, { companyId }); await dbOrTx .delete(issueRelations) .where( @@ -10612,6 +10614,7 @@ export function issueService(db: Db) { } if (values.monitorNextCheckAt) await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors); await notifyRunDispatch(tx, { companyId: values.companyId }); + await notifyDeferredExecutionEligibility(tx, { companyId: values.companyId }); await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers); const [issue] = await tx.insert(issues).values(values).returning(); await attachOwnedDraftImages(tx, issue, issue.description, { userId: issueData.createdByUserId, agentId: issueData.createdByAgentId }); @@ -11503,6 +11506,7 @@ export function issueService(db: Db) { || patch.executionState !== undefined || patch.executionRunId !== undefined || patch.checkoutRunId !== undefined || blockedByIssueIds !== undefined) { await notifyRunDispatch(tx, { companyId: receiptExisting.companyId }); + await notifyDeferredExecutionEligibility(tx, { companyId: receiptExisting.companyId }); await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers); } if (patch.monitorNextCheckAt !== undefined || (receiptExisting.monitorNextCheckAt && @@ -11968,6 +11972,7 @@ export function issueService(db: Db) { await notifyDecisionTargetIssueRemoval(tx, id); await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors); await notifyRunDispatch(tx, owner ? { companyId: owner.companyId } : undefined); + if (owner) await notifyDeferredExecutionEligibility(tx, owner); await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers); removedIssue = await tx .delete(issues) @@ -12097,6 +12102,7 @@ export function issueService(db: Db) { const updated = await db.transaction(async tx => { await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors); await notifyRunDispatch(tx, { companyId: issueCompany.companyId }); + await notifyDeferredExecutionEligibility(tx, { companyId: issueCompany.companyId }); await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers); return tx .update(issues) @@ -12227,6 +12233,7 @@ export function issueService(db: Db) { const adopted = await db.transaction(async tx => { await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors); await notifyRunDispatch(tx, { companyId: issueCompany.companyId }); + await notifyDeferredExecutionEligibility(tx, { companyId: issueCompany.companyId }); await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers); return tx .update(issues) @@ -12483,6 +12490,7 @@ export function issueService(db: Db) { await notifyNativeStatusReassessment(tx, { companyId: existing.companyId, issueId: id }); await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors); await notifyRunDispatch(tx, { companyId: existing.companyId }); + await notifyDeferredExecutionEligibility(tx, { companyId: existing.companyId }); await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers); const updated = await tx .update(issues) @@ -12544,6 +12552,7 @@ export function issueService(db: Db) { } await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors); await notifyRunDispatch(tx, { companyId: existing.companyId }); + await notifyDeferredExecutionEligibility(tx, { companyId: existing.companyId }); await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers); const updated = await tx .update(issues) diff --git a/server/src/services/native-runtime/status-decision-committer.ts b/server/src/services/native-runtime/status-decision-committer.ts index a43868dd25f..3ea3096b710 100644 --- a/server/src/services/native-runtime/status-decision-committer.ts +++ b/server/src/services/native-runtime/status-decision-committer.ts @@ -1,3 +1,4 @@ +import { notifyNativeStatusWakeWork } from "../deferred-execution-notifications.js"; import { notifyExternalAgentSettlement } from "../../modules/external-agents/index.js"; import { lockCurrentNativeWorkAssessmentEvidence } from "./work-assessments.js"; import { hasNativeWorkspaceFinalizationOwner } from "./native-workspace-finalization-recovery-state.js"; @@ -478,6 +479,7 @@ async function enqueueWake(input: { .limit(1) .then((rows) => rows[0] ?? null); if (existing) return existing.id; + await notifyNativeStatusWakeWork(input.tx, { companyId: input.companyId }); const inserted = await input.tx .insert(agentWakeupRequests) .values({ diff --git a/server/src/services/task-retry.ts b/server/src/services/task-retry.ts index 17075c34eb9..ba5959b3eb6 100644 --- a/server/src/services/task-retry.ts +++ b/server/src/services/task-retry.ts @@ -339,8 +339,10 @@ export async function requestTaskRetry(input: { /** Chat owns admission and authorization. The task receipt follows that * durable outcome; it never launches a parallel generic board execution. */ -export async function reconcileTaskChatRetryReceipts(db: Db) { +export async function reconcileTaskChatRetryReceipts(db: Db, scope: { companyId?: string; requestId?: string; settleOnly?: boolean } = {}) { const pending = await db.select().from(agentWakeupRequests).where(and( + scope.companyId ? eq(agentWakeupRequests.companyId, scope.companyId) : undefined, + scope.requestId ? eq(agentWakeupRequests.id, scope.requestId) : undefined, eq(agentWakeupRequests.reason, "task_retry"), eq(agentWakeupRequests.status, "deferred_issue_execution"), sql`${agentWakeupRequests.payload}#>>'{taskRetry,chatActionId}' is not null`, )).orderBy(agentWakeupRequests.updatedAt).limit(50); @@ -356,7 +358,7 @@ export async function reconcileTaskChatRetryReceipts(db: Db) { // A deferred chat receipt is admitted input, not a completed dispatch. // Re-stage that exact action through chat's owner; it rechecks authority // and normal queue policy before updating the existing receipt. - if (chat?.status === "processed" && chat.payload.taskRetryIntentId === wake.id && + if (!scope.settleOnly && chat?.status === "processed" && chat.payload.taskRetryIntentId === wake.id && chat.payload.failedRunId === intent.sourceRunId && receipt?.status === "deferred_issue_execution" && !receipt.runId && receipt.payload?.taskRetryRequestId === wake.id) { await db.transaction(async tx => { From 71d7b9f09e5a1a0c5c38bbd4084802dfd52f313d Mon Sep 17 00:00:00 2001 From: Dotta Date: Fri, 30 Oct 2026 19:35:20 -0500 Subject: [PATCH 62/70] fix(server): bound blocker facts and preserve origin authority Resolve liveness/stranded references through trusted company issue rows, including valid none sentinels. Bound shared blocker nodes, edges and query results; return private deferred-root completeness so retention immediately performs finite exact source checks. Preserve already-observed live work, completed-root classification and true deadlines. Cover historical producer metadata, moved recovery tasks, many-heavy graph limits, quiet rechecks and faithful transaction fixtures. Co-Authored-By: Paperclip --- .../2026-10-30-attention-retention-sleep.md | 2 +- .../attention-retention-work.test.ts | 126 ++++++++++++++++-- server/src/__tests__/issues-service.test.ts | 2 +- server/src/__tests__/work-products.test.ts | 28 +++- server/src/services/attention-retention.ts | 11 +- .../issue-thread-interactions.test.ts | 19 ++- server/src/services/issues.ts | 120 +++++++++-------- .../services/retention-work-notifications.ts | 19 ++- 8 files changed, 250 insertions(+), 77 deletions(-) diff --git a/doc/plans/2026-10-30-attention-retention-sleep.md b/doc/plans/2026-10-30-attention-retention-sleep.md index 24212b4b98a..aa78f1bb746 100644 --- a/doc/plans/2026-10-30-attention-retention-sleep.md +++ b/doc/plans/2026-10-30-attention-retention-sleep.md @@ -25,7 +25,7 @@ The feed and worker share confirmation collapse, newer-run probes, issue visibil ## Existing mutation boundaries reactivate work -Committed hints identify a company, source kind, source ID or changed issue where those values are known. Routine issue writers pass identity to the retention domain helper before their mutation. The helper captures old and new parent paths, reverse blocker ancestors, run-linked issues and liveness-origin references; issue-scoped source queries then select reviews, blockers, interactions, decisions, recovery actions and failed runs by their issue identity. Budget incidents and agent errors remain tied to their own source changes. Ordinary issue writes do not reconstruct the unrelated company feed. Queue keys explicitly pick supported identity fields. Retention's own writes use exact source hints; blocker retention changes invalidate the company/source because multiple blocked roots can project one terminal subject. Reconstruction pages and due batches remain bounded. Review attention and interaction invokability are computed in company batches per candidate page; a full 100-candidate review page uses at most 20 statements and 100 confirmations use at most six. Decisions, recovery source links, runs, eligible agents, issue visibility, newer-run probes and blocker graphs also load in company batches per page. A 100-source decision or recovery page uses at most three statements; 100 failed runs use at most six; 100 blocker roots use at most 30 graph/source statements plus retention-state reads for their emitted entries. These evidence maps exist only for one load. The blocker owner shares fetched node and edge facts while tracking depth, node count, expanded nodes and live-work reachability independently for each root. An unrelated graph reaching its limit cannot change a shallow root’s terminal subject or deadline. All later graph queries, including recovery source IDs, agent IDs and workspace-finalize inputs, use bounded chunks. A SQL-generated 70,000-node graph proves PostgreSQL parameter limits are respected at the driver boundary. Archive checks still lock and freshly re-read sources, terminal subjects and related eligibility. Ordinary execution, decision and budget mutations pass their affected company; exact decision sources are included. +Committed hints identify a company, source kind, source ID or changed issue where those values are known. Routine issue writers pass identity to the retention domain helper before their mutation. The helper captures old and new parent paths, reverse blocker ancestors, run-linked issues and liveness-origin references; issue-scoped source queries then select reviews, blockers, interactions, decisions, recovery actions and failed runs by their issue identity. Budget incidents and agent errors remain tied to their own source changes. Ordinary issue writes do not reconstruct the unrelated company feed. Opaque liveness and stranded-recovery origin references resolve through company-scoped issue rows before entering UUID queries, so valid none sentinels do not roll back writes and moved recovery tasks still invalidate their original review. Queue keys explicitly pick supported identity fields. Retention's own writes use exact source hints; blocker retention changes invalidate the company/source because multiple blocked roots can project one terminal subject. Reconstruction pages and due batches remain bounded. Review attention and interaction invokability are computed in company batches per candidate page; a full 100-candidate review page uses at most 20 statements and 100 confirmations use at most six. Decisions, recovery source links, runs, eligible agents, issue visibility, newer-run probes and blocker graphs also load in company batches per page. A 100-source decision or recovery page uses at most three statements; 100 failed runs use at most six; 100 blocker roots use at most 30 graph/source statements plus retention-state reads for their emitted entries. These evidence maps exist only for one load. The blocker owner shares fetched node and edge facts while tracking depth, node count, expanded nodes and live-work reachability independently for each root. The shared graph fetch stores at most 2,000 nodes and 4,000 edges, and bounds each frontier query before materializing rows. An internal batch result identifies roots whose evaluation is incomplete; the legacy map API exposes no extra transport fields. Retention schedules only those roots for an immediate fresh single-root check, which disarms or saves the actual source deadline. It never archives a partial intermediary or adds a recurring incomplete-root poll. Fully evaluated roots preserve their exact classification. All later graph queries, including recovery source IDs, agent IDs and workspace-finalize inputs, use bounded chunks. A SQL-generated 70,000-node graph proves PostgreSQL parameter limits are respected at the driver boundary. Archive checks still lock and freshly re-read sources, terminal subjects and related eligibility. Ordinary execution, decision and budget mutations pass their affected company; exact decision sources are included. Producer coverage includes decision creation and changes; interaction creation, resolution, expiry and supersession; issue creation, updates, blocker relations, checkout, adoption and release; agent hire, configuration, lifecycle, error clearing and deletion; budget incidents and company pause/resume; exhausted-run events and retry-run creation; native status, review, finalization and reconstruction; external conversation transitions; and imported blocker relations. Existing service transactions retain outer commit ownership. Savepoint release does not publish work before outer commit. diff --git a/server/src/__tests__/attention-retention-work.test.ts b/server/src/__tests__/attention-retention-work.test.ts index b1e7949a8b7..275b4f64570 100644 --- a/server/src/__tests__/attention-retention-work.test.ts +++ b/server/src/__tests__/attention-retention-work.test.ts @@ -17,7 +17,7 @@ import { buildIssueGraphLivenessIncidentKey, RECOVERY_ORIGIN_KINDS } from "../se import { issueThreadInteractionService } from "../services/issue-thread-interactions.js"; import { createAgentLifecycle } from "../services/agent-lifecycle.js"; import { BLOCKER_ATTENTION_MAX_DEPTH, BLOCKER_ATTENTION_MAX_NODES, issueService } from "../services/issues.js"; -import { notifyAttentionRetentionWork } from "../services/retention-work-notifications.js"; +import { notifyAttentionRetentionWork, notifyIssueAttentionRetentionWork } from "../services/retention-work-notifications.js"; import { idleWorkSnapshot } from "../services/task-admission.js"; import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; @@ -278,12 +278,16 @@ const delay = DEFAULT_DECISION_ARCHIVE_DAYS * 86_400_000; expect(expected.terminalBlockerIssueId).toBe(terminal.id); expect(limited.terminalBlockerIssueId).toBe(limited.directBlockerIssueId); for (const input of [[root, heavy], [heavy, root]]) { - const batch = await owner.listBlockerAttention(companyId, input); - expect(batch.get(root.id)).toEqual(expected); expect(batch.get(heavy.id)).toEqual(limited); + const batch = await owner.listBlockerAttentionBatch(companyId, input); + if (!batch.deferredRootIds.has(root.id)) expect(batch.attention.get(root.id)).toEqual(expected); + if (!batch.deferredRootIds.has(heavy.id)) expect(batch.attention.get(heavy.id)).toEqual(limited); + expect(batch.nodeCount).toBeLessThanOrEqual(BLOCKER_ATTENTION_MAX_NODES); + if (limit === "depth") expect(batch.deferredRootIds.has(root.id)).toBe(false); } const page = await attentionRetentionService(db).load({ companyId, sourceKind: "blocker_attention" }); const entry = page.rows!.find(item => item.rootId === root.id && item.variant === "terminal")!; - expect(entry.sourceId).toBe(terminal.id); expect(entry.dueAt).toBeLessThan(Date.now()); + expect(entry.dueAt).toBeLessThanOrEqual(Date.now()); + if (limit === "depth") expect(entry.sourceId).toBe(terminal.id); await attentionRetentionService(db).check(entry); expect((await state("blocker_attention", terminal.id))?.archivedAt).toBeInstanceOf(Date); expect(await db.select().from(decisionArchiveNotificationOutbox).where(eq(decisionArchiveNotificationOutbox.sourceId, terminal.id))).toHaveLength(1); @@ -317,20 +321,79 @@ const delay = DEFAULT_DECISION_ARCHIVE_DAYS * 86_400_000; await db.execute(sql`insert into issues (id, company_id, title, status, updated_at) select gen_random_uuid(), ${companyId}::uuid, 'parameter-root-' || n, 'blocked', ${old().toISOString()}::timestamptz from generate_series(1, 70) n`); - await db.execute(sql`insert into issues (id, company_id, parent_id, title, status, assignee_agent_id, updated_at) - select gen_random_uuid(), ${companyId}::uuid, root.id, 'parameter-leaf-' || n, 'backlog', ${agentId}::uuid, ${old().toISOString()}::timestamptz - from issues root cross join generate_series(1, 999) n + await db.execute(sql`insert into issues (id, company_id, parent_id, title, status, updated_at) + select gen_random_uuid(), ${companyId}::uuid, root.id, 'parameter-branch-' || n, 'todo', ${old().toISOString()}::timestamptz + from issues root cross join generate_series(1, 10) n where root.company_id = ${companyId}::uuid and root.status = 'blocked'`); + await db.execute(sql`insert into issues (id, company_id, parent_id, title, status, assignee_agent_id, updated_at) + select gen_random_uuid(), ${companyId}::uuid, branch.id, 'parameter-leaf-' || n, 'backlog', ${agentId}::uuid, ${old().toISOString()}::timestamptz + from issues branch cross join generate_series(1, 99) n + where branch.company_id = ${companyId}::uuid and branch.title like 'parameter-branch-%'`); + const completed = await insertIssue("blocked"); + const shallow = await insertIssue("blocked", { updatedAt: new Date() }); + const middle = await insertIssue("todo", { updatedAt: new Date() }); + const overdue = await insertIssue("backlog", { assigneeAgentId: agentId }); + await db.insert(issueRelations).values([ + { companyId, issueId: middle.id, relatedIssueId: shallow.id, type: "blocks" }, + { companyId, issueId: overdue.id, relatedIssueId: middle.id, type: "blocks" }, + ]); const roots = await db.select().from(issues).where(and(eq(issues.companyId, companyId), eq(issues.status, "blocked"))); parameterCounts.length = 0; - const result = await issueService(db).listBlockerAttention(companyId, roots).catch(error => { + const result = await issueService(db).listBlockerAttentionBatch(companyId, roots).catch(error => { throw new Error(`Large graph failed: ${String(error.cause?.message ?? error.message).slice(0, 300)}`); }); - expect(result.size).toBe(70); - expect([...result.values()].every(item => item.state === "needs_attention" && item.terminalBlockerIssueId)).toBe(true); + expect(result.attention.size).toBe(72); + expect(result.deferredRootIds.has(completed.id)).toBe(false); + expect(result.attention.get(completed.id)?.terminalBlockerIssueId).toBe(completed.id); + expect([...result.attention.values()].every(item => item.state === "needs_attention" && item.terminalBlockerIssueId)).toBe(true); expect(Math.max(...parameterCounts)).toBeLessThanOrEqual(1100); + expect(parameterCounts.length).toBeLessThanOrEqual(100); + expect(result.nodeCount).toBeLessThanOrEqual(BLOCKER_ATTENTION_MAX_NODES); + expect(result.edgeCount).toBeLessThanOrEqual(BLOCKER_ATTENTION_MAX_NODES * 2); + expect(result.fetchedRows).toBeLessThanOrEqual(BLOCKER_ATTENTION_MAX_NODES * 6); + expect(result.deferredRootIds.has(shallow.id)).toBe(true); + const service = attentionRetentionService(db); + const page = await service.load({ companyId, sourceKind: "blocker_attention" }); + const immediate = page.rows!.find(entry => entry.rootId === shallow.id && entry.variant === "terminal")!; + expect(immediate.dueAt).toBeLessThanOrEqual(Date.now()); + await service.check(immediate); + expect((await state("blocker_attention", overdue.id))?.archivedAt).toBeInstanceOf(Date); }, 120_000); + it("preserves observed live work when a single large root exhausts shared fetch capacity", async () => { + const root = await insertIssue("blocked"); + const live = await insertIssue("in_progress", { assigneeAgentId: agentId }); + await db.insert(issueRelations).values({ companyId, issueId: live.id, relatedIssueId: root.id, type: "blocks" }); + await db.execute(sql`insert into issues (id, company_id, parent_id, title, status, assignee_agent_id, updated_at) + select gen_random_uuid(), ${companyId}::uuid, ${root.id}::uuid, 'bounded-leaf-' || n, 'backlog', ${agentId}::uuid, ${old().toISOString()}::timestamptz + from generate_series(1, 3000) n`); + const batch = await issueService(db).listBlockerAttentionBatch(companyId, [root]); + expect(batch.deferredRootIds.has(root.id)).toBe(true); + expect(batch.attention.get(root.id)?.blockingTreeLive).toBe(true); + const service = attentionRetentionService(db), page = await service.load({ companyId, sourceKind: "blocker_attention" }); + const entry = page.rows!.find(item => item.rootId === root.id && item.variant === "terminal")!; + expect(await service.check(entry)).toBeNull(); + expect(await state("blocker_attention", root.id)).toBeNull(); + expect(await db.select().from(decisionArchiveNotificationOutbox)).toHaveLength(0); + }); + + it("disarms finite deferred-root checks when large retained graphs have no archive obligation", async () => { + await db.execute(sql`insert into issues (id, company_id, title, status, updated_at) + select gen_random_uuid(), ${companyId}::uuid, 'held-root-' || n, 'blocked', ${old().toISOString()}::timestamptz + from generate_series(1, 4) n`); + await db.execute(sql`insert into issues (id, company_id, parent_id, title, status, assignee_agent_id, updated_at) + select gen_random_uuid(), ${companyId}::uuid, root.id, 'held-leaf-' || n, 'backlog', ${agentId}::uuid, ${old().toISOString()}::timestamptz + from issues root cross join generate_series(1, 600) n + where root.company_id = ${companyId}::uuid and root.status = 'blocked'`); + await db.execute(sql`insert into decision_retention (company_id, source_kind, source_id, source_activity_at, keep) + select company_id, 'blocker_attention', id::text, updated_at, true from issues + where company_id = ${companyId}::uuid and status = 'backlog'`); + const service = attentionRetentionService(db), check = vi.spyOn(service, "check"); + await start(service); await quiet(); + expect(check).toHaveBeenCalledTimes(4); + expect(await db.select().from(decisionArchiveNotificationOutbox)).toHaveLength(0); + }); + it("canonicalizes wide circular notification variables to the supported identity fields", async () => { const sourceId = randomUUID(), issueId = randomUUID(), hints: string[] = []; const wide = { companyId, sourceKind: "review" as const, sourceId, issueId, ignored: {} as unknown }; @@ -418,6 +481,49 @@ const delay = DEFAULT_DECISION_ARCHIVE_DAYS * 86_400_000; await archived("blocker_attention", terminal.id); await quiet(); }); + it("resolves opaque run issue references through trusted company issue rows", async () => { + const issue = await insertIssue(); + const [valid, opaque] = await db.insert(heartbeatRuns).values([ + { companyId, agentId, status: "succeeded", contextSnapshot: { issueId: issue.id } }, + { companyId, agentId, status: "succeeded", contextSnapshot: { issueId: "none" } }, + ]).returning(); + const hints: string[] = []; + const unsubscribe = subscribeDatabaseWork(db, DELIVERY_QUEUES.attentionRetention, (event, values) => { + if (event === "settled") hints.push(...values ?? []); + }); + try { + await db.transaction(tx => notifyIssueAttentionRetentionWork(tx, { companyId, runId: valid.id })); + await db.transaction(tx => notifyIssueAttentionRetentionWork(tx, { companyId, runId: opaque.id })); + } finally { unsubscribe(); } + expect(hints).toEqual([JSON.stringify({ companyId, issueId: issue.id }), JSON.stringify({ companyId })]); + }); + + it("accepts ordinary liveness incident creation and completion with a none leaf sentinel", async () => { + const source = await insertIssue("blocked", { responsibleUserId: "board" }); + const service = issueService(db); + const incident = await service.create(companyId, { title: "Invalid participant", status: "todo", assigneeUserId: "board", + createdByUserId: "board", originKind: RECOVERY_ORIGIN_KINDS.issueGraphLivenessEscalation, + originId: `harness_liveness:${companyId}:${source.id}:invalid_review_participant:none` }); + await service.update(source.id, { blockedByIssueIds: [incident.id] }); + await service.update(incident.id, { status: "done" }); + expect((await service.getById(incident.id))?.status).toBe("done"); + expect((await service.getRelationSummaries(source.id)).blockedBy).toEqual([]); + }); + + it.each(["hidden", "done"] as const)("reactivates an older review when moved stranded recovery is %s", async transition => { + const review = await insertIssue("in_review", { assigneeAgentId: agentId, responsibleUserId: "board" }); + const parent = await insertIssue("todo", { responsibleUserId: "board" }); + const recovery = await insertIssue("todo", { parentId: review.id, originKind: RECOVERY_ORIGIN_KINDS.strandedIssueRecovery, + originId: review.id, assigneeUserId: "board", responsibleUserId: "board" }); + await issueService(db).update(recovery.id, { parentId: parent.id }); + expect((await issueService(db).listReviewAttention(companyId, [review])).get(review.id)?.state).not.toBe("stalled"); + await start(); await quiet(); + await issueService(db).update(recovery.id, transition === "hidden" ? { hiddenAt: new Date() } : { status: "done" }); + await settle(); + expect((await state("review", review.id))?.archivedAt).toBeInstanceOf(Date); + expect(await db.select().from(decisionArchiveNotificationOutbox).where(eq(decisionArchiveNotificationOutbox.sourceId, review.id))).toHaveLength(1); + }); + it("scopes ordinary execution-control writes to their company", async () => { const otherCompany = randomUUID(); await db.insert(companies).values({ id: otherCompany, name: "Other", issuePrefix: "OTHER" }); diff --git a/server/src/__tests__/issues-service.test.ts b/server/src/__tests__/issues-service.test.ts index fb13ff43e82..0d28858b95c 100644 --- a/server/src/__tests__/issues-service.test.ts +++ b/server/src/__tests__/issues-service.test.ts @@ -4151,7 +4151,7 @@ const dependencyTestDatabaseUrl = resolveExternalTestDatabaseUrl("PAPERCLIP_ISSU beforeAll(async () => { if (!dependencyTestDatabaseUrl) tempDb = await startEmbeddedPostgresTestDatabase("paperclip-issues-blockers-"); - db = createDb(dependencyTestDatabaseUrl ?? tempDb!.connectionString); + db = createDb(dependencyTestDatabaseUrl ?? tempDb!.connectionString, { maxConnections: 6 }); svc = issueService(db); await ensureIssueRelationsTable(db); }, 20_000); diff --git a/server/src/__tests__/work-products.test.ts b/server/src/__tests__/work-products.test.ts index 476e1156184..cef395409fd 100644 --- a/server/src/__tests__/work-products.test.ts +++ b/server/src/__tests__/work-products.test.ts @@ -1,3 +1,5 @@ +import { PgDialect } from "drizzle-orm/pg-core"; +import type { SQL } from "drizzle-orm"; import { describe, expect, it, vi } from "vitest"; import { enrichWorkProductMetadataWithDiff, @@ -33,6 +35,18 @@ function createWorkProductRow(overrides: Partial> = {}) }; } +function retentionReads(insideTransaction: () => boolean) { + return vi.fn(async (query: SQL) => { + const statement = new PgDialect().sqlToQuery(query); + expect(insideTransaction()).toBe(true); + expect(statement.params).toContain("company-1"); + expect(statement.params).toContain("issue-1"); + if (statement.sql.includes("origin_kind")) return [{ id: "issue-1", originKind: null, originId: null }]; + if (statement.sql.includes("with recursive affected")) return [{ issueId: "issue-1" }]; + throw new Error(`Unexpected work-product fixture query: ${statement.sql}`); + }); +} + describe("workProductService", () => { it("extracts runner totals and enriches work-product metadata", () => { const summary = workProductDiffSummaryFromEventPayload({ @@ -158,11 +172,16 @@ describe("workProductService", () => { const insertValues = vi.fn(() => ({ returning: insertReturning })); const txInsert = vi.fn(() => ({ values: insertValues })); + let transactionOpen = false; const tx = { + execute: retentionReads(() => transactionOpen), update: txUpdate, insert: txInsert, }; - const transaction = vi.fn(async (callback: (input: typeof tx) => Promise) => await callback(tx)); + const transaction = vi.fn(async (callback: (input: typeof tx) => Promise) => { + transactionOpen = true; + try { return await callback(tx); } finally { transactionOpen = false; } + }); const svc = workProductService({ transaction } as any); const result = await svc.createForIssue("issue-1", "company-1", { @@ -194,11 +213,16 @@ describe("workProductService", () => { const updateSet = vi.fn(() => ({ where: updateWhere })); const txUpdate = vi.fn(() => ({ set: updateSet })); + let transactionOpen = false; const tx = { + execute: retentionReads(() => transactionOpen), select: txSelect, update: txUpdate, }; - const transaction = vi.fn(async (callback: (input: typeof tx) => Promise) => await callback(tx)); + const transaction = vi.fn(async (callback: (input: typeof tx) => Promise) => { + transactionOpen = true; + try { return await callback(tx); } finally { transactionOpen = false; } + }); const svc = workProductService({ transaction } as any); const result = await svc.update("work-product-1", { diff --git a/server/src/services/attention-retention.ts b/server/src/services/attention-retention.ts index 27aedba8c47..ebe91416f52 100644 --- a/server/src/services/attention-retention.ts +++ b/server/src/services/attention-retention.ts @@ -32,6 +32,7 @@ type PageEvidence = { agents: Set; visibleIssues: Map; newerRuns: Set; + deferredRoots: Set; blockers: Awaited["listBlockerAttention"]>>; }; @@ -184,7 +185,9 @@ export function attentionRetentionService(db: Db) { const visibleRows = await readIssues(issueIds); const visible = new Map(visibleRows.map(row => [row.id, row])); const blockerRoots = visibleRows.filter(row => blockerIds.includes(row.id)); - const blockers: PageEvidence["blockers"] = blockerRoots.length ? await issueService(database).listBlockerAttention(companyId, blockerRoots) : new Map(); + const blockerBatch = blockerRoots.length ? await issueService(database).listBlockerAttentionBatch(companyId, blockerRoots) + : { attention: new Map(), deferredRootIds: new Set() }; + const blockers: PageEvidence["blockers"] = blockerBatch.attention; const terminalIds = [...blockers.values()].filter(terminalBlockerRequiresAttention) .map(attention => attention.terminalBlockerIssueId).filter((id): id is string => id != null && !visible.has(id)); for (const row of await readIssues(terminalIds)) visible.set(row.id, row); @@ -192,13 +195,17 @@ export function attentionRetentionService(db: Db) { decisions: new Map(decisionRows.map(row => [row.id, row])), recoveries: new Map(recoveryRows.map(row => [row.id, row])), runs: new Map(runRows.map(row => [row.id, row])), agents: new Set(runAgents.map(row => row.id)), visibleIssues: visible, newerRuns: await listFailedRunsWithNewerRun(database, companyId, - runRows.map(run => ({ ...run, issueId: readRunIssueId(run.contextSnapshot) }))), blockers, + runRows.map(run => ({ ...run, issueId: readRunIssueId(run.contextSnapshot) }))), blockers, deferredRoots: blockerBatch.deferredRootIds, }); } return result; } async function eligible(database: Db, candidate: Candidate, variant?: "owner" | "terminal", lock = false, reviewAttention?: Map, interactionAttention?: Map, evidence?: PageEvidence): Promise { + if (candidate.sourceKind === "blocker_attention" && variant === "terminal" && evidence?.deferredRoots.has(candidate.sourceId)) { + return { ...candidate, rootId: candidate.sourceId, variant, + id: `${candidate.sourceKind}:${candidate.sourceId}:terminal`, dueAt: Date.now() }; + } let sourceId = candidate.sourceId; let activityAt = candidate.activityAt; let dueAt = candidate.deadlineAt; diff --git a/server/src/services/issue-thread-interactions.test.ts b/server/src/services/issue-thread-interactions.test.ts index 24e1489006f..5ad905a8a10 100644 --- a/server/src/services/issue-thread-interactions.test.ts +++ b/server/src/services/issue-thread-interactions.test.ts @@ -1,5 +1,6 @@ import { beforeEach, describe, expect, it, vi } from "vitest"; -import { getTableName } from "drizzle-orm"; +import { PgDialect } from "drizzle-orm/pg-core"; +import { getTableName, type SQL } from "drizzle-orm"; const mockCreateChild = vi.fn(); @@ -48,8 +49,18 @@ function createFakeDb(args: { const toolActionRequestUpdates: Array> = []; const inserts: Array<{ table: string; values: Record }> = []; let selectCallCount = 0; + let transactionDepth = 0; const db: any = { + execute: vi.fn(async (query: SQL) => { + const statement = new PgDialect().sqlToQuery(query); + expect(transactionDepth).toBeGreaterThan(0); + expect(statement.params).toContain(interactionRow.companyId); + expect(statement.params).toContain(interactionRow.issueId); + if (statement.sql.includes("origin_kind")) return [{ id: interactionRow.issueId, originKind: null, originId: null }]; + if (statement.sql.includes("with recursive affected")) return [{ issueId: interactionRow.issueId }]; + throw new Error(`Unexpected interaction fixture query: ${statement.sql}`); + }), select: vi.fn(() => { selectCallCount += 1; const rows = selectCallCount === 1 ? [interactionRow] : (args.parentRows ?? []); @@ -84,7 +95,10 @@ function createFakeDb(args: { inserts.push({ table: getTableName(table as never), values }); }, })), - transaction: async (callback: (tx: typeof db) => Promise) => callback(db), + transaction: async (callback: (tx: typeof db) => Promise) => { + transactionDepth++; + try { return await callback(db); } finally { transactionDepth--; } + }, }; return { @@ -177,6 +191,7 @@ describe("issueThreadInteractionService", () => { }; let selectCallCount = 0; + let transactionDepth = 0; const db: any = { select: vi.fn(() => { selectCallCount += 1; diff --git a/server/src/services/issues.ts b/server/src/services/issues.ts index c91a325ad39..4d5fd5f85f6 100644 --- a/server/src/services/issues.ts +++ b/server/src/services/issues.ts @@ -3436,6 +3436,7 @@ type IssueBlockerAttentionEdge = { blockerIssueId: string; }; type IssueBlockerAttentionQueryRow = IssueBlockerAttentionNode & { + executionWorkspaceId?: string | null; issueId: string | null; blockerIssueId: string; }; @@ -3824,11 +3825,15 @@ async function terminalExplicitBlockersByRoot( return terminalByRoot; } -async function listIssueBlockerAttentionMap( +async function listIssueBlockerAttentionMap(dbOrTx: any, companyId: string, issueRows: IssueBlockerAttentionInputNode[]) { + return (await readIssueBlockerAttentionBatch(dbOrTx, companyId, issueRows)).attention; +} + +async function readIssueBlockerAttentionBatch( dbOrTx: any, companyId: string, issueRows: IssueBlockerAttentionInputNode[], -): Promise> { +) { const roots = issueRows.filter( (row) => row.companyId === companyId && row.status === "blocked", ); @@ -3838,37 +3843,38 @@ async function listIssueBlockerAttentionMap( attentionMap.set(row.id, createIssueBlockerAttention()); } } - if (roots.length === 0) return attentionMap; + const deferredRootIds = new Set(roots.slice(BLOCKER_ATTENTION_MAX_NODES).map(root => root.id)); + let edgeCount = 0, fetchedRows = 0; + const result = () => ({ attention: attentionMap, deferredRootIds, nodeCount: nodesById.size, edgeCount, fetchedRows }); + if (roots.length === 0) return { attention: attentionMap, deferredRootIds, nodeCount: 0, edgeCount, fetchedRows }; const nodesById = new Map(); const edgesByIssueId = new Map(); - for (const root of roots) nodesById.set(root.id, { ...root }); + const admittedRoots = roots.slice(0, BLOCKER_ATTENTION_MAX_NODES); + for (const root of admittedRoots) nodesById.set(root.id, { ...root }); - type Traversal = { nodes: Set; expanded: Set; frontier: string[]; truncated: boolean }; - const traversals = new Map(roots.map(root => [root.id, { - nodes: new Set([root.id]), expanded: new Set(), frontier: [root.id], truncated: false, + type Traversal = { rootId: string; nodes: Set; expanded: Set; frontier: string[]; truncated: boolean }; + const traversals = new Map(admittedRoots.map(root => [root.id, { + rootId: root.id, nodes: new Set([root.id]), expanded: new Set(), frontier: [root.id], truncated: false, }])); const expanded = new Set(); + const queried = new Set(); const pendingFinalizeBlockerIssueIds = new Set(); for (let depth = 0; depth < BLOCKER_ATTENTION_MAX_DEPTH; depth += 1) { const active = [...traversals.values()].filter(traversal => !traversal.truncated && traversal.frontier.length > 0); if (active.length === 0) break; - const frontier = [...new Set(active.flatMap(traversal => traversal.frontier))].filter(id => !expanded.has(id)); + const unread = new Set(); + for (const traversal of active) for (const id of traversal.frontier) if (!expanded.has(id)) unread.add(id); + const frontier = [...unread]; for (const chunk of chunkList( frontier, ISSUE_LIST_RELATED_QUERY_CHUNK_SIZE, )) { - const readinessByIssueId = await listIssueDependencyReadinessMap( - dbOrTx, - companyId, - chunk, - ); - for (const readiness of readinessByIssueId.values()) { - for (const blockerIssueId of readiness.pendingFinalizeBlockerIssueIds) { - pendingFinalizeBlockerIssueIds.add(blockerIssueId); - } - } + const remaining = Math.min(BLOCKER_ATTENTION_MAX_NODES - nodesById.size, + BLOCKER_ATTENTION_MAX_NODES * 2 - edgeCount); + if (remaining <= 0) break; + const rowLimit = remaining + 1; const explicitBlockerRowsPromise: Promise< IssueBlockerAttentionQueryRow[] > = dbOrTx @@ -3882,6 +3888,7 @@ async function listIssueBlockerAttentionMap( title: issues.title, status: issues.status, executionRunId: issues.executionRunId, + executionWorkspaceId: issues.executionWorkspaceId, assigneeAgentId: issues.assigneeAgentId, assigneeUserId: issues.assigneeUserId, }) @@ -3894,7 +3901,7 @@ async function listIssueBlockerAttentionMap( inArray(issueRelations.relatedIssueId, chunk), eq(issues.companyId, companyId), ), - ); + ).limit(rowLimit); const childRowsPromise: Promise = dbOrTx .select({ issueId: issues.parentId, @@ -3919,56 +3926,56 @@ async function listIssueBlockerAttentionMap( BLOCKER_ATTENTION_CHILD_TERMINAL_STATUSES, ), ), - ); + ).limit(rowLimit); const [explicitBlockerRows, childRows] = await Promise.all([ explicitBlockerRowsPromise, childRowsPromise, ]); + for (const id of chunk) queried.add(id); + fetchedRows += explicitBlockerRows.length + childRows.length; + const pendingFinalize = await listPendingFinalizeBlockerIssueIds(dbOrTx, companyId, + explicitBlockerRows.flatMap(row => row.status === "done" && row.executionWorkspaceId + ? [{ blockerIssueId: row.blockerIssueId, executionWorkspaceId: row.executionWorkspaceId }] : [])); + for (const id of pendingFinalize) pendingFinalizeBlockerIssueIds.add(id); + let complete = explicitBlockerRows.length < rowLimit && childRows.length < rowLimit; const unresolvedExplicitBlockerRows = explicitBlockerRows.filter( (row) => row.status !== "done" || pendingFinalizeBlockerIssueIds.has(row.blockerIssueId), ); - appendBlockerAttentionEdges(edgesByIssueId, [ - ...unresolvedExplicitBlockerRows - .filter( - (row): row is IssueBlockerAttentionQueryRow & { issueId: string } => - row.issueId !== null, - ) - .map((row) => ({ - issueId: row.issueId, - blockerIssueId: row.blockerIssueId, - })), - ...childRows - .filter( - (row): row is IssueBlockerAttentionQueryRow & { issueId: string } => - row.issueId !== null, - ) - .map((row) => ({ - issueId: row.issueId, - blockerIssueId: row.blockerIssueId, - })), - ]); - for (const row of [...unresolvedExplicitBlockerRows, ...childRows]) { - if (!row.issueId || nodesById.has(row.blockerIssueId)) continue; - nodesById.set(row.blockerIssueId, { - id: row.blockerIssueId, - companyId: row.companyId, - parentId: row.parentId, - identifier: row.identifier, - title: row.title, - status: row.status, - executionRunId: row.executionRunId, - assigneeAgentId: row.assigneeAgentId, - assigneeUserId: row.assigneeUserId, - }); + if (!row.issueId) continue; + if (!nodesById.has(row.blockerIssueId)) { + if (nodesById.size >= BLOCKER_ATTENTION_MAX_NODES) { complete = false; continue; } + nodesById.set(row.blockerIssueId, { + id: row.blockerIssueId, companyId: row.companyId, parentId: row.parentId, identifier: row.identifier, + title: row.title, status: row.status, executionRunId: row.executionRunId, + assigneeAgentId: row.assigneeAgentId, assigneeUserId: row.assigneeUserId, + }); + } + const edges = edgesByIssueId.get(row.issueId) ?? []; + if (edges.some(edge => edge.blockerIssueId === row.blockerIssueId)) continue; + if (edgeCount >= BLOCKER_ATTENTION_MAX_NODES * 2) { complete = false; continue; } + edges.push({ issueId: row.issueId, blockerIssueId: row.blockerIssueId }); + edgesByIssueId.set(row.issueId, edges); + edgeCount++; } - for (const id of chunk) expanded.add(id); + if (complete) for (const id of chunk) expanded.add(id); + } for (const traversal of active) { + if (traversal.frontier.some(id => !expanded.has(id))) { + for (const id of traversal.frontier) if (queried.has(id)) { + traversal.expanded.add(id); + for (const edge of edgesByIssueId.get(id) ?? []) if (nodesById.has(edge.blockerIssueId)) traversal.nodes.add(edge.blockerIssueId); + } + traversal.truncated = true; + traversal.frontier = []; + deferredRootIds.add(traversal.rootId); + continue; + } const nextFrontier = new Set(); for (const id of traversal.frontier) { traversal.expanded.add(id); @@ -4548,7 +4555,7 @@ async function listIssueBlockerAttentionMap( ); } - return attentionMap; + return result(); } type IssueReviewAttentionInput = Pick; @@ -9383,6 +9390,9 @@ export function issueService(db: Db) { return listIssueDependencyReadinessMap(dbOrTx, companyId, issueIds); }, + listBlockerAttentionBatch: async (companyId: string, issueRows: IssueBlockerAttentionInputNode[], dbOrTx: any = db) => + readIssueBlockerAttentionBatch(dbOrTx, companyId, issueRows), + listBlockerAttention: async ( companyId: string, issueRows: IssueBlockerAttentionInputNode[], diff --git a/server/src/services/retention-work-notifications.ts b/server/src/services/retention-work-notifications.ts index cee2486f54c..ba0379afee1 100644 --- a/server/src/services/retention-work-notifications.ts +++ b/server/src/services/retention-work-notifications.ts @@ -20,9 +20,10 @@ export async function notifyIssueAttentionRetentionWork(transaction: object, where company_id = ${scope.companyId}::uuid and id = ${scope.runId}::uuid and issue_id is not null union select native_issue_id::text from heartbeat_runs where company_id = ${scope.companyId}::uuid and id = ${scope.runId}::uuid and native_issue_id is not null - union select coalesce(context_snapshot->>'issueId', context_snapshot->>'taskId') from heartbeat_runs - where company_id = ${scope.companyId}::uuid and id = ${scope.runId}::uuid - and coalesce(context_snapshot->>'issueId', context_snapshot->>'taskId') is not null`); + union select issue.id::text from heartbeat_runs run inner join issues issue + on issue.company_id = run.company_id + and issue.id::text = coalesce(run.context_snapshot->>'issueId', run.context_snapshot->>'taskId') + where run.company_id = ${scope.companyId}::uuid and run.id = ${scope.runId}::uuid`); for (const row of rows) seeds.push(row.id as string); } if (!seeds.length) return notifyAttentionRetentionWork(transaction, { companyId: scope.companyId }); @@ -31,10 +32,20 @@ export async function notifyIssueAttentionRetentionWork(transaction: object, and id in (${sql.join([...new Set(seeds)].map(id => sql`${id}::uuid`), sql`, `)})`); const current = origins.find(source => source.id === scope.issueId); origins.push({ originKind: scope.originKind ?? current?.originKind, originId: scope.originId ?? current?.originId }); + const references: string[] = []; for (const source of origins) { + if (source.originKind === RECOVERY_ORIGIN_KINDS.strandedIssueRecovery && typeof source.originId === "string") { + references.push(source.originId); + } const incident = source.originKind === RECOVERY_ORIGIN_KINDS.issueGraphLivenessEscalation ? parseIssueGraphLivenessIncidentKey(source.originId as string | null) : null; - if (incident?.companyId === scope.companyId) seeds.push(incident.issueId, incident.leafIssueId); + if (incident?.companyId === scope.companyId) references.push(incident.issueId, incident.leafIssueId); + } + if (references.length) { + const trusted = await database.execute(sql`select id::text as id from issues + where company_id = ${scope.companyId}::uuid + and id::text in (${sql.join([...new Set(references)].map(id => sql`${id}`), sql`, `)})`); + for (const row of trusted) seeds.push(row.id as string); } const affected = await database.execute(sql`with recursive affected(id) as ( select unnest(array[${sql.join([...new Set(seeds)].map(id => sql`${id}::uuid`), sql`, `)}]) From 7bb1af657aa2eda4573c8b77b21afaf0fc226d9b Mon Sep 17 00:00:00 2001 From: Dotta Date: Fri, 30 Oct 2026 19:25:05 -0500 Subject: [PATCH 63/70] test: await issue recovery dispatch and cancellation settlement Co-Authored-By: Paperclip --- .../issue-recovery-deadlines.test.ts | 35 ++++++++++++++++--- 1 file changed, 31 insertions(+), 4 deletions(-) diff --git a/server/src/__tests__/issue-recovery-deadlines.test.ts b/server/src/__tests__/issue-recovery-deadlines.test.ts index b3d65f4dbcc..f5416fec571 100644 --- a/server/src/__tests__/issue-recovery-deadlines.test.ts +++ b/server/src/__tests__/issue-recovery-deadlines.test.ts @@ -1,3 +1,4 @@ +import { setImmediate as immediate } from "node:timers/promises"; import { idleWorkSnapshot } from "../services/task-admission.js"; import { randomUUID } from "node:crypto"; import { and, eq, sql } from "drizzle-orm"; @@ -27,6 +28,7 @@ describe.skipIf(!support.supported)("issue recovery production producers and dea let draining = false; const tasks = new Map number | null; hasPending: () => Promise }>(); const errors: unknown[] = []; + const passes = new Set>(); const extraStops: Array<() => Promise> = []; const extraDeadlines: Array<() => number | null> = []; beforeAll(async () => { @@ -52,7 +54,14 @@ describe.skipIf(!support.supported)("issue recovery production producers and dea onError: error => errors.push(error) }); tasks.clear(); const register = coordinator.register.bind(coordinator); - coordinator.register = (queue, task) => { tasks.set(queue, task); return register(queue, task); }; + coordinator.register = (queue, task) => { + tasks.set(queue, task); + return register(queue, { ...task, run: signal => { + const work = task.run(signal); passes.add(work); + void work.then(() => passes.delete(work), () => passes.delete(work)); + return work; + } }); + }; worker = heartbeat.registerIssueStateRecoveryWork(coordinator, () => !draining); dispatch = heartbeat.registerRunDispatchWork(coordinator, () => !draining); cancellation = heartbeat.registerScheduledRetryCancellationWork(coordinator, () => !draining); @@ -89,6 +98,7 @@ describe.skipIf(!support.supported)("issue recovery production producers and dea } async function quiet() { await settled(); + expect(await db.select().from(heartbeatRuns).where(sql`${heartbeatRuns.status} in ('queued', 'running')`)).toHaveLength(0); const reads = vi.spyOn(db, "select"); const executes = vi.spyOn(db, "execute"); const selected = reads.mock.calls.length, executed = executes.mock.calls.length; if (!vi.isFakeTimers()) vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"] }); @@ -97,15 +107,32 @@ describe.skipIf(!support.supported)("issue recovery production producers and dea } async function finish(issueId: string) { await issueService(db).update(issueId, { status: "done" }); - await heartbeat.drainActiveRunExecutions(); + // Closing one blocker can legitimately start its dependent. Wait for this + // task's dispatch and execution; quiet() verifies the whole fixture later. + await vi.waitFor(async () => { + await Promise.allSettled([...passes]); + await heartbeat.drainActiveRunExecutions(); + expect(await db.select().from(heartbeatRuns).where(and( + sql`${heartbeatRuns.contextSnapshot}->>'issueId' = ${issueId}`, + sql`${heartbeatRuns.status} in ('queued', 'running')`, + ))).toHaveLength(0); + }, { timeout: 15_000 }); const saved = await db.select().from(heartbeatRuns).where(and(eq(heartbeatRuns.status, "scheduled_retry"), sql`${heartbeatRuns.contextSnapshot}->>'issueId' = ${issueId}`)); - expect(await db.select().from(heartbeatRuns).where(sql`${heartbeatRuns.status} in ('queued', 'running')`)).toHaveLength(0); if (!vi.isFakeTimers()) vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"] }); if (saved.length > 0) { const due = Math.max(...saved.map(run => run.scheduledRetryAt!.getTime())); - await vi.advanceTimersByTimeAsync(Math.max(0, due - Date.now()) + 1); + // Keep PostgreSQL I/O and coordinator settlement ahead of virtual time. + // A single leap can otherwise occur before the retry deadline is armed. + while (Date.now() <= due) { + await vi.advanceTimersByTimeAsync(Math.min(500, due - Date.now() + 1)); + await Promise.allSettled([...passes]); + await immediate(); + } await vi.waitFor(async () => { + await vi.advanceTimersByTimeAsync(500); + await Promise.allSettled([...passes]); + await heartbeat.drainActiveRunExecutions(); const remaining = await db.select().from(heartbeatRuns).where(and(eq(heartbeatRuns.status, "scheduled_retry"), sql`${heartbeatRuns.contextSnapshot}->>'issueId' = ${issueId}`)); expect(remaining).toHaveLength(0); From e3fff84b6f81448c730dd165253a1a673f8d19b3 Mon Sep 17 00:00:00 2001 From: Dotta Date: Fri, 30 Oct 2026 19:39:21 -0500 Subject: [PATCH 64/70] fix: scope comment-triggered recovery to the affected issue Preserve company reconstruction for graph, run and eligibility changes. A real PostgreSQL producer regression verifies one assignment check among1000 open tasks, with broader eligibility reconstruction still intact. Co-Authored-By: Paperclip --- doc/plans/2026-10-30-issue-recovery-sleep.md | 6 ++- .../issue-recovery-deadlines.test.ts | 47 +++++++++++++++++++ server/src/services/issue-comment-writes.ts | 3 +- .../services/issue-recovery-notifications.ts | 11 +++-- ...ative-status-reassessment-notifications.ts | 2 +- server/src/services/recovery/work.ts | 5 +- .../services/run-dispatch-notifications.ts | 2 +- 7 files changed, 67 insertions(+), 9 deletions(-) diff --git a/doc/plans/2026-10-30-issue-recovery-sleep.md b/doc/plans/2026-10-30-issue-recovery-sleep.md index f191d5dca42..0c6cdf6b4d3 100644 --- a/doc/plans/2026-10-30-issue-recovery-sleep.md +++ b/doc/plans/2026-10-30-issue-recovery-sleep.md @@ -1,6 +1,6 @@ # Issue recovery without periodic scans -The issue recovery owners reconstruct their current obligations once at startup. Committing source writes invalidate only their company. The coordinator keeps scopes, per-issue deadlines, and per-entry failures in memory. Its pending check does not query the database. +The issue recovery owners reconstruct their current obligations once at startup. Ordinary comment writes invalidate only the affected issue for assignment and lock recovery. Lifecycle, graph, run, configuration and eligibility changes invalidate their company. The coordinator keeps scopes, per-issue deadlines, and per-entry failures in memory. Its pending check does not query the database. | Owner | Durable truth and producer boundary | Deadline or event | Exit | | --- | --- | --- | --- | @@ -11,6 +11,10 @@ The issue recovery owners reconstruct their current obligations once at startup. | Task watchdog | Active issue watchdogs and their current source subtree; normal watchdog upsert/disable; issue/comment/document/work-product and wait changes | Exact source creation plus the existing 15-second first-run grace. An incomplete scan retains the watchdog with error backoff. | Live path, open/reviewed reusable review, disabled configuration, or ineligible source. | | Stale issue lock | Existing checkout/execution run references; issue and run lifecycle changes | A committing ownership change, or the fresh owning controller's saved lease expiry. | No stale reference, a valid owner, or the existing guarded terminalization and lock clear. | +Local comment hints do not broaden into an assignment check for every open task. A real PostgreSQL regression starts with1,000 open assigned issues, writes one comment through `issueService.addComment`, and observes exactly one assignment check; a subsequent company eligibility notification still checks all1,000. Restoring the previous comment hint makes that same regression check all1,000 after the comment. The test substitutes the expensive assignment decision only to count its invocations; it uses the real transaction, notification, coordinator and paged SQL. Separate producer acceptance cases exercise real recovery decisions and process execution. + +Parent/descendant disposition repair and shared execution references make status and run changes broader than a comment. Their existing notification boundaries deliberately retain company scope. Orphan blockers, active recovery actions, resolved dependencies and watchdogs also keep their company/subtree duties; an issue comment does not drop those independent obligations. The scope serializer selects known primitive fields and never serializes an entire insert or update object. + Reconstruction uses keyset pages of at most100rows. Repeated company hints coalesce reconstruction for30seconds. Already-known entry deadlines preempt that spacing without an early probe on each comment. Scheduling suppression is read once per pass. A failed read keeps scoped backoff debt. The two subsidiary company repairs do not run inside the per-issue loop. Legacy direct aggregate API calls retain both repairs. The owner methods accept company and issue/watchdog identity. A narrowed lock check still reads other same-company references before using terminal-issue authority. A stale terminal alias cannot end an active issue's shared run. The controller lease owner supplies its fresh expiry; the existing write predicate revalidates the observed boot ID and database-clock expiry. diff --git a/server/src/__tests__/issue-recovery-deadlines.test.ts b/server/src/__tests__/issue-recovery-deadlines.test.ts index f5416fec571..e23a672c0dc 100644 --- a/server/src/__tests__/issue-recovery-deadlines.test.ts +++ b/server/src/__tests__/issue-recovery-deadlines.test.ts @@ -9,6 +9,7 @@ import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } fro import { heartbeatService } from "../services/heartbeat.js"; import { issueService } from "../services/issues.js"; import { issueRecoveryActionService } from "../services/issue-recovery-actions.js"; +import { registerIssueRecoveryWork } from "../services/recovery/work.js"; import { notifyIssueRecovery } from "../services/issue-recovery-notifications.js"; import { taskWatchdogService } from "../services/task-watchdogs.js"; import { createDeliveryWorkCoordinator } from "../services/delivery-work-coordinator.js"; @@ -223,6 +224,52 @@ describe.skipIf(!support.supported)("issue recovery production producers and dea expect((await originalSelect().from(issueRecoveryActions).where(eq(issueRecoveryActions.id, foreignAction.id)))[0].status).toBe("active"); expect(await originalSelect().from(heartbeatRuns)).toHaveLength(0); await quiet(); }); + it("limits an ordinary comment to its issue among one thousand open assignments", async () => { + const f = await fixture(); + const rows = await db.insert(issues).values(Array.from({ length: 1000 }, (_, index) => ({ + companyId: f.companyId, title: `Open work ${index}`, status: "todo", assigneeAgentId: f.agentId, + }))).returning({ id: issues.id }); + const checked: string[] = []; + vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"] }); + coordinator = createDeliveryWorkCoordinator({ owner: db, canRun: () => true, onError: error => errors.push(error) }); + const register = coordinator.register.bind(coordinator); + coordinator.register = (queue, task) => register(queue, { ...task, run: signal => { + const pass = task.run(signal); passes.add(pass); + void pass.then(() => passes.delete(pass), () => passes.delete(pass)); + return pass; + } }); + // Observe the actual paged reconstruction boundary without executing one + // thousand adapters. The producer, transaction, coordinator and SQL are real; + // the other acceptance cases exercise the resulting recovery decisions. + worker = registerIssueRecoveryWork(db, coordinator, { + stranded: async scope => { checked.push(scope.issueId); }, + orphanBlockers: async () => {}, recoveryActions: async () => {}, + dependencies: async () => ({ enqueueFailed: 0 }), locks: async () => {}, + watchdogs: async () => ({ incomplete: 0 }), schedulingSuppressed: async () => false, + }, () => true); + await worker.ready; + async function settleWork() { + for (let step = 0; step < 120; step++) { + await vi.advanceTimersByTimeAsync(1000); + await Promise.allSettled([...passes]); await immediate(); + if (coordinator.nextWakeAt() === null && idleWorkSnapshot().active === 0) return; + } + throw new Error("Recovery did not settle its committed work within two virtual minutes"); + } + await settleWork(); + expect(new Set(checked).size).toBe(1000); + checked.length = 0; + await issueService(db).addComment(rows[0].id, "New evidence on this task", { userId: "recovery-user" }); + await settleWork(); + expect(checked).toEqual([rows[0].id]); + // Company eligibility changes still reconstruct all assignments. Narrowing + // comment hints must not hide unrelated work after an operator resumes it. + checked.length = 0; + await db.transaction(tx => notifyIssueRecovery(tx, { companyId: f.companyId })); + await settleWork(); + expect(new Set(checked).size).toBe(1000); + await quiet(); + }); it("dispatches an ordinary new assignment after commit and returns to quiet after completion", async () => { const f = await fixture(); await start(); await quiet(); const issue = await issueService(db).create(f.companyId, { title: "Recover assignment", status: "todo", assigneeAgentId: f.agentId }); diff --git a/server/src/services/issue-comment-writes.ts b/server/src/services/issue-comment-writes.ts index a968ecb1fec..39d8f6f891c 100644 --- a/server/src/services/issue-comment-writes.ts +++ b/server/src/services/issue-comment-writes.ts @@ -7,7 +7,8 @@ type WriteDb = Db | Parameters[0]>[0]; export function insertIssueComment(db: WriteDb, input: PgInsertValue & { companyId: string }) { return db.transaction(async tx => { - await notifyIssueRecovery(tx, { companyId: input.companyId }); + await notifyIssueRecovery(tx, { companyId: input.companyId, + issueId: typeof input.issueId === "string" ? input.issueId : undefined }); await notifyDeliveryWork(tx, DELIVERY_QUEUES.dotEvents, `company:${input.companyId}`); return tx.insert(issueComments).values(input).returning(); }); diff --git a/server/src/services/issue-recovery-notifications.ts b/server/src/services/issue-recovery-notifications.ts index 3c974228d86..7b709960831 100644 --- a/server/src/services/issue-recovery-notifications.ts +++ b/server/src/services/issue-recovery-notifications.ts @@ -1,11 +1,16 @@ import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js"; -export type IssueRecoveryScope = { companyId: string }; +/** An issue hint is reserved for local facts such as comments. Lifecycle, + * graph, run and eligibility changes retain company scope because their + * consequences can include ancestors and shared execution references. */ +export type IssueRecoveryScope = { companyId: string; issueId?: string }; export async function notifyIssueRecovery(transaction: object, scope?: IssueRecoveryScope) { - const hint = scope ? JSON.stringify({ companyId: scope.companyId }) : "*"; + const companyHint = scope ? JSON.stringify({ companyId: scope.companyId }) : "*"; + const issueHint = scope ? JSON.stringify({ companyId: scope.companyId, issueId: scope.issueId }) : "*"; for (const queue of [DELIVERY_QUEUES.orphanBlockers, DELIVERY_QUEUES.activeRecoveryActions, DELIVERY_QUEUES.strandedAssignments, DELIVERY_QUEUES.resolvedDependencies, DELIVERY_QUEUES.taskWatchdogs, DELIVERY_QUEUES.staleIssueLocks]) { - await notifyDeliveryWork(transaction, queue, hint); + await notifyDeliveryWork(transaction, queue, + queue === DELIVERY_QUEUES.strandedAssignments || queue === DELIVERY_QUEUES.staleIssueLocks ? issueHint : companyHint); } } diff --git a/server/src/services/native-status-reassessment-notifications.ts b/server/src/services/native-status-reassessment-notifications.ts index 4b5eac73b2f..d3b543d9914 100644 --- a/server/src/services/native-status-reassessment-notifications.ts +++ b/server/src/services/native-status-reassessment-notifications.ts @@ -4,7 +4,7 @@ import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notificatio export type NativeStatusReassessmentScope = { companyId: string; issueId?: string; runId?: string; agentId?: string }; export async function notifyNativeStatusReassessment(tx: object, scope: NativeStatusReassessmentScope) { - await notifyIssueRecovery(tx, scope); + await notifyIssueRecovery(tx, scope ? { companyId: scope.companyId } : undefined); return notifyDeliveryWork(tx, DELIVERY_QUEUES.nativeStatusReassessment, JSON.stringify({ companyId: scope.companyId, issueId: scope.issueId, runId: scope.runId, agentId: scope.agentId })); } diff --git a/server/src/services/recovery/work.ts b/server/src/services/recovery/work.ts index 8bf7b75c7f4..407e4f1eb6d 100644 --- a/server/src/services/recovery/work.ts +++ b/server/src/services/recovery/work.ts @@ -32,13 +32,14 @@ export function registerIssueRecoveryWork( reconstructionIntervalMs: 30_000, beforePass: async () => { enabled = !(await owner.schedulingSuppressed()); }, decodeScope: hint => hint === "*" ? undefined : JSON.parse(hint) as IssueRecoveryScope, - matches: (entry, scope) => !scope || entry.companyId === scope.companyId, + matches: (entry, scope) => !scope || (entry.companyId === scope.companyId && (!scope.issueId || entry.id === scope.issueId)), load: async scope => { - const key = scope?.companyId ?? "*"; + const key = scope ? JSON.stringify([scope.companyId, scope.issueId]) : "*"; if (!enabled) { cursors.delete(key); return []; } const cursor = cursors.get(key); const rows = await db.select({ id: issues.id, companyId: issues.companyId }).from(issues).where(and( scope ? eq(issues.companyId, scope.companyId) : undefined, + scope?.issueId ? eq(issues.id, scope.issueId) : undefined, cursor ? gt(issues.id, cursor) : undefined, statuses ? inArray(issues.status, statuses) : or(isNotNull(issues.checkoutRunId), isNotNull(issues.executionRunId)), statuses ? isNull(issues.hiddenAt) : undefined, diff --git a/server/src/services/run-dispatch-notifications.ts b/server/src/services/run-dispatch-notifications.ts index 9e8784b7481..cba58357cf0 100644 --- a/server/src/services/run-dispatch-notifications.ts +++ b/server/src/services/run-dispatch-notifications.ts @@ -4,7 +4,7 @@ import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notificatio export type RunDispatchScope = { companyId: string }; export async function notifyRunDispatch(transaction: object, scope?: RunDispatchScope) { - await notifyIssueRecovery(transaction, scope); + await notifyIssueRecovery(transaction, scope ? { companyId: scope.companyId } : undefined); const hint = scope ? JSON.stringify({ companyId: scope.companyId }) : "*"; await notifyDeliveryWork(transaction, DELIVERY_QUEUES.scheduledRetryDispatch, hint); await notifyDeliveryWork(transaction, DELIVERY_QUEUES.queuedRunDispatch, hint); From 6469daec07fed5afc68d8d441f4a0009dcea9673 Mon Sep 17 00:00:00 2001 From: Dotta Date: Fri, 30 Oct 2026 19:41:19 -0500 Subject: [PATCH 65/70] fix: bound status-card active-hours calculation work Reuse bounded timezone formatters and upcoming openings only within their exact searched closed ranges. Short-circuit empty windows while preserving the existing cold DST search and precise instant eligibility. Verify cost bounds, cache eviction, timezone transitions and twenty-card ordinary source delivery. Co-Authored-By: Paperclip --- doc/plans/2026-10-30-status-cards-sleep.md | 14 ++ .../status-card-evaluation-work.test.ts | 47 +++++++ .../status-card-update-engine.test.ts | 124 +++++++++++++++++- .../src/services/status-card-update-engine.ts | 53 ++++++-- 4 files changed, 225 insertions(+), 13 deletions(-) diff --git a/doc/plans/2026-10-30-status-cards-sleep.md b/doc/plans/2026-10-30-status-cards-sleep.md index 61d451caed5..3f3b8933cbc 100644 --- a/doc/plans/2026-10-30-status-cards-sleep.md +++ b/doc/plans/2026-10-30-status-cards-sleep.md @@ -141,3 +141,17 @@ deadline-reconstruction hint. Every card mutation requires a company ID; only instance-level settings retain wildcard ingress. Deleting an already-missing card emits no hint. The real two-company regression proves deletion does not re-run unchanged reactive searches or wake either summarizer, then stays quiet. + +Active-hours eligibility keeps the existing exact UTC-minute search across DST +changes. A bounded 32-entry timezone formatter cache removes repeated ICU +construction. A bounded 256-entry cache reuses the next opening for the exact +(timezone,start,end) policy only while the candidate instant stays within its +previously searched closed range. The precise instant is checked first, so +historical second-based offsets, the opening itself, and millisecond deadlines +keep their behavior. Clock rollback and policy changes cannot reuse the wrong +range. Empty windows return without formatter or search work. A cold nonempty +policy can still perform at most 2881 conversions including its first check; warm +closed-range evaluations perform one conversion, not another minute walk. +Both caches use bounded LRU retention and hold no company/card data. Tests prove +eviction, skipped/repeated hours, half-hour DST, date-line changes, rollback, +strict boundaries, and actual twenty-card source delivery followed by quiet. diff --git a/server/src/__tests__/status-card-evaluation-work.test.ts b/server/src/__tests__/status-card-evaluation-work.test.ts index e4588b16be1..d012a239efd 100644 --- a/server/src/__tests__/status-card-evaluation-work.test.ts +++ b/server/src/__tests__/status-card-evaluation-work.test.ts @@ -272,6 +272,53 @@ const support = external ? { supported: true } : await getEmbeddedPostgresTestSu await quiet(); expect(errors).toEqual([]); }, 30_000); + it("bounds active-hours conversion work across twenty cards and repeated ordinary company writes", async () => { + vi.setSystemTime(new Date("2026-11-02T18:00:00Z")); + const hoursPolicy = { ...policy(), activeHours: { start: "09:00", end: "17:00", timezone: "UTC" } }; + const cards = []; + for (let i = 0; i < 20; i++) cards.push(await card("reactive", hoursPolicy)); + await start(); await quiet(); + const RealFormatter = Intl.DateTimeFormat; + const actualParts = RealFormatter.prototype.formatToParts; + let constructions = 0, conversions = 0; + const constructor = vi.spyOn(Intl, "DateTimeFormat").mockImplementation(function (...args: ConstructorParameters) { + if (args[0] === "en-GB" && args[1]?.hourCycle === "h23") constructions++; + return new RealFormatter(...args); + } as unknown as typeof Intl.DateTimeFormat); + const parts = vi.spyOn(RealFormatter.prototype, "formatToParts").mockImplementation(function (this: Intl.DateTimeFormat, date) { + if (this.resolvedOptions().locale === "en-GB" && this.resolvedOptions().hourCycle === "h23") conversions++; + return actualParts.call(this, date); + }); + const target = await issueService(db).create(companyId, { title: "Closed-hours blocker", status: "blocked", priority: "medium" }); + await settled(); + expect(constructions).toBeLessThanOrEqual(1); + expect(conversions).toBeLessThanOrEqual(1000); + const cold = { constructions, conversions }; + constructions = 0; conversions = 0; + for (let i = 0; i < 10; i++) { + await issueService(db).addComment(target.id, `Closed-hours human update ${i}`, { userId: "board-user" }); + await settled(); + } + expect(constructions).toBe(0); + expect(conversions).toBeGreaterThanOrEqual(200); + expect(conversions).toBeLessThanOrEqual(800); + expect(wakeup).not.toHaveBeenCalled(); + const opening = new Date("2026-11-03T09:00:00Z"); + for (const created of cards) expect((await statusCardService(db).getById(created.id))) + .toMatchObject({ state: "paused_hours", pendingChangeCount: 1, nextEvalAt: opening }); + process.stdout.write(JSON.stringify({ measurement: "status-card company active-hours workload", cards: 20, + sourceWrites: 11, cold, warm: { constructions, conversions } }) + "\n"); + constructor.mockRestore(); parts.mockRestore(); + await vi.advanceTimersByTimeAsync(opening.getTime() - Date.now()); await settled(); + expect(wakeup).toHaveBeenCalledTimes(20); + for (const created of cards) { + const generated = (await statusCardService(db).getById(created.id))!; + expect(generated.generatingIssueId).toBeTruthy(); + await writeback(created.id, generated.generatingIssueId!); + } + await quiet(); expect(errors).toEqual([]); + }, 60_000); + it("cancels archived or disabled future work and replaces an earlier interval deadline", async () => { await start(); let created = await card("interval"); await settled(); const old = coordinator!.nextWakeAt(); diff --git a/server/src/__tests__/status-card-update-engine.test.ts b/server/src/__tests__/status-card-update-engine.test.ts index b8309415770..9fdf18584e9 100644 --- a/server/src/__tests__/status-card-update-engine.test.ts +++ b/server/src/__tests__/status-card-update-engine.test.ts @@ -1,4 +1,4 @@ -import { describe, expect, it } from "vitest"; +import { afterEach, describe, expect, it, vi } from "vitest"; import { statusCardRefreshPolicySchema } from "@paperclipai/shared"; import { chooseStatusCardUpdateKind, @@ -13,6 +13,7 @@ import { } from "../services/status-card-update-engine.js"; describe("status card update engine", () => { + afterEach(() => vi.restoreAllMocks()); const defaultPolicy = statusCardRefreshPolicySchema.parse({ mode: "interval", intervalMinutes: 15 }); it("retains non-terminal and terminal status transitions plus membership changes", () => { @@ -101,6 +102,127 @@ describe("status card update engine", () => { expect(nextStatusCardPolicyEligibility({ policy, now, lastChangeAt: now, completedUpdateTimes: [], tokensToday: 0 })).toBeNull(); }); + it("bounds empty-window formatter work before any timezone conversion", () => { + const policy = statusCardRefreshPolicySchema.parse({ mode: "reactive", debounceSeconds: 1, + activeHours: { start: "09:00", end: "09:00", timezone: "Pacific/Chatham" } }); + const formatter = vi.spyOn(Intl, "DateTimeFormat").mockImplementation(function () { + throw new Error("Empty active hours must not construct a formatter"); + }); + const now = new Date("2026-07-23T14:00:00Z"); + expect(isWithinStatusCardActiveHours(policy, now)).toBe(false); + expect(nextStatusCardPolicyEligibility({ policy, now, lastChangeAt: now, completedUpdateTimes: [], tokensToday: 0 })).toBeNull(); + expect(formatter).not.toHaveBeenCalled(); + }); + + it("bounds formatter construction and reuses the next opening across cards and source writes", () => { + const policy = statusCardRefreshPolicySchema.parse({ mode: "reactive", debounceSeconds: 1, + activeHours: { start: "09:00", end: "17:00", timezone: "America/St_Johns" } }); + const RealFormatter = Intl.DateTimeFormat; + let constructions = 0; + vi.spyOn(Intl, "DateTimeFormat").mockImplementation(function (...args: ConstructorParameters) { + if (++constructions > 1) throw new Error("Card evaluation repeatedly constructed timezone formatters"); + return new RealFormatter(...args); + } as unknown as typeof Intl.DateTimeFormat); + const conversions = vi.spyOn(RealFormatter.prototype, "formatToParts"); + const now = new Date("2026-07-23T22:00:00Z"); + const input = { policy, now, lastChangeAt: new Date(now.getTime() - 60_000), completedUpdateTimes: [], tokensToday: 0 }; + const started = performance.now(); + expect(nextStatusCardPolicyEligibility(input)).toEqual(new Date("2026-07-24T11:30:00Z")); + const coldMs = performance.now() - started; + const coldConversions = conversions.mock.calls.length; + expect(coldConversions).toBeLessThanOrEqual(2881); + conversions.mockClear(); + const warmStarted = performance.now(); + for (let write = 0; write < 100; write++) for (let card = 0; card < 20; card++) { + expect(nextStatusCardPolicyEligibility({ ...input, + policy: { ...policy, activeHours: { ...policy.activeHours! } }, + now: new Date(now.getTime() + write * 1000), + })).toEqual(new Date("2026-07-24T11:30:00Z")); + } + expect(constructions).toBeLessThanOrEqual(1); + expect(conversions.mock.calls.length).toBeLessThanOrEqual(2000); + process.stdout.write(JSON.stringify({ measurement: "status-card active-hours cost", coldMs, coldConversions, constructions, + warmMs: performance.now() - warmStarted, warmConversions: conversions.mock.calls.length, evaluations: 2000 }) + "\n"); + }); + + it.each([ + ["America/New_York", "02:15", "03:30", "2026-03-08T06:50:00Z", "2026-03-08T07:00:00Z"], + ["America/New_York", "02:15", "02:45", "2026-03-08T06:50:00Z", "2026-03-09T06:15:00Z"], + ["America/New_York", "02:15", "02:45", "2026-03-07T07:45:00Z", "2026-03-09T06:15:00Z"], + ["America/New_York", "01:15", "01:45", "2026-11-01T05:50:00Z", "2026-11-01T06:15:00Z"], + ["America/New_York", "23:30", "01:30", "2026-11-01T05:40:00Z", "2026-11-01T06:00:00Z"], + ["Australia/Lord_Howe", "02:15", "02:45", "2026-10-03T15:20:00Z", "2026-10-03T15:30:00Z"], + ["Australia/Lord_Howe", "01:35", "01:45", "2026-04-04T14:50:00Z", "2026-04-04T15:05:00Z"], + ["Asia/Kathmandu", "09:00", "17:00", "2026-07-23T12:00:00Z", "2026-07-24T03:15:00Z"], + ["Pacific/Apia", "00:10", "00:20", "2011-12-30T09:50:00Z", "2011-12-30T10:10:00Z"], + ])("preserves the first real opening in %s %s-%s after %s", (timezone, start, end, time, expected) => { + const policy = statusCardRefreshPolicySchema.parse({ mode: "reactive", debounceSeconds: 1, activeHours: { start, end, timezone } }); + const now = new Date(time); + const input = { policy, now, lastChangeAt: new Date(now.getTime() - 60_000), completedUpdateTimes: [], tokensToday: 0 }; + expect(nextStatusCardPolicyEligibility(input)).toEqual(new Date(expected)); + // A later closed instant reuses the opening, while the exact opening itself + // remains eligible without rounding it into the next minute or day. + expect(nextStatusCardPolicyEligibility({ ...input, now: new Date(now.getTime() + 1000) })).toEqual(new Date(expected)); + const opening = new Date(expected); + expect(nextStatusCardPolicyEligibility({ ...input, now: opening })).toEqual(opening); + expect(nextStatusCardPolicyEligibility({ ...input, now: new Date(opening.getTime() + 1) })).toEqual(new Date(opening.getTime() + 1)); + }); + + it("invalidates an upcoming opening for clock rollback and changed policy values", () => { + const policy = statusCardRefreshPolicySchema.parse({ mode: "reactive", debounceSeconds: 1, + activeHours: { start: "09:00", end: "17:00", timezone: "UTC" } }); + const calculate = (time: string, activeHours = policy.activeHours!) => { + const now = new Date(time); + return nextStatusCardPolicyEligibility({ policy: { ...policy, activeHours }, now, + lastChangeAt: new Date(now.getTime() - 60_000), completedUpdateTimes: [], tokensToday: 0 }); + }; + expect(calculate("2026-07-23T18:00:00Z")).toEqual(new Date("2026-07-24T09:00:00Z")); + expect(calculate("2026-07-23T08:00:00Z")).toEqual(new Date("2026-07-23T09:00:00Z")); + expect(calculate("2026-07-23T17:00:00Z")).toEqual(new Date("2026-07-24T09:00:00Z")); + expect(calculate("2026-07-23T18:00:00Z", { ...policy.activeHours!, start: "10:00" })).toEqual(new Date("2026-07-24T10:00:00Z")); + expect(calculate("2026-07-23T18:00:00Z", { ...policy.activeHours!, end: "19:00" })).toEqual(new Date("2026-07-23T18:00:00Z")); + expect(calculate("2026-07-23T18:00:00Z", { ...policy.activeHours!, timezone: "Asia/Tokyo" })).toEqual(new Date("2026-07-24T00:00:00Z")); + }); + + it("checks the precise instant before a cached minute opening for historical second offsets", () => { + const policy = statusCardRefreshPolicySchema.parse({ mode: "reactive", debounceSeconds: 1, + activeHours: { start: "09:00", end: "10:00", timezone: "Africa/Monrovia" } }); + const now = new Date("1970-01-01T08:00:00Z"); + const input = { policy, now, lastChangeAt: new Date(now.getTime() - 60_000), completedUpdateTimes: [], tokensToday: 0 }; + expect(nextStatusCardPolicyEligibility(input)).toEqual(new Date("1970-01-01T09:45:00Z")); + const inside = new Date("1970-01-01T09:44:45Z"); + expect(isWithinStatusCardActiveHours(policy, inside)).toBe(true); + expect(nextStatusCardPolicyEligibility({ ...input, now: inside })).toEqual(inside); + }); + + it("bounds formatter and upcoming-opening cache retention", () => { + const now = new Date("2026-07-25T00:00:00Z"); + const timezonePolicy = (timezone: string) => statusCardRefreshPolicySchema.parse({ mode: "reactive", debounceSeconds: 1, + activeHours: { start: "09:00", end: "17:00", timezone } }); + const zones = Intl.supportedValuesOf("timeZone").slice(0, 34); + const first = timezonePolicy(zones[0]!); + isWithinStatusCardActiveHours(first, now); + for (const zone of zones.slice(1)) isWithinStatusCardActiveHours(timezonePolicy(zone), now); + const RealFormatter = Intl.DateTimeFormat; + const constructor = vi.spyOn(Intl, "DateTimeFormat").mockImplementation(function (...args: ConstructorParameters) { + return new RealFormatter(...args); + } as unknown as typeof Intl.DateTimeFormat); + isWithinStatusCardActiveHours(first, now); + expect(constructor).toHaveBeenCalledTimes(1); + constructor.mockRestore(); + const time = (minute: number) => `${String(Math.floor(minute / 60)).padStart(2, "0")}:${String(minute % 60).padStart(2, "0")}`; + const calculate = (minute: number) => { + const policy = statusCardRefreshPolicySchema.parse({ mode: "reactive", debounceSeconds: 1, + activeHours: { start: time(minute), end: time(minute + 1), timezone: "UTC" } }); + return nextStatusCardPolicyEligibility({ policy, now, lastChangeAt: new Date(now.getTime() - 60_000), completedUpdateTimes: [], tokensToday: 0 }); + }; + calculate(1); + for (let minute = 2; minute <= 258; minute++) calculate(minute); + const conversions = vi.spyOn(Intl.DateTimeFormat.prototype, "formatToParts"); + expect(calculate(1)).toEqual(new Date("2026-07-25T00:01:00Z")); + expect(conversions.mock.calls.length).toBeGreaterThan(1); + }); + it("selects full rebuilds for bounded drift rules and incremental otherwise", () => { const base = { hasDocument: true, changeCount: 2, queryVersion: 3, lastUpdateQueryVersion: 3, incrementalCount: 2, configurationChanged: false }; expect(chooseStatusCardUpdateKind(base)).toBe("incremental"); diff --git a/server/src/services/status-card-update-engine.ts b/server/src/services/status-card-update-engine.ts index 1f734227808..0b1e5325446 100644 --- a/server/src/services/status-card-update-engine.ts +++ b/server/src/services/status-card-update-engine.ts @@ -109,21 +109,36 @@ export function statusCardFingerprintHash(fingerprint: StatusCardFingerprint) { return createHash("sha256").update(JSON.stringify(stable)).digest("hex"); } +// These caches contain only timezone/policy calculations, never card or company +// data. Bound both maps because timezone aliases and hour pairs are user input. +const activeHoursFormatters = new Map(); +const upcomingActiveHours = new Map(); + +function remember(cache: Map, key: K, value: V, limit: number): V { + cache.delete(key); + cache.set(key, value); + if (cache.size > limit) cache.delete(cache.keys().next().value!); + return value; +} + +function activeHoursFormatter(timezone: string) { + const cached = activeHoursFormatters.get(timezone); + return remember(activeHoursFormatters, timezone, cached ?? new Intl.DateTimeFormat("en-GB", { + timeZone: timezone, hour: "2-digit", minute: "2-digit", hourCycle: "h23", + }), 32); +} + export function isWithinStatusCardActiveHours(policy: StatusCardRefreshPolicy, now: Date) { if (!policy.activeHours) return true; - const parts = new Intl.DateTimeFormat("en-GB", { - timeZone: policy.activeHours.timezone, - hour: "2-digit", - minute: "2-digit", - hourCycle: "h23", - }).formatToParts(now); - const hour = Number(parts.find((part) => part.type === "hour")?.value ?? 0); - const minute = Number(parts.find((part) => part.type === "minute")?.value ?? 0); - const current = hour * 60 + minute; const [startHour, startMinute] = policy.activeHours.start.split(":").map(Number); const [endHour, endMinute] = policy.activeHours.end.split(":").map(Number); const start = startHour! * 60 + startMinute!; const end = endHour! * 60 + endMinute!; + if (start === end) return false; + const parts = activeHoursFormatter(policy.activeHours.timezone).formatToParts(now); + const hour = Number(parts.find((part) => part.type === "hour")?.value ?? 0); + const minute = Number(parts.find((part) => part.type === "minute")?.value ?? 0); + const current = hour * 60 + minute; return start <= end ? current >= start && current < end : current >= start || current < end; } @@ -181,6 +196,8 @@ export function nextStatusCardPolicyEligibility(input: { tokensToday: number; }): Date | null { if (input.policy.mode === "manual") return null; + const hours = input.policy.activeHours; + if (hours && hours.start === hours.end) return null; let at = input.now.getTime(); if (input.tokensToday >= (input.policy.dailyTokenCap ?? 100_000)) { const day = new Date(input.now); @@ -194,13 +211,25 @@ export function nextStatusCardPolicyEligibility(input: { .filter(time => time > at - 3_600_000).sort((a, b) => a - b); if (recent.length >= cap) at = Math.max(at, recent[recent.length - cap]! + 3_600_000 + 1); } - if (isWithinStatusCardActiveHours(input.policy, new Date(at))) return new Date(at); + // Check the precise instant before reusing a minute-aligned opening. Some + // historical timezone offsets include seconds, so a window may be open now. + if (!hours || isWithinStatusCardActiveHours(input.policy, new Date(at))) return new Date(at); + const key = JSON.stringify([hours.timezone, hours.start, hours.end]); + const cached = upcomingActiveHours.get(key); + if (cached && at >= cached.closedAt && at < cached.opensAt) { + remember(upcomingActiveHours, key, cached, 256); + return new Date(cached.opensAt); + } + const closedAt = at; // Active-hour precision is one local minute. Walking UTC minutes also honors // skipped/repeated local times at daylight-saving transitions. const end = at + 48 * 3_600_000; for (at = Math.floor(at / 60_000) * 60_000 + 60_000; at <= end; at += 60_000) { - if (isWithinStatusCardActiveHours(input.policy, new Date(at))) return new Date(at); + if (isWithinStatusCardActiveHours(input.policy, new Date(at))) { + remember(upcomingActiveHours, key, { closedAt, opensAt: at }, 256); + return new Date(at); + } } - // Equal start/end describes an empty window; retain changes, awaiting config. + // No opening inside the existing bounded search horizon. Retain changes. return null; } From c74f5f12bad434068fcb7209c4e22e45eed7149c Mon Sep 17 00:00:00 2001 From: Dotta Date: Fri, 30 Oct 2026 19:45:15 -0500 Subject: [PATCH 66/70] Retain moved receipt deadlines and wake released execution gates Co-Authored-By: Paperclip --- .../deferred-execution-receipt-work.test.ts | 198 +++++++++++++++++- server/src/services/access.ts | 2 + .../src/services/deferred-execution-wait.ts | 24 +++ .../heartbeat/deferred-receipt-work.ts | 18 ++ server/src/services/heartbeat/queue.ts | 23 +- server/src/services/heartbeat/run-control.ts | 8 +- server/src/services/issue-tree-control.ts | 5 + 7 files changed, 249 insertions(+), 29 deletions(-) create mode 100644 server/src/services/deferred-execution-wait.ts diff --git a/server/src/__tests__/deferred-execution-receipt-work.test.ts b/server/src/__tests__/deferred-execution-receipt-work.test.ts index ba0f13bed58..eb60e2ed449 100644 --- a/server/src/__tests__/deferred-execution-receipt-work.test.ts +++ b/server/src/__tests__/deferred-execution-receipt-work.test.ts @@ -1,7 +1,14 @@ +import { DELIVERY_QUEUES, subscribeDeliveryWork } from "../services/delivery-work-notifications.js"; +import { accessService } from "../services/access.js"; +import { writeDeferredExecutionWait } from "../services/deferred-execution-wait.js"; +import { createQueuedCommentIssueLockWriter } from "../modules/wake-queue/adapters/queued-comment-postgres.js"; +import { createHeartbeatQueue } from "../services/heartbeat/queue.js"; +import { createHeartbeatQueueTestDependencies } from "./helpers/heartbeat-queue.js"; +import { issueTreeControlService } from "../services/issue-tree-control.js"; import { randomUUID } from "node:crypto"; import { and, eq, sql } from "drizzle-orm"; import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; -import { agents, agentWakeupRequests, authUsers, chatActions, chatEndpoints, toolApplications, toolConnections, companies, companyMemberships, createDb, heartbeatRuns, issues } from "@paperclipai/db"; +import { agents, agentWakeupRequests, authUsers, chatActions, chatEndpoints, toolApplications, toolConnections, companies, companyMemberships, createDb, reconcileDatabaseWork, heartbeatRuns, issueComments, issues } from "@paperclipai/db"; import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js"; import { startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; import { createDeliveryWorkCoordinator } from "../services/delivery-work-coordinator.js"; @@ -61,6 +68,15 @@ describe("deferred receipt durable dispatch", () => { await vi.advanceTimersByTimeAsync(20 * 60_000); expect(reads).toHaveBeenCalledTimes(selected); expect(executes).toHaveBeenCalledTimes(executed); } + async function finish(issueId: string) { + await issueService(db).update(issueId, { status: "done" }); + await heartbeat.drainActiveRunExecutions(); + const saved = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.status, "scheduled_retry")); + expect(await db.select().from(heartbeatRuns).where(sql`${heartbeatRuns.status} in ('queued', 'running')`)).toHaveLength(0); + vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"] }); + if (saved.length) await vi.advanceTimersByTimeAsync(Math.max(0, Math.max(...saved.map(run => run.scheduledRetryAt!.getTime())) - Date.now()) + 1); + await vi.waitFor(async () => expect(await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.status, "scheduled_retry"))).toHaveLength(0)); + } it("is quiet after empty reconstruction and terminal receipt history", async () => { const f = await fixture(); await db.insert(agentWakeupRequests).values({ companyId: f.companyId, agentId: f.agent.id, @@ -86,13 +102,7 @@ describe("deferred receipt durable dispatch", () => { expect(run.status, JSON.stringify({ error: run.error, code: run.errorCode })).toBe("succeeded"); expect(run.stdoutExcerpt).toContain("receipt-proof"); }, { timeout: 15_000 }); - await issueService(db).update(f.issue.id, { status: "done" }); - await heartbeat.drainActiveRunExecutions(); - const saved = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.status, "scheduled_retry")); - expect(await db.select().from(heartbeatRuns).where(sql`${heartbeatRuns.status} in ('queued', 'running')`)).toHaveLength(0); - vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"] }); - if (saved.length) await vi.advanceTimersByTimeAsync(Math.max(0, Math.max(...saved.map(run => run.scheduledRetryAt!.getTime())) - Date.now()) + 1); - await vi.waitFor(async () => expect(await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.status, "scheduled_retry"))).toHaveLength(0)); + await finish(f.issue.id); const linked = await db.select().from(agentWakeupRequests).where(and(eq(agentWakeupRequests.reason, "task_retry"), sql`${agentWakeupRequests.runId} is not null`)); expect(linked).toHaveLength(2); expect(new Set(linked.map(row => row.runId))).toEqual(new Set([runId])); @@ -208,6 +218,178 @@ describe("deferred receipt durable dispatch", () => { .toBe("issued"), { timeout: 5_000 }); await quiet(); }); + it("retains a moved interrupt grace deadline after an ordinary saved queue reorder", async () => { + const f = await fixture(), initialAt = new Date(); + const comments = await db.insert(issueComments).values(["first", "second"].map(body => ({ + companyId: f.companyId, issueId: f.issue.id, body, authorType: "user", authorUserId: f.actorId, + }))).returning(); + const payload = { issueId: f.issue.id, queuedCommentInterrupt: { actorId: f.actorId }, + _paperclipWakeContext: { wakeCommentIds: comments.map(row => row.id) } }; + const [wake] = await db.insert(agentWakeupRequests).values({ companyId: f.companyId, agentId: f.agent.id, + source: "on_demand", status: "deferred_issue_execution", payload, updatedAt: initialAt }).returning(); + vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"] }); + const deps = createHeartbeatQueueTestDependencies(db), queue = createHeartbeatQueue(db, deps); + await registerDeferredReceiptWork(db, coordinator, { resume: scope => queue.resumeDeferredExecutionReceipts(scope), + settleTaskChat: async () => {}, dispatchNative: async () => {}, schedulingSuppressed: async () => false }, () => !drain).ready; + await vi.advanceTimersByTimeAsync(initialAt.getTime() + 10_000 - Date.now()); + const movedAt = new Date(); + const writer = createQueuedCommentIssueLockWriter(db, { syncCommentReferences: async () => {}, + deleteCommentReferenceSource: async () => {}, syncCommentExternalObjectsSafely: async () => {} }); + await writer.withLockedQueue({ issue: f.issue, queueId: wake.id, + actor: { actorType: "user", actorId: f.actorId, agentId: null, runId: null, agentApiKeyId: null } }, + async (_locked, tx) => { await tx.updateWakeQueuedCommentIds({ wakeId: wake.id, payload, + ids: comments.map(row => row.id).reverse(), updatedAt: movedAt }); }); + await vi.advanceTimersByTimeAsync(initialAt.getTime() + 30_000 - Date.now()); + await vi.waitFor(() => expect(idleWorkSnapshot().active).toBe(0)); + expect(deps.resumeQueuedCommentInterrupt).not.toHaveBeenCalled(); + expect(coordinator.nextWakeAt()).toBe(movedAt.getTime() + 30_000); + await vi.advanceTimersByTimeAsync(movedAt.getTime() + 30_000 - Date.now()); + await vi.waitFor(() => expect(deps.resumeQueuedCommentInterrupt).toHaveBeenCalledExactlyOnceWith(f.companyId, wake.id)); + await quiet(); + }); + it("reactivates a held task Retry when the authorized subtree hold is released without wakeAgents", async () => { + const f = await fixture(), tree = issueTreeControlService(db); + const actor = { actorType: "user" as const, actorId: f.actorId, userId: f.actorId }; + const hold = await tree.createHold(f.companyId, f.issue.id, { mode: "pause", actor }); + await heartbeat.registerDeferredReceiptWork(coordinator, () => !drain).ready; + await heartbeat.registerRunDispatchWork(coordinator, () => !drain).ready; + await heartbeat.registerScheduledRetryCancellationWork(coordinator, () => !drain).ready; + const request = await requestTaskRetry({ db, companyId: f.companyId, issueId: f.issue.id, actorId: f.actorId, + observedRunId: f.source.id, idempotencyKey: randomUUID() }); + expect(request.requestId).toBeTruthy(); + await vi.waitFor(async () => { + const [wake] = await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, request.requestId!)); + expect(wake.payload?.executionWait).toMatchObject({ reason: "issue_tree_hold_active" }); + expect(wake.runId).toBeNull(); + }, { timeout: 5_000 }); + await vi.waitFor(() => { expect(coordinator.nextWakeAt()).toBeNull(); expect(idleWorkSnapshot().active).toBe(0); }, { timeout: 40_000 }); + await tree.releaseHold(f.companyId, f.issue.id, hold.hold.id, { actor, reason: "Resume saved Retry" }); + let runId: string | null = null; + await vi.waitFor(async () => { + const [wake] = await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, request.requestId!)); + expect(wake.runId, JSON.stringify(wake)).toBeTruthy(); runId = wake.runId; + }, { timeout: 40_000 }); + await vi.waitFor(async () => { + await heartbeat.drainActiveRunExecutions(); + const [run] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, runId!)); + expect(run.status, JSON.stringify({ error: run.error, code: run.errorCode })).toBe("succeeded"); + expect(run.stdoutExcerpt).toContain("receipt-proof"); + }, { timeout: 15_000 }); + expect(await db.select().from(heartbeatRuns).where(sql`${heartbeatRuns.contextSnapshot}->>'taskRetryRequestId' = ${request.requestId}`)).toHaveLength(1); + await finish(f.issue.id); await quiet(); + }); + it("keeps a moved native dispatch error deadline instead of retrying at its old time", async () => { + const f = await fixture(), initialAt = new Date(); + const [wake] = await db.insert(agentWakeupRequests).values({ companyId: f.companyId, agentId: f.agent.id, + source: "automation", status: "queued", error: "Transient dispatch failure", updatedAt: initialAt, + requestedByActorType: "system", requestedByActorId: "native-status-committer" }).returning(); + vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"] }); + const dispatchNative = vi.fn(async () => {}); + await registerDeferredReceiptWork(db, coordinator, { resume: async () => {}, settleTaskChat: async () => {}, + dispatchNative, schedulingSuppressed: async () => false }, () => !drain).ready; + await vi.advanceTimersByTimeAsync(initialAt.getTime() + 10_000 - Date.now()); + const movedAt = new Date(); + await db.update(agentWakeupRequests).set({ updatedAt: movedAt }).where(eq(agentWakeupRequests.id, wake.id)); + await vi.advanceTimersByTimeAsync(initialAt.getTime() + 30_000 - Date.now()); + await vi.waitFor(() => expect(idleWorkSnapshot().active).toBe(0)); + expect(dispatchNative).not.toHaveBeenCalled(); expect(coordinator.nextWakeAt()).toBe(movedAt.getTime() + 30_000); + await vi.advanceTimersByTimeAsync(movedAt.getTime() + 30_000 - Date.now()); + await vi.waitFor(() => expect(dispatchNative).toHaveBeenCalledTimes(1)); await quiet(); + }); + it("arms one new wait reason grace and stays quiet when that same gate is still closed", async () => { + const f = await fixture(); let reason = "operator_pause"; + const resume = vi.fn(async (scope: { companyId?: string; requestId?: string }) => { + await writeDeferredExecutionWait(db, { companyId: scope.companyId!, requestId: scope.requestId! }, { reason }); + }); + await registerDeferredReceiptWork(db, coordinator, { resume, settleTaskChat: async () => {}, dispatchNative: async () => {}, + schedulingSuppressed: async () => false }, () => !drain).ready; + const [wake] = await db.insert(agentWakeupRequests).values({ companyId: f.companyId, agentId: f.agent.id, + source: "on_demand", status: "deferred_issue_execution", reason: "task_retry" }).returning(); + vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"] }); + for (const nextReason of ["operator_pause", "budget_pause"]) { + reason = nextReason; + await writeDeferredExecutionWait(db, { companyId: f.companyId, requestId: wake.id }, { reason }); + const [current] = await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, wake.id)); + const due = current.updatedAt.getTime() + 30_000; + await vi.advanceTimersByTimeAsync(1_000); + await vi.waitFor(() => expect(coordinator.nextWakeAt()).toBe(due)); + const checked = resume.mock.calls.length; + await vi.advanceTimersByTimeAsync(due - Date.now()); + await vi.waitFor(() => expect(resume).toHaveBeenCalledTimes(checked + 1)); + await quiet(); + } + expect(resume).toHaveBeenCalledTimes(2); + }); + it("reactivates a saved Retry through ordinary restore cancelled task", async () => { + const f = await fixture(), actor = { actorType: "user" as const, actorId: f.actorId, userId: f.actorId }; + const request = await requestTaskRetry({ db, companyId: f.companyId, issueId: f.issue.id, actorId: f.actorId, + observedRunId: f.source.id, idempotencyKey: randomUUID() }); + const tree = issueTreeControlService(db); + const cancel = await tree.createHold(f.companyId, f.issue.id, { mode: "cancel", actor }); + await tree.cancelIssueStatusesForHold(f.companyId, f.issue.id, cancel.hold.id); + const restore = await tree.createHold(f.companyId, f.issue.id, { mode: "restore", actor }); + const commitEligibility = () => tree.restoreIssueStatusesForHold(f.companyId, f.issue.id, restore.hold.id, { actor }); + await heartbeat.registerDeferredReceiptWork(coordinator, () => !drain).ready; + await heartbeat.registerRunDispatchWork(coordinator, () => !drain).ready; + await heartbeat.registerScheduledRetryCancellationWork(coordinator, () => !drain).ready; + await vi.waitFor(() => { expect(coordinator.nextWakeAt()).toBeNull(); expect(idleWorkSnapshot().active).toBe(0); }, { timeout: 5_000 }); + expect((await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, request.requestId!)))[0].runId).toBeNull(); + await commitEligibility(); + let runId: string | null = null; + await vi.waitFor(async () => { + const [wake] = await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, request.requestId!)); + expect(wake.runId, JSON.stringify(wake)).toBeTruthy(); runId = wake.runId; + }, { timeout: 15_000 }); + await vi.waitFor(async () => { + await heartbeat.drainActiveRunExecutions(); + const [run] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, runId!)); + expect(run.status, JSON.stringify({ error: run.error, code: run.errorCode })).toBe("succeeded"); + expect(run.stdoutExcerpt).toContain("receipt-proof"); + }, { timeout: 15_000 }); + expect(await db.select().from(heartbeatRuns).where(sql`${heartbeatRuns.contextSnapshot}->>'taskRetryRequestId' = ${request.requestId}`)).toHaveLength(1); + await finish(f.issue.id); await quiet(); + }); + it("member archive publishes committed eligibility while preserving a cancelled Retry tombstone", async () => { + const f = await fixture(), departing = `departing-${randomUUID()}`; + const request = await requestTaskRetry({ db, companyId: f.companyId, issueId: f.issue.id, actorId: f.actorId, + observedRunId: f.source.id, idempotencyKey: randomUUID() }); + await db.insert(authUsers).values({ id: departing, name: "Departing member", email: `${departing}@example.test`, + createdAt: new Date(), updatedAt: new Date() }); + const [member] = await db.insert(companyMemberships).values({ companyId: f.companyId, principalId: departing, + principalType: "user", status: "active", membershipRole: "member" }).returning(); + await issueService(db).update(f.issue.id, { assigneeUserId: departing, assigneeAgentId: null }); + const readIntent = async () => (await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, request.requestId!)))[0]; + expect(await readIntent()).toMatchObject({ status: "cancelled", runId: null }); + const deferred = vi.fn(), native = vi.fn(); + const unsubscribes = [subscribeDeliveryWork(db, DELIVERY_QUEUES.deferredExecutionReceipts, deferred), + subscribeDeliveryWork(db, DELIVERY_QUEUES.nativeStatusWakeIntents, native)]; + try { + await expect(db.transaction(async tx => { + await accessService(tx as unknown as typeof db).archiveMember(f.companyId, member.id, + { reassignment: { assigneeAgentId: f.agent.id } }); + expect(deferred).not.toHaveBeenCalled(); expect(native).not.toHaveBeenCalled(); + throw new Error("roll back member archive"); + })).rejects.toThrow("roll back member archive"); + expect(deferred).toHaveBeenCalledTimes(1); expect(native).toHaveBeenCalledTimes(1); + expect((await db.select().from(issues).where(eq(issues.id, f.issue.id)))[0].assigneeUserId).toBe(departing); + await reconcileDatabaseWork(db, DELIVERY_QUEUES.deferredExecutionReceipts); + await reconcileDatabaseWork(db, DELIVERY_QUEUES.nativeStatusWakeIntents); + deferred.mockClear(); native.mockClear(); + await db.transaction(async tx => { + await accessService(tx as unknown as typeof db).archiveMember(f.companyId, member.id, + { reassignment: { assigneeAgentId: f.agent.id } }); + expect(deferred).not.toHaveBeenCalled(); expect(native).not.toHaveBeenCalled(); + }); + expect(deferred).toHaveBeenCalledTimes(1); expect(native).toHaveBeenCalledTimes(1); + expect((await db.select().from(issues).where(eq(issues.id, f.issue.id)))[0]).toMatchObject({ + assigneeUserId: null, assigneeAgentId: f.agent.id, + }); + expect((await db.select().from(companyMemberships).where(eq(companyMemberships.id, member.id)))[0].status).toBe("archived"); + expect(await readIntent()).toMatchObject({ status: "cancelled", runId: null }); + expect(await db.select().from(heartbeatRuns).where(sql`${heartbeatRuns.contextSnapshot}->>'taskRetryRequestId' = ${request.requestId}`)).toHaveLength(0); + } finally { for (const unsubscribe of unsubscribes) unsubscribe(); } + await quiet(); + }); it("outer rollback and nested savepoint rollback do not dispatch receipt intent", async () => { const f = await fixture(); const dispatchNative = vi.fn(async () => {}); await registerDeferredReceiptWork(db, coordinator, { resume: vi.fn(), settleTaskChat: vi.fn(), dispatchNative, diff --git a/server/src/services/access.ts b/server/src/services/access.ts index aa0f9dcc828..32086736eb2 100644 --- a/server/src/services/access.ts +++ b/server/src/services/access.ts @@ -1,3 +1,4 @@ +import { notifyDeferredExecutionEligibility } from "./deferred-execution-notifications.js"; import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js"; import { notifyNativeStatusReassessment } from "./native-status-reassessment-notifications.js"; import { notifyUnsafeWorkspaceExportRecovery } from "./unsafe-workspace-export-notifications.js"; @@ -743,6 +744,7 @@ export function accessService(db: Db) { sql`${issues.status} not in ('done', 'cancelled')`, ); await notifyNativeStatusReassessment(tx, { companyId }); + await notifyDeferredExecutionEligibility(tx, { companyId }); await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors); await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers); const resetInProgress = await tx diff --git a/server/src/services/deferred-execution-wait.ts b/server/src/services/deferred-execution-wait.ts new file mode 100644 index 00000000000..c4857d5b692 --- /dev/null +++ b/server/src/services/deferred-execution-wait.ts @@ -0,0 +1,24 @@ +import { and, eq, sql } from "drizzle-orm"; +import { agentWakeupRequests, type Db } from "@paperclipai/db"; +import { notifyDeferredExecutionWork } from "./deferred-execution-notifications.js"; + +export async function writeDeferredExecutionWait(db: Db, scope: { companyId: string; requestId: string }, + condition: Record, options: { replace?: boolean } = {}) { + return db.transaction(async tx => { + const [current] = await tx.select({ payload: agentWakeupRequests.payload }).from(agentWakeupRequests) + .where(and(eq(agentWakeupRequests.companyId, scope.companyId), eq(agentWakeupRequests.id, scope.requestId), + eq(agentWakeupRequests.status, "deferred_issue_execution"))).for("update"); + if (!current) return; + const previous = current.payload?.executionWait; + const previousReason = previous && typeof previous === "object" && !Array.isArray(previous) + ? (previous as Record).reason : undefined; + if (!previous || previousReason !== condition.reason) await notifyDeferredExecutionWork(tx, scope); + await tx.update(agentWakeupRequests).set({ + payload: sql`jsonb_set(coalesce(${agentWakeupRequests.payload}, '{}'::jsonb), '{executionWait}', + ${options.replace ? sql`'{}'::jsonb` : sql`coalesce(${agentWakeupRequests.payload}->'executionWait', '{}'::jsonb)`} + || ${JSON.stringify(condition)}::jsonb)`, + updatedAt: new Date(), + }).where(and(eq(agentWakeupRequests.companyId, scope.companyId), eq(agentWakeupRequests.id, scope.requestId), + eq(agentWakeupRequests.status, "deferred_issue_execution"))); + }); +} diff --git a/server/src/services/heartbeat/deferred-receipt-work.ts b/server/src/services/heartbeat/deferred-receipt-work.ts index fddbce3be8d..7a69bc3be58 100644 --- a/server/src/services/heartbeat/deferred-receipt-work.ts +++ b/server/src/services/heartbeat/deferred-receipt-work.ts @@ -79,6 +79,24 @@ export function registerDeferredReceiptWork( }, check: async entry => { if (!enabled) return null; + if (kind !== "task_chat") { + const [current] = await db.select({ status: agentWakeupRequests.status, payload: agentWakeupRequests.payload, + updatedAt: agentWakeupRequests.updatedAt, claimedAt: agentWakeupRequests.claimedAt, error: agentWakeupRequests.error, + interruptedLegacy: sql`exists (select 1 from heartbeat_runs r + where r.company_id = ${agentWakeupRequests.companyId} and r.agent_id = ${agentWakeupRequests.agentId} + and r.runtime_mode = 'legacy' and r.status = 'cancelled' + and r.result_json->>'queuedCommentInterruptQueueId' = ${agentWakeupRequests.id}::text)`, + }).from(agentWakeupRequests).where(and(eq(agentWakeupRequests.companyId, entry.companyId), + eq(agentWakeupRequests.id, entry.id))); + if (!current) return null; + const due = kind === "native" && current.status === "claimed" && current.claimedAt + ? current.claimedAt.getTime() + 60_000 + : kind === "native" && current.status === "queued" && current.error ? current.updatedAt.getTime() + 30_000 + : kind === "deferred" && current.status === "deferred_issue_execution" && !current.interruptedLegacy + && (current.payload?.queuedCommentInterrupt || current.payload?.executionWait) + ? current.updatedAt.getTime() + 30_000 : null; + if (due !== null && due > Date.now()) return { ...entry, dueAt: due }; + } const errors: unknown[] = []; let next: number | null = null; const scope = { companyId: entry.companyId, requestId: entry.id }; diff --git a/server/src/services/heartbeat/queue.ts b/server/src/services/heartbeat/queue.ts index ef3ecfeb7ac..962f15f327e 100644 --- a/server/src/services/heartbeat/queue.ts +++ b/server/src/services/heartbeat/queue.ts @@ -1,3 +1,4 @@ +import { writeDeferredExecutionWait } from "../deferred-execution-wait.js"; import { notifyDeferredExecutionWork } from "../deferred-execution-notifications.js"; import type { DeferredReceiptRecoveryOptions } from "../deferred-execution-notifications.js"; import { notifyRunDispatch } from "../run-dispatch-notifications.js"; @@ -2523,17 +2524,11 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende // its one durable receipt ID on a skipped wake. if (caseRequest) return { created: false }; if (executionWaitRequestId) { - await db.update(agentWakeupRequests).set({ - payload: sql`jsonb_set(coalesce(${agentWakeupRequests.payload}, '{}'::jsonb), '{executionWait}', - coalesce(${agentWakeupRequests.payload}->'executionWait', '{}'::jsonb) || ${JSON.stringify({ - reason: skipReason, message: patch.error ?? (skipReason === "issue_tree_hold_active" - ? "This task is paused. Resume it to send your saved message." - : "Waiting for task execution to be enabled. Your message is saved."), - })}::jsonb)`, - updatedAt: new Date(), - }).where(and(eq(agentWakeupRequests.id, executionWaitRequestId), - eq(agentWakeupRequests.companyId, agent.companyId), eq(agentWakeupRequests.agentId, agentId), - eq(agentWakeupRequests.status, "deferred_issue_execution"))); + await writeDeferredExecutionWait(db, { companyId: agent.companyId, requestId: executionWaitRequestId }, { + reason: skipReason, message: patch.error ?? (skipReason === "issue_tree_hold_active" + ? "This task is paused. Resume it to send your saved message." + : "Waiting for task execution to be enabled. Your message is saved."), + }); return { created: false }; } const request = { @@ -3421,10 +3416,8 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende opts.issueStateGuard?.onExecutionDeferred?.(executionBlocker.cause); const condition = { recoveryActionId: executionBlocker.recoveryActionId, ...continuationWait }; if (executionWaitRequestId) { - await tx.update(agentWakeupRequests).set({ - payload: sql`jsonb_set(coalesce(${agentWakeupRequests.payload}, '{}'::jsonb), '{executionWait}', ${JSON.stringify(condition)}::jsonb)`, - updatedAt: new Date(), - }).where(eq(agentWakeupRequests.id, executionWaitRequestId)); + await writeDeferredExecutionWait(tx as unknown as Db, { companyId: agent.companyId, + requestId: executionWaitRequestId }, condition, { replace: true }); return { kind: "deferred" as const, admitted: true }; } if (durableRequest || wakeCommentId || diff --git a/server/src/services/heartbeat/run-control.ts b/server/src/services/heartbeat/run-control.ts index 691309a1d76..a410d905d6a 100644 --- a/server/src/services/heartbeat/run-control.ts +++ b/server/src/services/heartbeat/run-control.ts @@ -1,3 +1,4 @@ +import { writeDeferredExecutionWait } from "../deferred-execution-wait.js"; import type { DeferredReceiptRecoveryOptions } from "../deferred-execution-notifications.js"; import { observeStoppedWorkspaceRecovery } from "../workspace-recovery.js"; import { notifyScheduledRetryCancellation } from "../scheduled-retry-cancellation-notifications.js"; @@ -617,12 +618,7 @@ export function createHeartbeatRunControl(db: Db, dependencies: HeartbeatRunCont onBlocked: (reason, message) => { wait = { reason, message }; }, }); if (!admitted) { - await db.update(agentWakeupRequests).set({ - payload: sql`jsonb_set(coalesce(${agentWakeupRequests.payload}, '{}'::jsonb), '{executionWait}', - coalesce(${agentWakeupRequests.payload}->'executionWait', '{}'::jsonb) || ${JSON.stringify(wait)}::jsonb)`, - updatedAt: new Date(), - }).where(and(eq(agentWakeupRequests.id, wake.id), eq(agentWakeupRequests.companyId, run.companyId), - eq(agentWakeupRequests.status, "deferred_issue_execution"))); + await writeDeferredExecutionWait(db, { companyId: run.companyId, requestId: wake.id }, wait); continue; } } diff --git a/server/src/services/issue-tree-control.ts b/server/src/services/issue-tree-control.ts index ff558b81ef0..2c05df61ceb 100644 --- a/server/src/services/issue-tree-control.ts +++ b/server/src/services/issue-tree-control.ts @@ -1,3 +1,4 @@ +import { notifyDeferredExecutionEligibility } from "./deferred-execution-notifications.js"; import { notifyIssueRecovery } from "./issue-recovery-notifications.js"; import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js"; import { notifyNativeStatusReassessment } from "./native-status-reassessment-notifications.js"; @@ -760,6 +761,7 @@ export function issueTreeControlService(db: Db) { const { hold: resumeHold } = await db.transaction(async (tx) => { await notifyIssueRecovery(tx, { companyId }); + await notifyDeferredExecutionEligibility(tx, { companyId }); const [createdHold] = await tx .insert(issueTreeHolds) .values({ @@ -839,6 +841,7 @@ export function issueTreeControlService(db: Db) { const { hold, members } = await db.transaction(async (tx) => { await notifyIssueRecovery(tx, { companyId }); + await notifyDeferredExecutionEligibility(tx, { companyId }); const [createdHold] = await tx .insert(issueTreeHolds) .values({ @@ -1003,6 +1006,7 @@ export function issueTreeControlService(db: Db) { const now = new Date(); const releasedCancelHoldIds = activeCancelHolds.map((hold) => hold.id); const updatedIssues = await db.transaction(async (tx) => { + await notifyDeferredExecutionEligibility(tx, { companyId }); await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors); await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers); const restored: TreeStatusUpdateResult["updatedIssues"] = []; @@ -1182,6 +1186,7 @@ export function issueTreeControlService(db: Db) { const [updated] = await db.transaction(async tx => { await notifyIssueRecovery(tx, { companyId }); + await notifyDeferredExecutionEligibility(tx, { companyId }); return tx .update(issueTreeHolds) .set({ From 8911cca5d0b43a9f8e8adddc8677b333b3f3d2bb Mon Sep 17 00:00:00 2001 From: Dotta Date: Fri, 30 Oct 2026 19:49:06 -0500 Subject: [PATCH 67/70] fix(server): retain live evidence beyond blocker graph admission Keep bounded returned child facts independent of graph node and edge admission, and use the existing live-work predicate for running status, active runs and pending waits. Cover the opposite query ordering with 1,999 explicit blockers, fresh retention checks, no archive/outbox and unchanged SQL/resource bounds. Co-Authored-By: Paperclip --- .../2026-10-30-attention-retention-sleep.md | 2 +- .../attention-retention-work.test.ts | 31 +++++++++++++++++ server/src/services/issues.ts | 34 +++++++++++++------ 3 files changed, 55 insertions(+), 12 deletions(-) diff --git a/doc/plans/2026-10-30-attention-retention-sleep.md b/doc/plans/2026-10-30-attention-retention-sleep.md index aa78f1bb746..2be011ccf9d 100644 --- a/doc/plans/2026-10-30-attention-retention-sleep.md +++ b/doc/plans/2026-10-30-attention-retention-sleep.md @@ -25,7 +25,7 @@ The feed and worker share confirmation collapse, newer-run probes, issue visibil ## Existing mutation boundaries reactivate work -Committed hints identify a company, source kind, source ID or changed issue where those values are known. Routine issue writers pass identity to the retention domain helper before their mutation. The helper captures old and new parent paths, reverse blocker ancestors, run-linked issues and liveness-origin references; issue-scoped source queries then select reviews, blockers, interactions, decisions, recovery actions and failed runs by their issue identity. Budget incidents and agent errors remain tied to their own source changes. Ordinary issue writes do not reconstruct the unrelated company feed. Opaque liveness and stranded-recovery origin references resolve through company-scoped issue rows before entering UUID queries, so valid none sentinels do not roll back writes and moved recovery tasks still invalidate their original review. Queue keys explicitly pick supported identity fields. Retention's own writes use exact source hints; blocker retention changes invalidate the company/source because multiple blocked roots can project one terminal subject. Reconstruction pages and due batches remain bounded. Review attention and interaction invokability are computed in company batches per candidate page; a full 100-candidate review page uses at most 20 statements and 100 confirmations use at most six. Decisions, recovery source links, runs, eligible agents, issue visibility, newer-run probes and blocker graphs also load in company batches per page. A 100-source decision or recovery page uses at most three statements; 100 failed runs use at most six; 100 blocker roots use at most 30 graph/source statements plus retention-state reads for their emitted entries. These evidence maps exist only for one load. The blocker owner shares fetched node and edge facts while tracking depth, node count, expanded nodes and live-work reachability independently for each root. The shared graph fetch stores at most 2,000 nodes and 4,000 edges, and bounds each frontier query before materializing rows. An internal batch result identifies roots whose evaluation is incomplete; the legacy map API exposes no extra transport fields. Retention schedules only those roots for an immediate fresh single-root check, which disarms or saves the actual source deadline. It never archives a partial intermediary or adds a recurring incomplete-root poll. Fully evaluated roots preserve their exact classification. All later graph queries, including recovery source IDs, agent IDs and workspace-finalize inputs, use bounded chunks. A SQL-generated 70,000-node graph proves PostgreSQL parameter limits are respected at the driver boundary. Archive checks still lock and freshly re-read sources, terminal subjects and related eligibility. Ordinary execution, decision and budget mutations pass their affected company; exact decision sources are included. +Committed hints identify a company, source kind, source ID or changed issue where those values are known. Routine issue writers pass identity to the retention domain helper before their mutation. The helper captures old and new parent paths, reverse blocker ancestors, run-linked issues and liveness-origin references; issue-scoped source queries then select reviews, blockers, interactions, decisions, recovery actions and failed runs by their issue identity. Budget incidents and agent errors remain tied to their own source changes. Ordinary issue writes do not reconstruct the unrelated company feed. Opaque liveness and stranded-recovery origin references resolve through company-scoped issue rows before entering UUID queries, so valid none sentinels do not roll back writes and moved recovery tasks still invalidate their original review. Queue keys explicitly pick supported identity fields. Retention's own writes use exact source hints; blocker retention changes invalidate the company/source because multiple blocked roots can project one terminal subject. Reconstruction pages and due batches remain bounded. Review attention and interaction invokability are computed in company batches per candidate page; a full 100-candidate review page uses at most 20 statements and 100 confirmations use at most six. Decisions, recovery source links, runs, eligible agents, issue visibility, newer-run probes and blocker graphs also load in company batches per page. A 100-source decision or recovery page uses at most three statements; 100 failed runs use at most six; 100 blocker roots use at most 30 graph/source statements plus retention-state reads for their emitted entries. These evidence maps exist only for one load. The blocker owner shares fetched node and edge facts while tracking depth, node count, expanded nodes and live-work reachability independently for each root. The shared graph fetch stores at most 2,000 nodes and 4,000 edges, and bounds each frontier query before materializing rows. Positive live-work evidence from the bounded returned rows is retained independently of node/edge admission, including running status, active runs and pending confirmations. Filling the graph with explicit blockers cannot hide a subsequently returned active child. An internal batch result identifies roots whose evaluation is incomplete; the legacy map API exposes no extra transport fields. Retention schedules only those roots for an immediate fresh single-root check, which disarms or saves the actual source deadline. It never archives a partial intermediary or adds a recurring incomplete-root poll. Fully evaluated roots preserve their exact classification. All later graph queries, including recovery source IDs, agent IDs and workspace-finalize inputs, use bounded chunks. A SQL-generated 70,000-node graph proves PostgreSQL parameter limits are respected at the driver boundary. Archive checks still lock and freshly re-read sources, terminal subjects and related eligibility. Ordinary execution, decision and budget mutations pass their affected company; exact decision sources are included. Producer coverage includes decision creation and changes; interaction creation, resolution, expiry and supersession; issue creation, updates, blocker relations, checkout, adoption and release; agent hire, configuration, lifecycle, error clearing and deletion; budget incidents and company pause/resume; exhausted-run events and retry-run creation; native status, review, finalization and reconstruction; external conversation transitions; and imported blocker relations. Existing service transactions retain outer commit ownership. Savepoint release does not publish work before outer commit. diff --git a/server/src/__tests__/attention-retention-work.test.ts b/server/src/__tests__/attention-retention-work.test.ts index 275b4f64570..fb2dca6f6b5 100644 --- a/server/src/__tests__/attention-retention-work.test.ts +++ b/server/src/__tests__/attention-retention-work.test.ts @@ -377,6 +377,37 @@ const delay = DEFAULT_DECISION_ARCHIVE_DAYS * 86_400_000; expect(await db.select().from(decisionArchiveNotificationOutbox)).toHaveLength(0); }); + it.each(["in_progress", "run", "confirmation"])("preserves observed child live evidence (%s) after explicit blockers fill the node budget", async (kind) => { + const root = await insertIssue("blocked"); + await db.execute(sql`with blockers as ( + insert into issues (id, company_id, title, status, assignee_agent_id, updated_at) + select gen_random_uuid(), ${companyId}::uuid, 'explicit-leaf-' || n, 'backlog', ${agentId}::uuid, ${old().toISOString()}::timestamptz + from generate_series(1, 1999) n returning id) + insert into issue_relations (company_id, issue_id, related_issue_id, type) + select ${companyId}::uuid, id, ${root.id}::uuid, 'blocks' from blockers`); + const child = await insertIssue(kind === "in_progress" ? "in_progress" : "todo", { parentId: root.id, assigneeAgentId: agentId }); + if (kind === "run") { + const [run] = await db.insert(heartbeatRuns).values({ companyId, agentId, status: "running" }).returning(); + await db.update(issues).set({ executionRunId: run.id }).where(eq(issues.id, child.id)); + } else if (kind === "confirmation") { + await db.insert(issueThreadInteractions).values({ companyId, issueId: child.id, kind: "request_confirmation", status: "pending", payload: { prompt: "Approve?" } }); + } + parameterCounts.length = 0; + const batch = await issueService(db).listBlockerAttentionBatch(companyId, [root]); + expect(batch.nodeCount).toBeLessThanOrEqual(BLOCKER_ATTENTION_MAX_NODES); + expect(batch.edgeCount).toBeLessThanOrEqual(BLOCKER_ATTENTION_MAX_NODES * 2); + expect(batch.fetchedRows).toBeLessThanOrEqual(BLOCKER_ATTENTION_MAX_NODES * 6); + expect(parameterCounts.length).toBeLessThanOrEqual(100); + expect(Math.max(...parameterCounts)).toBeLessThanOrEqual(1100); + expect(batch.deferredRootIds.has(root.id)).toBe(true); + expect(batch.attention.get(root.id)?.blockingTreeLive).toBe(true); + const service = attentionRetentionService(db), page = await service.load({ companyId, sourceKind: "blocker_attention" }); + const entry = page.rows!.find(item => item.rootId === root.id && item.variant === "terminal")!; + expect(await service.check(entry)).toBeNull(); + expect(await db.select().from(decisionRetention)).toHaveLength(0); + expect(await db.select().from(decisionArchiveNotificationOutbox)).toHaveLength(0); + }); + it("disarms finite deferred-root checks when large retained graphs have no archive obligation", async () => { await db.execute(sql`insert into issues (id, company_id, title, status, updated_at) select gen_random_uuid(), ${companyId}::uuid, 'held-root-' || n, 'blocked', ${old().toISOString()}::timestamptz diff --git a/server/src/services/issues.ts b/server/src/services/issues.ts index 4d5fd5f85f6..b8f78009a59 100644 --- a/server/src/services/issues.ts +++ b/server/src/services/issues.ts @@ -3850,6 +3850,10 @@ async function readIssueBlockerAttentionBatch( const nodesById = new Map(); const edgesByIssueId = new Map(); + // Returned rows are already bounded by the residual fetch budget. Keep their + // positive live-work evidence even when they cannot enter the traversal graph. + const observedNodes = new Map(); + const observedChildrenByParent = new Map>(); const admittedRoots = roots.slice(0, BLOCKER_ATTENTION_MAX_NODES); for (const root of admittedRoots) nodesById.set(root.id, { ...root }); @@ -3946,6 +3950,10 @@ async function readIssueBlockerAttentionBatch( ); for (const row of [...unresolvedExplicitBlockerRows, ...childRows]) { if (!row.issueId) continue; + observedNodes.set(row.blockerIssueId, row); + const observed = observedChildrenByParent.get(row.issueId) ?? new Set(); + observed.add(row.blockerIssueId); + observedChildrenByParent.set(row.issueId, observed); if (!nodesById.has(row.blockerIssueId)) { if (nodesById.size >= BLOCKER_ATTENTION_MAX_NODES) { complete = false; continue; } nodesById.set(row.blockerIssueId, { @@ -3993,11 +4001,12 @@ async function readIssueBlockerAttentionBatch( if (traversal.frontier.length > 0) traversal.truncated = true; } - const nodeIds = [...nodesById.keys()]; + const evidenceNodes = new Map([...nodesById, ...observedNodes]); + const nodeIds = [...evidenceNodes.keys()]; const activeIssueIds = new Set(); const agentIds = new Set(); const issueIdByExecutionRunId = new Map(); - for (const node of nodesById.values()) { + for (const node of evidenceNodes.values()) { if (node.assigneeAgentId) agentIds.add(node.assigneeAgentId); if (node.executionRunId) issueIdByExecutionRunId.set(node.executionRunId, node.id); @@ -4055,7 +4064,7 @@ async function readIssueBlockerAttentionBatch( } } - const explicitWaitCandidateIds = [...nodesById.values()] + const explicitWaitCandidateIds = [...evidenceNodes.values()] .filter((node) => node.status !== "done") .map((node) => node.id); const explicitWaitingIssueIds = new Set(); @@ -4400,19 +4409,22 @@ async function readIssueBlockerAttentionBatch( }; }; + const nodeHasLiveWork = (node: IssueBlockerAttentionNode) => + node.status === "in_progress" || activeIssueIds.has(node.id) || + explicitWaitingIssueIds.has(node.id) || Boolean(node.assigneeUserId); + const observedChildHasLiveWork = (parentId: string) => + [...observedChildrenByParent.get(parentId) ?? []].some(id => { + const node = observedNodes.get(id); + return node?.companyId === companyId && nodeHasLiveWork(node); + }); const pathHasLiveWork = (nodeId: string, seen: Set, traversal: Traversal): boolean => { if (seen.has(nodeId) || !traversal.nodes.has(nodeId)) return false; const node = nodesById.get(nodeId); if (!node || node.companyId !== companyId) return false; - if ( - node.status === "in_progress" || - activeIssueIds.has(node.id) || - explicitWaitingIssueIds.has(node.id) || - Boolean(node.assigneeUserId) - ) - return true; + if (nodeHasLiveWork(node)) return true; if (!traversal.expanded.has(nodeId)) return false; + if (observedChildHasLiveWork(nodeId)) return true; const nextSeen = new Set(seen); nextSeen.add(nodeId); return (edgesByIssueId.get(node.id) ?? []).some((edge) => { @@ -4539,7 +4551,7 @@ async function readIssueBlockerAttentionBatch( stalledEntry?.result.sampleStalledBlockerIdentifier ?? sampleStalledFromChain ?? null, - blockingTreeLive: topLevelEdges.some((edge) => + blockingTreeLive: observedChildHasLiveWork(root.id) || topLevelEdges.some((edge) => pathHasLiveWork(edge.blockerIssueId, new Set([root.id]), traversal), ), directBlockerIssueId: sampleEntry?.edge.blockerIssueId ?? null, From dff6c00468671a088dc307f18132334e855f5d59 Mon Sep 17 00:00:00 2001 From: Dotta Date: Fri, 30 Oct 2026 19:50:27 -0500 Subject: [PATCH 68/70] Join receipt acceptance passes before advancing saved deadlines Co-Authored-By: Paperclip --- .../2026-10-30-deferred-execution-sleep.md | 4 +++ .../deferred-execution-receipt-work.test.ts | 29 +++++++++++++++---- 2 files changed, 28 insertions(+), 5 deletions(-) diff --git a/doc/plans/2026-10-30-deferred-execution-sleep.md b/doc/plans/2026-10-30-deferred-execution-sleep.md index 0478fe79300..56e3611d939 100644 --- a/doc/plans/2026-10-30-deferred-execution-sleep.md +++ b/doc/plans/2026-10-30-deferred-execution-sleep.md @@ -17,3 +17,7 @@ Three scoped owners reconstruct only their actual durable receipt predicates, wi The generic run-dispatch notifier and broad issue review-state notifier deliberately do not fan out deferred eligibility. Queue bookkeeping may write them while processing its own saved input. Separate eligibility commits and settlement-only chat notifications avoid a self-restaging loop. C2 remains separate and mandatory: active output silence deadlines, remaining orphan/process/controller/remote authority and resource debts, legacy shutdown settlements, stranded agent status, and final aggregate chain removal. Retained workspace and migrated native finalization workers remain their existing owners. + +Review controls cover queue edits that move an already loaded grace deadline. Each due check re-reads current stored grace or claim time before admission. Subtree hold release, subtree restore, and member archival publish eligibility inside their own transactions. Member archival preserves the cancelled Retry tombstone created by an earlier user reassignment. + +A resume that first records a new execution-wait reason arms its existing 30-second grace once. Writing the same closed-gate reason again does not publish another hint. A clock obligation, an eligibility change, and an infrastructure failure therefore retain separate exits: stored deadline, owning committed producer, and bounded failure backoff. Refreshing a diagnostic timestamp alone does not create permanent polling behind a deliberate pause. diff --git a/server/src/__tests__/deferred-execution-receipt-work.test.ts b/server/src/__tests__/deferred-execution-receipt-work.test.ts index eb60e2ed449..914f33fe609 100644 --- a/server/src/__tests__/deferred-execution-receipt-work.test.ts +++ b/server/src/__tests__/deferred-execution-receipt-work.test.ts @@ -1,3 +1,4 @@ +import { setImmediate as immediate } from "node:timers/promises"; import { DELIVERY_QUEUES, subscribeDeliveryWork } from "../services/delivery-work-notifications.js"; import { accessService } from "../services/access.js"; import { writeDeferredExecutionWait } from "../services/deferred-execution-wait.js"; @@ -27,6 +28,7 @@ describe("deferred receipt durable dispatch", () => { let coordinator: ReturnType; let drain = false; const errors: unknown[] = []; + const passes = new Set>(); beforeAll(async () => { const external = resolveExternalTestDatabaseUrl("PAPERCLIP_DEFERRED_RECEIPT_TEST_DATABASE_URL"); if (!external) temporary = await startEmbeddedPostgresTestDatabase("deferred-receipts-"); @@ -38,6 +40,11 @@ describe("deferred receipt durable dispatch", () => { heartbeat = heartbeatService(db); coordinator = createDeliveryWorkCoordinator({ owner: db, canRun: () => !drain, canReconcile: () => true, onError: error => errors.push(error) }); + const register = coordinator.register.bind(coordinator); + coordinator.register = (queue, task) => register(queue, { ...task, run: signal => { + const pass = Promise.resolve(task.run(signal)); passes.add(pass); + void pass.then(() => passes.delete(pass), () => passes.delete(pass)); return pass; + } }); }); afterEach(async () => { await coordinator.stop(); await heartbeat.drainActiveRunExecutions(); @@ -70,12 +77,24 @@ describe("deferred receipt durable dispatch", () => { } async function finish(issueId: string) { await issueService(db).update(issueId, { status: "done" }); - await heartbeat.drainActiveRunExecutions(); + await vi.waitFor(async () => { + await Promise.allSettled([...passes]); await heartbeat.drainActiveRunExecutions(); + expect(await db.select().from(heartbeatRuns).where(sql`${heartbeatRuns.status} in ('queued', 'running')`)).toHaveLength(0); + }, { timeout: 15_000 }); const saved = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.status, "scheduled_retry")); - expect(await db.select().from(heartbeatRuns).where(sql`${heartbeatRuns.status} in ('queued', 'running')`)).toHaveLength(0); - vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"] }); - if (saved.length) await vi.advanceTimersByTimeAsync(Math.max(0, Math.max(...saved.map(run => run.scheduledRetryAt!.getTime())) - Date.now()) + 1); - await vi.waitFor(async () => expect(await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.status, "scheduled_retry"))).toHaveLength(0)); + if (!vi.isFakeTimers()) vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"] }); + if (saved.length) { + const due = Math.max(...saved.map(run => run.scheduledRetryAt!.getTime())); + while (Date.now() <= due) { + await vi.advanceTimersByTimeAsync(Math.min(500, due - Date.now() + 1)); + await Promise.allSettled([...passes]); await immediate(); + } + } + await vi.waitFor(async () => { + await vi.advanceTimersByTimeAsync(500); await Promise.allSettled([...passes]); + await heartbeat.drainActiveRunExecutions(); + expect(await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.status, "scheduled_retry"))).toHaveLength(0); + }); } it("is quiet after empty reconstruction and terminal receipt history", async () => { const f = await fixture(); From 255713e0cd0505a471cc9f0d264a3891da31a47c Mon Sep 17 00:00:00 2001 From: Dotta Date: Fri, 30 Oct 2026 20:02:36 -0500 Subject: [PATCH 69/70] fix(server): bound and coalesce issue retention invalidations Bound reverse ancestor discovery before materializing adjacency, deduplicate cycles and emit one canonical affected-issue batch per write. Overflow delegates complete company reconstruction to existing paging instead of dropping eligibility or creating per-ancestor worker debt. Prove real high-fanin producer bursts, PostgreSQL execution limits, canonical cyclic batches and far ancestor archive deadlines. Co-Authored-By: Paperclip --- .../2026-10-30-attention-retention-sleep.md | 2 + .../attention-retention-work.test.ts | 107 +++++++++++++++++- server/src/services/attention-retention.ts | 12 +- .../services/retention-work-notifications.ts | 29 +++-- 4 files changed, 134 insertions(+), 16 deletions(-) diff --git a/doc/plans/2026-10-30-attention-retention-sleep.md b/doc/plans/2026-10-30-attention-retention-sleep.md index 2be011ccf9d..92241aefc4f 100644 --- a/doc/plans/2026-10-30-attention-retention-sleep.md +++ b/doc/plans/2026-10-30-attention-retention-sleep.md @@ -46,3 +46,5 @@ The focused suite supports `PAPERCLIP_ATTENTION_RETENTION_TEST_DATABASE_URL`, va Local embedded PostgreSQL could not start on the test host because system semaphores were exhausted. The accepted runs use isolated PostgreSQL 17 databases. A mixed consumer run also exposed fixture contamination from a lifecycle suite that retains native finalization rows; isolated reruns are required and preserve the failed run in the review evidence. Broad Linux CI remains the required full-suite gate. Review the source parity fixtures and producer hints together. The intended outcome is unchanged archive state and notifications with no recurring retention SQL when the sources remain unchanged. No merge or deployment is part of this change. + +Issue producer invalidation has its own finite discovery budget. Reverse parent/blocker traversal returns at most 101 distinct UUIDs in one recursive query, with each relation adjacency capped at 101 unique edges before materialization. At most 100 affected issues produce one sorted canonical batch hint; an overflow sentinel produces one company hint and the existing complete paged reconstruction. UNION deduplication terminates cycles. A PostgreSQL EXPLAIN ANALYZE on 2,100 reverse edges proves the outer limit stops at 101 CTE rows while the indexed relation branch returns only 100 rows. Repeated identical writes coalesce by canonical key. The shared scheduler is unchanged: a broader company hint does not cancel already pending narrower scopes; those remain finite batches of at most 100 IDs, processed at the existing 25-scope pass limit. This permits redundant bounded reconstruction instead of discarding invalidations or introducing cursor/subsumption policy across other owners. A 151-node cyclic chain proves overflow still registers the far ancestor's exact archive deadline and settles quietly afterward. diff --git a/server/src/__tests__/attention-retention-work.test.ts b/server/src/__tests__/attention-retention-work.test.ts index fb2dca6f6b5..f06355e1ae7 100644 --- a/server/src/__tests__/attention-retention-work.test.ts +++ b/server/src/__tests__/attention-retention-work.test.ts @@ -1,5 +1,6 @@ import { randomUUID } from "node:crypto"; -import { and, eq, sql } from "drizzle-orm"; +import { and, eq, sql, type SQL } from "drizzle-orm"; +import { PgDialect } from "drizzle-orm/pg-core"; import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; import { subscribeDatabaseWork, authUsers, companyMemberships, agents, approvals, budgetIncidents, budgetPolicies, companies, createDb, applyPendingMigrations, decisionArchiveNotificationOutbox, decisionRetention, decisions, costEvents, heartbeatRunEvents, heartbeatRuns, issueApprovals, issueRecoveryActions, @@ -425,6 +426,106 @@ const delay = DEFAULT_DECISION_ARCHIVE_DAYS * 86_400_000; expect(await db.select().from(decisionArchiveNotificationOutbox)).toHaveLength(0); }); + it("bounds high-fanin ordinary issue writes to one coalescible hint", async () => { + const service = attentionRetentionService(db), original = service.load, load = vi.spyOn(service, "load"); + await start(service); await settle(); load.mockClear(); + const leaf = await insertIssue("in_review", { updatedAt: new Date(), assigneeUserId: "board" }); + await db.execute(sql`with parents as ( + insert into issues (id, company_id, title, status) + select gen_random_uuid(), ${companyId}::uuid, 'fan-in-' || n, 'todo' + from generate_series(1, 2100) n returning id) + insert into issue_relations (company_id, issue_id, related_issue_id, type) + select ${companyId}::uuid, ${leaf.id}::uuid, id, 'blocks' from parents`); + const hints: string[] = []; + const unsubscribe = subscribeDatabaseWork(db, DELIVERY_QUEUES.attentionRetention, (event, values) => { + if (event === "settled") hints.push(...values ?? []); + }); + let release!: () => void, entered = false; + const gate = new Promise(resolve => { release = resolve; }); + load.mockImplementationOnce(async (...args) => { const page = await original(...args); entered = true; await gate; return page; }); + statements.length = 0; + try { + await issueService(db).update(leaf.id, { title: "High fan-in ordinary edit 0" }); + await vi.waitFor(() => expect(entered).toBe(true), { timeout: 15_000 }); + for (let n = 1; n < 8; n++) await issueService(db).update(leaf.id, { title: `High fan-in ordinary edit ${n}` }); + } finally { release(); unsubscribe(); } + expect(statements.filter(query => query.includes("with recursive affected"))).toHaveLength(8); + expect(hints).toHaveLength(8); + expect(new Set(hints)).toEqual(new Set([JSON.stringify({ companyId })])); + await settle(); + expect(load.mock.calls.length).toBeLessThanOrEqual(3); + expect(load.mock.calls.every(([scope]) => scope?.companyId === companyId && !scope.issueId && !scope.issueIds)).toBe(true); + let query!: Parameters[0], returned = 0; + await db.transaction(async tx => { + const execute = tx.execute.bind(tx); + vi.spyOn(tx, "execute").mockImplementation(async statement => { + const result = await execute(statement); + if (new PgDialect().sqlToQuery(statement as SQL).sql.includes("with recursive affected")) { + query = statement; returned = result.length; + } + return result; + }); + await notifyIssueAttentionRetentionWork(tx, { companyId, issueId: leaf.id }); + }); + expect(returned).toBe(101); + const plans = await db.execute(sql`explain (analyze, format json) ${query as SQL}`); + const plan = (plans[0]["QUERY PLAN"] as any)[0].Plan; + expect(plan["Node Type"]).toBe("Limit"); + expect(plan["Actual Rows"]).toBe(101); + const nodes: any[] = [], collect = (node: any) => { nodes.push(node); for (const child of node.Plans ?? []) collect(child); }; + collect(plan); + const recursive = nodes.filter(node => node["Node Type"] === "Recursive Union"); + const relations = nodes.filter(node => node["Relation Name"] === "issue_relations"); + expect(recursive).toHaveLength(1); + expect(recursive[0]["Actual Rows"]).toBe(101); + expect(relations.length).toBeGreaterThan(0); + expect(relations.every(node => node["Actual Rows"] * node["Actual Loops"] <= 101)).toBe(true); + process.stdout.write(JSON.stringify({ reverseFanInPlan: plan }) + "\n"); + }); + + it("retains far ancestor deadlines when a cyclic chain exceeds the precise hint budget", async () => { + const service = attentionRetentionService(db), load = vi.spyOn(service, "load"); + await start(service); await settle(); load.mockClear(); + await db.execute(sql`insert into issues (id, company_id, title, status, updated_at, assignee_user_id) + select gen_random_uuid(), ${companyId}::uuid, 'chain-' || n, + case when n = 150 then 'in_review' else 'todo' end, + case when n = 150 then ${new Date(Date.now() - delay + 15_000).toISOString()}::timestamptz else now() end, + case when n = 150 then 'board' else null end + from generate_series(0, 150) n`); + await db.execute(sql`update issues child set parent_id = parent.id from issues parent + where child.company_id = ${companyId}::uuid and parent.company_id = child.company_id + and parent.title = 'chain-' || ((substring(child.title from 7)::integer + 1) % 151)`); + const [leaf, far] = await Promise.all([db.select().from(issues).where(and(eq(issues.companyId, companyId), eq(issues.title, "chain-0"))).then(rows => rows[0]), + db.select().from(issues).where(and(eq(issues.companyId, companyId), eq(issues.title, "chain-150"))).then(rows => rows[0])]); + await issueService(db).update(leaf.id, { title: "Ordinary deep chain edit" }); + await settle(); + expect(load.mock.calls.some(([scope]) => scope?.companyId === companyId && !scope.issueId && !scope.issueIds)).toBe(true); + expect(await state("review", far.id)).toBeNull(); + expect(coordinator!.nextWakeAt()).not.toBeNull(); + await archived("review", far.id); + await quiet(); + }); + + it("canonicalizes a complete cyclic affected set into one bounded batch hint", async () => { + const a = await insertIssue(), b = await insertIssue(), c = await insertIssue(); + await db.update(issues).set({ parentId: b.id }).where(eq(issues.id, a.id)); + await db.update(issues).set({ parentId: c.id }).where(eq(issues.id, b.id)); + await db.update(issues).set({ parentId: a.id }).where(eq(issues.id, c.id)); + const hints: string[] = []; + const unsubscribe = subscribeDatabaseWork(db, DELIVERY_QUEUES.attentionRetention, (event, values) => { + if (event === "settled") hints.push(...values ?? []); + }); + try { + await db.transaction(tx => notifyIssueAttentionRetentionWork(tx, { companyId, issueId: a.id })); + await db.transaction(tx => notifyIssueAttentionRetentionWork(tx, { companyId, issueId: c.id })); + const wide = { companyId, issueIds: [b.id, a.id, c.id, a.id], ignored: {} as unknown }; + wide.ignored = wide; + await db.transaction(tx => notifyAttentionRetentionWork(tx, wide)); + } finally { unsubscribe(); } + expect(hints).toHaveLength(3); + expect(new Set(hints)).toEqual(new Set([JSON.stringify({ companyId, issueIds: [a.id, b.id, c.id].sort() })])); + }); + it("canonicalizes wide circular notification variables to the supported identity fields", async () => { const sourceId = randomUUID(), issueId = randomUUID(), hints: string[] = []; const wide = { companyId, sourceKind: "review" as const, sourceId, issueId, ignored: {} as unknown }; @@ -487,14 +588,14 @@ const delay = DEFAULT_DECISION_ARCHIVE_DAYS * 86_400_000; const terminal = await insertIssue("backlog", { parentId: oldParent.id, hiddenAt: old(), assigneeAgentId: agentId, responsibleUserId: "board" }); const service = attentionRetentionService(db), load = vi.spyOn(service, "load"); await start(service); await settle(); load.mockClear(); await issueService(db).update(terminal.id, { parentId: newParent.id }); await settle(); - const ids = load.mock.calls.map(([scope]) => scope?.issueId); + const ids = load.mock.calls.flatMap(([scope]) => scope?.issueIds ?? (scope?.issueId ? [scope.issueId] : [])); expect(ids).toEqual(expect.arrayContaining([oldParent.id, newParent.id, terminal.id])); load.mockClear(); await issueService(db).update(terminal.id, { hiddenAt: null }); await settle(); expect(await state("blocker_attention", terminal.id)).toBeNull(); const visible = await service.load({ companyId, issueId: newParent.id }); expect(visible.rows!.some(entry => entry.sourceId === terminal.id && entry.dueAt > Date.now())).toBe(true); - expect(load.mock.calls.some(([scope]) => scope?.issueId === newParent.id)).toBe(true); + expect(load.mock.calls.some(([scope]) => scope?.issueId === newParent.id || scope?.issueIds?.includes(newParent.id))).toBe(true); const next = await insertIssue("backlog", { assigneeAgentId: agentId }); await issueService(db).update(oldParent.id, { blockedByIssueIds: [next.id] }); await archived("blocker_attention", next.id); }); diff --git a/server/src/services/attention-retention.ts b/server/src/services/attention-retention.ts index ebe91416f52..a0bca4281f5 100644 --- a/server/src/services/attention-retention.ts +++ b/server/src/services/attention-retention.ts @@ -1,7 +1,6 @@ import { and, eq, getTableColumns, inArray, isNull, sql, type SQL } from "drizzle-orm"; import { agents, decisionRetention, decisions, heartbeatRuns, issueApprovals, issueRecoveryActions, issueThreadInteractions, issues, approvals, type Db } from "@paperclipai/db"; -import type { AttentionSourceKind } from "@paperclipai/shared"; import { evaluateAgentInvokability } from "./agent-invokability.js"; import { HUMAN_RECOVERY_OWNER_TYPES, ATTENTION_BUDGET_WARNING_PERCENT, ATTENTION_AGENT_ERROR_STATUS, collapsePendingConfirmationsToNewest, listFailedRunsWithNewerRun, readRunIssueId, requiresReviewAttention, terminalBlockerRequiresAttention } from "./attention-source-eligibility.js"; import { decisionRetentionService, DEFAULT_DECISION_ARCHIVE_DAYS } from "./decision-retention.js"; @@ -13,12 +12,13 @@ import { issueService } from "./issues.js"; import { parseIssueExecutionState } from "./issue-execution-policy.js"; import { isProspectiveBlockedTransition } from "./routable-blocked.js"; import { registerScopedDeadlineWork } from "./scoped-deadline-work.js"; +import type { AttentionRetentionScope } from "./retention-work-notifications.js"; const archiveDelay = DEFAULT_DECISION_ARCHIVE_DAYS * 86_400_000; const kinds = ["decision", "issue_thread_interaction", "recovery_action", "blocker_attention", "review", "failed_run", "budget_alert", "agent_error_alert"] as const; type Kind = typeof kinds[number]; -type Scope = { companyId?: string; sourceKind?: AttentionSourceKind; sourceId?: string; issueId?: string }; +type Scope = AttentionRetentionScope; type Candidate = { sourceKind: Kind; sourceId: string; companyId: string; activityAt: string; deadlineAt: number; issueId: string | null }; type Entry = Candidate & { id: string; dueAt: number; rootId: string; variant?: "owner" | "terminal" }; type Cursor = { sourceKind: string; sourceId: string }; @@ -66,8 +66,9 @@ const sources: Record = { export function attentionRetentionService(db: Db) { const cursors = new Map(); async function candidates(database: Db, scope: Scope = {}, cursor?: Cursor, sourceId?: string): Promise { + const issueIds = scope.issueIds ?? (scope.issueId ? [scope.issueId] : []); const selected = kinds.filter(kind => (!scope.sourceKind || scope.sourceKind === kind) - && (!scope.issueId || !["budget_alert", "agent_error_alert"].includes(kind))); + && (!issueIds.length || !["budget_alert", "agent_error_alert"].includes(kind))); if (!selected.length) return []; const rows = await database.execute(sql`with source_candidates(source_kind, source_id, company_id, activity_at, issue_id) as (${sql.join(selected.map(kind => sources[kind]), sql` union all `)}) select s.source_kind as "sourceKind", s.source_id as "sourceId", s.company_id as "companyId", @@ -75,7 +76,7 @@ export function attentionRetentionService(db: Db) { from source_candidates s inner join companies c on c.id = s.company_id left join decision_retention r on r.company_id = s.company_id and r.source_kind = s.source_kind and r.source_id = s.source_id where c.status = 'active' - and (${scope.issueId ?? null}::text is null or s.issue_id = ${scope.issueId ?? null}) + ${issueIds.length ? sql`and s.issue_id in (${sql.join(issueIds.map(id => sql`${id}`), sql`, `)})` : sql``} and (${scope.companyId ?? null}::uuid is null or s.company_id = ${scope.companyId ?? null}::uuid) and (${sourceId ?? scope.sourceId ?? null}::text is null or s.source_id = ${sourceId ?? scope.sourceId ?? null}) and (s.source_kind = 'blocker_attention' or ((r.id is null or r.keep = false) and r.archived_at is null)) @@ -312,7 +313,8 @@ export function registerAttentionRetentionWork(coordinator: ReturnType (!scope?.companyId || scope.companyId === entry.companyId) && (!scope?.sourceKind || scope.sourceKind === entry.sourceKind) && (!scope?.sourceId || scope.sourceId === entry.sourceId) - && (!scope?.issueId || scope.issueId === entry.issueId), + && (!scope?.issueId || scope.issueId === entry.issueId) + && (!scope?.issueIds?.length || entry.issueId !== null && scope.issueIds.includes(entry.issueId)), load: service.load, check: service.check, }, canRun); } diff --git a/server/src/services/retention-work-notifications.ts b/server/src/services/retention-work-notifications.ts index ba0379afee1..79cffd624a8 100644 --- a/server/src/services/retention-work-notifications.ts +++ b/server/src/services/retention-work-notifications.ts @@ -5,9 +5,16 @@ import type { Db } from "@paperclipai/db"; import type { AttentionSourceKind } from "@paperclipai/shared"; import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js"; -export async function notifyAttentionRetentionWork(transaction: object, scope: { companyId?: string; sourceKind?: AttentionSourceKind; issueId?: string; sourceId?: string } = {}) { +export const ATTENTION_RETENTION_MAX_ISSUE_IDS = 100; +export type AttentionRetentionScope = { companyId?: string; sourceKind?: AttentionSourceKind; + issueId?: string; issueIds?: string[]; sourceId?: string }; + +export async function notifyAttentionRetentionWork(transaction: object, scope: AttentionRetentionScope = {}) { + const issueIds = [...new Set([...(scope.issueIds ?? []), ...(scope.issueId ? [scope.issueId] : [])])].sort(); await notifyDeliveryWork(transaction, DELIVERY_QUEUES.attentionRetention, JSON.stringify({ - companyId: scope.companyId, sourceKind: scope.sourceKind, sourceId: scope.sourceId, issueId: scope.issueId, + companyId: scope.companyId, sourceKind: scope.sourceKind, sourceId: scope.sourceId, + issueId: issueIds.length === 1 ? issueIds[0] : undefined, + issueIds: issueIds.length > 1 && issueIds.length <= ATTENTION_RETENTION_MAX_ISSUE_IDS ? issueIds : undefined, })); } @@ -47,18 +54,24 @@ export async function notifyIssueAttentionRetentionWork(transaction: object, and id::text in (${sql.join([...new Set(references)].map(id => sql`${id}`), sql`, `)})`); for (const row of trusted) seeds.push(row.id as string); } + // UNION deduplicates visited UUIDs (including cycles). Each relation edge is + // unique by schema, so 101 neighbours guarantee overflow of a 100-ID scope. + // Bound that adjacency before materialization and stream at most 101 unique + // IDs to the caller; an inner UNION DISTINCT would consume the full fan-in. const affected = await database.execute(sql`with recursive affected(id) as ( select unnest(array[${sql.join([...new Set(seeds)].map(id => sql`${id}::uuid`), sql`, `)}]) union select parent.id from affected current cross join lateral ( - select related_issue_id as id from issue_relations where company_id = ${scope.companyId}::uuid - and issue_id = current.id and type = 'blocks' - union select parent_id from issues where company_id = ${scope.companyId}::uuid + (select related_issue_id as id from issue_relations where company_id = ${scope.companyId}::uuid + and issue_id = current.id and type = 'blocks' limit ${ATTENTION_RETENTION_MAX_ISSUE_IDS + 1}) + union all select parent_id from issues where company_id = ${scope.companyId}::uuid and id = current.id and parent_id is not null ) parent - ) select id::text as "issueId" from affected`); - for (const row of affected) await notifyAttentionRetentionWork(transaction, { - companyId: scope.companyId, issueId: row.issueId as string, + ) select id::text as "issueId" from affected limit ${ATTENTION_RETENTION_MAX_ISSUE_IDS + 1}`); + // The sentinel means the affected set is incomplete. Reconstruct the whole + // company through the existing paged authority instead of dropping ancestors. + await notifyAttentionRetentionWork(transaction, { companyId: scope.companyId, + issueIds: affected.length <= ATTENTION_RETENTION_MAX_ISSUE_IDS ? affected.map(row => row.issueId as string) : undefined, }); } From 8f58be556a566235779319cb664aa085d8afee1f Mon Sep 17 00:00:00 2001 From: Dotta Date: Fri, 30 Oct 2026 20:18:34 -0500 Subject: [PATCH 70/70] fix(ui): retain startup autosave through session recovery Co-Authored-By: Paperclip --- tests/e2e/disconnect-write-recovery.spec.ts | 23 +++++++- ui/src/hooks/useAccountBoundMutation.ts | 60 +-------------------- ui/src/hooks/useDurableAutosave.test.tsx | 45 ++++++++++++++++ ui/src/hooks/useDurableAutosave.ts | 57 +++++++++++++------- ui/src/lib/account-scope.ts | 59 ++++++++++++++++++++ 5 files changed, 167 insertions(+), 77 deletions(-) create mode 100644 ui/src/lib/account-scope.ts diff --git a/tests/e2e/disconnect-write-recovery.spec.ts b/tests/e2e/disconnect-write-recovery.spec.ts index 8f81ee178dc..780f6f27a04 100644 --- a/tests/e2e/disconnect-write-recovery.spec.ts +++ b/tests/e2e/disconnect-write-recovery.spec.ts @@ -178,10 +178,19 @@ test("replaying an accepted comment does not reopen completed work", async ({ pa } }); -test("startup work retries a failed session lookup while other APIs stay healthy", async ({ page, request }) => { +for (const delayWorkspacePreference of [false, true]) test(delayWorkspacePreference + ? "startup title survives session recovery while workspace preferences remount the editor" + : "startup work retries a failed session lookup while other APIs stay healthy", async ({ page, request }) => { const company = await json(await request.post("/api/companies", { data: { name: `Session recovery ${Date.now()}` } })); const session = await json(await request.get("/api/auth/get-session")); let healthy = false, saving = false, failedAfterSave = 0; + let releasePreference!: () => void; + const preferenceGate = new Promise(resolve => { releasePreference = resolve; }); + let sawPreference!: () => void; + const preferenceRequested = new Promise(resolve => { sawPreference = resolve; }); + if (delayWorkspacePreference) await page.route("**/api/workspace-preferences/me", async route => { + sawPreference(); await preferenceGate; await route.continue(); + }); try { const issue = await json(await request.post(`/api/companies/${company.id}/issues`, { data: { title: "Before session recovery", status: "backlog" }, @@ -199,12 +208,24 @@ test("startup work retries a failed session lookup while other APIs stay healthy saving = true; await header.locator("textarea").press("Enter"); await expect.poll(() => failedAfterSave).toBeGreaterThan(0); + if (delayWorkspacePreference) { + await expect(page.getByText("Couldn't update the task", { exact: true })).toBeVisible(); + await page.evaluate(() => new Promise(resolve => requestAnimationFrame(() => resolve(null)))); + } healthy = true; + if (delayWorkspacePreference) { + // Returning to the tab retries session lookup while the saved edit waits. + await page.evaluate(() => window.dispatchEvent(new Event("visibilitychange"))); + await preferenceRequested; + await expect(page.getByTestId("task-chat-history-content")).toBeHidden(); + releasePreference(); + } await expect.poll(async () => (await json(await request.get(`/api/issues/${issue.id}`))).title, { timeout: 15_000 }).toBe("Saved after session recovery"); await page.reload(); await expect(header.getByRole("heading", { name: "Saved after session recovery", exact: true })).toBeVisible(); } finally { + releasePreference(); await request.patch(`/api/companies/${company.id}`, { data: { status: "archived" } }); } }); diff --git a/ui/src/hooks/useAccountBoundMutation.ts b/ui/src/hooks/useAccountBoundMutation.ts index b011b9ec8ad..5cbf156b703 100644 --- a/ui/src/hooks/useAccountBoundMutation.ts +++ b/ui/src/hooks/useAccountBoundMutation.ts @@ -5,68 +5,12 @@ import { QueryClientContext, type DefaultError, type MutateOptions, - type QueryClient, type UseMutationOptions, type UseMutationResult, type MutationFunctionContext, } from "@tanstack/react-query"; -import { queryKeys } from "@/lib/queryKeys"; -import { authApi } from "@/api/auth"; -import { classifySendFailure, PendingSendAccountChangedError } from "@/lib/pending-send"; - -type AccountGeneration = { owner: string | undefined; generation: number }; -const accounts = new WeakMap(); - -function currentOwner(client: QueryClient): string | undefined { - const session = client.getQueryData<{ user?: { id?: string }; session?: { userId?: string } } | null>(queryKeys.auth.session); - return session === undefined ? undefined : session?.user?.id ?? session?.session?.userId ?? "signed-out"; -} - -function bindAccount(client: QueryClient) { - let account = accounts.get(client); - if (!account) { - account = { owner: currentOwner(client), generation: 0 }; - accounts.set(client, account); - const tracked = account; - client.getQueryCache().subscribe(event => { - const owner = currentOwner(client); - const removed = event.type === "removed" && JSON.stringify(event.query.queryKey) === JSON.stringify(queryKeys.auth.session); - if (owner !== tracked.owner || removed) { - if (removed || tracked.owner !== undefined || tracked.generation !== 0) tracked.generation += 1; - tracked.owner = owner; - } - }); - } - const { owner, generation } = account; - const isCurrentScope = () => account.generation === generation && (owner === undefined ? generation === 0 : account.owner === owner); - const isCurrentAccount = () => account.owner !== undefined && isCurrentScope(); - const whenAccountReady = async () => { - if (account.owner !== undefined || !isCurrentScope()) return; - let stopped = false; - let unsubscribe = () => {}; - const ready = new Promise(resolve => { - unsubscribe = client.getQueryCache().subscribe(() => { - if (account.owner !== undefined || !isCurrentScope()) resolve(); - }); - }); - const lookup = async () => { - for (let attempt = 0; attempt < 3; attempt += 1) { - if (stopped || account.owner !== undefined || !isCurrentScope()) return; - try { - await client.fetchQuery({ queryKey: queryKeys.auth.session, queryFn: () => authApi.getSession(), retry: false }); - return; - } catch (error) { - if (stopped || account.owner !== undefined || !isCurrentScope()) return; - if (attempt === 2 || classifySendFailure(error) !== "pending") throw error; - await new Promise(resolve => setTimeout(resolve, 250 * 2 ** attempt)); - } - } - }; - try { await Promise.race([ready, lookup()]); } - finally { stopped = true; unsubscribe(); } - }; - return { isCurrentAccount, isCurrentScope, whenAccountReady }; -} +import { PendingSendAccountChangedError } from "@/lib/pending-send"; +import { bindAccount } from "@/lib/account-scope"; export function useCurrentAccountGuard(): () => boolean { const client = useContext(QueryClientContext); diff --git a/ui/src/hooks/useDurableAutosave.test.tsx b/ui/src/hooks/useDurableAutosave.test.tsx index edad11f04a0..af5860e0a85 100644 --- a/ui/src/hooks/useDurableAutosave.test.tsx +++ b/ui/src/hooks/useDurableAutosave.test.tsx @@ -173,6 +173,51 @@ describe("useDurableAutosave", () => { client.clear(); }); + it.each([false, true])("keeps an unowned startup edit when identity and unmount race (identityFirst=%s)", async identityFirst => { + store.setBrowserOnline(false); + const client = new QueryClient({ defaultOptions: { queries: { staleTime: Infinity, retry: false, enabled: false } } }); + const save = vi.fn().mockResolvedValue(undefined); + act(() => root.render( + + )); + await act(async () => { await current.run("Retained startup title"); }); + expect(current.status).toBe("waiting"); + act(() => { + if (identityFirst) client.setQueryData(queryKeys.auth.session, { user: { id: "user-1" } }); + root.unmount(); root = createRoot(container); + if (!identityFirst) client.setQueryData(queryKeys.auth.session, { user: { id: "user-1" } }); + }); + await act(async () => { store.setBrowserOnline(true); store.reportSuccess(); }); + await advance(2_000); + expect(save).toHaveBeenCalledExactlyOnceWith("Retained startup title"); + expect(store.getSnapshot().pendingWrites).toBe(0); + client.clear(); + }); + + it.each(["different-account", "round-trip", "removed-session", "signed-out"])( + "does not adopt a later identity for a detached startup edit (%s)", async transition => { + store.setBrowserOnline(false); + const client = new QueryClient({ defaultOptions: { queries: { staleTime: Infinity, retry: false, enabled: false } } }); + const save = vi.fn().mockResolvedValue(undefined); + act(() => root.render( + + )); + await act(async () => { await current.run("Private startup title"); }); + act(() => { root.unmount(); root = createRoot(container); }); + act(() => { + if (transition === "removed-session") client.removeQueries({ queryKey: queryKeys.auth.session }); + else client.setQueryData(queryKeys.auth.session, transition === "signed-out" ? null : { user: { id: "user-1" } }); + if (transition === "round-trip") client.setQueryData(queryKeys.auth.session, { user: { id: "user-2" } }); + client.setQueryData(queryKeys.auth.session, { user: { id: transition === "different-account" ? "user-2" : "user-1" } }); + }); + await act(async () => { store.setBrowserOnline(true); store.reportSuccess(); }); + await advance(2_000); + expect(save).not.toHaveBeenCalled(); + expect(store.getSnapshot().pendingWrites).toBe(0); + client.clear(); + }, + ); + it("bounds retries after an editor unmounts", async () => { const save = vi.fn().mockRejectedValue(outage()); render({ save }); diff --git a/ui/src/hooks/useDurableAutosave.ts b/ui/src/hooks/useDurableAutosave.ts index 658d68cd149..0293975cc8b 100644 --- a/ui/src/hooks/useDurableAutosave.ts +++ b/ui/src/hooks/useDurableAutosave.ts @@ -1,4 +1,6 @@ -import { useCallback, useEffect, useId, useRef, useState } from "react"; +import { useCallback, useContext, useEffect, useId, useRef, useState } from "react"; +import { QueryClientContext } from "@tanstack/react-query"; +import { bindAccount } from "@/lib/account-scope"; import { classifyError, describeError, type ErrorDescription } from "@/api/errors"; import { loadStructuredDraft, saveStructuredDraft } from "@/lib/composer-draft"; import { useConnectivityStore } from "@/lib/connectivity"; @@ -108,6 +110,7 @@ const SAVED_LINGER_MS = 1600; * - Errors become readable copy for "Couldn't save (reason)". */ export function useDurableAutosave(options: DurableAutosaveOptions): DurableAutosave { + const client = useContext(QueryClientContext); const owner = useDraftOwner(); const readOwner = useDraftOwnerReader(); const instanceId = useId(); @@ -120,7 +123,7 @@ export function useDurableAutosave(options: DurableAutosaveOptions): Durab const targetKey = options.targetKey ?? options.sourceId; const targetRef = useRef(targetKey); targetRef.current = targetKey; - type Job = { token: symbol; registryKey: string; resends: number; saving?: boolean; id: number; target: string; owner: string | undefined; draft: T; + type Job = { token: symbol; registryKey: string; resends: number; saving?: boolean; id: number; target: string; owner: string | undefined; account: ReturnType | null; draft: T; save: (draft: T) => Promise; matchesServer?: DurableAutosaveOptions["matchesServer"]; onRetryFailed?: DurableAutosaveJob["onRetryFailed"]; storageKey: string | null; action: string }; const jobs = useRef(new Map()); @@ -142,6 +145,22 @@ export function useDurableAutosave(options: DurableAutosaveOptions): Durab entry?.active.delete(job.token); if (entry && !entry.active.size) registry.delete(job.registryKey); }; + // Resolve first identity from the same generation guard used by mutations. + // This also runs after unmount, when React cannot migrate the job for us. + function owned(job: Job) { + if (job.account && !job.account.isCurrentScope()) return false; + const current = readOwner(); + if (job.owner === undefined && current !== undefined) { + const nextKey = JSON.stringify([current, job.target]); + if (registry.get(job.registryKey)?.latest !== job.token || registry.has(nextKey)) return false; + releaseJob(job); + job.owner = current; + job.registryKey = nextKey; + registry.set(nextKey, { latest: job.token, active: new Set([job.token]) }); + } + return current === job.owner && generations.current.get(job.target) === job.id + && registry.get(job.registryKey)?.latest === job.token; + } const mounted = useRef(true); const savingTimer = useRef | null>(null); const savedTimer = useRef | null>(null); @@ -160,17 +179,11 @@ export function useDurableAutosave(options: DurableAutosaveOptions): Durab if (previous === undefined && owner !== undefined) { for (const job of jobs.current.values()) { if (job.owner !== undefined) continue; - const nextKey = JSON.stringify([owner, job.target]); - const superseded = registry.get(job.registryKey)?.latest !== job.token || registry.has(nextKey); - releaseJob(job); - if (superseded) { + if (!owned(job)) { + releaseJob(job); jobs.current.delete(job.target); if (generations.current.get(job.target) === job.id) generations.current.delete(job.target); - continue; } - job.owner = owner; - job.registryKey = nextKey; - registry.set(nextKey, { latest: job.token, active: new Set([job.token]) }); } refresh(value => value + 1); return; @@ -194,12 +207,12 @@ export function useDurableAutosave(options: DurableAutosaveOptions): Durab return stored?.version === 1 && stored.draft !== undefined ? stored.draft : null; }, [storageKey]); const execute = useCallback(async (job: Job, retry = false): Promise => { - const owned = () => readOwner() === job.owner && generations.current.get(job.target) === job.id && registry.get(job.registryKey)?.latest === job.token; - const visible = () => mounted.current && owned() && targetRef.current === job.target; + const stillOwned = () => owned(job); + const visible = () => mounted.current && stillOwned() && targetRef.current === job.target; const publish = (status: DurableAutosaveStatus, error: ErrorDescription | null = null) => { if (visible()) setView({ target: job.target, status, error }); }; - if (!owned()) { + if (!stillOwned()) { if (jobs.current.get(job.target) === job) jobs.current.delete(job.target); releaseJob(job); if (mounted.current) refresh(value => value + 1); @@ -234,7 +247,12 @@ export function useDurableAutosave(options: DurableAutosaveOptions): Durab remove(); return { kind: "saved" }; }; - try { await job.save(job.draft); return done(); } + try { + if (!mounted.current && job.account && !job.account.isCurrentAccount()) await job.account.whenAccountReady(); + if (!stillOwned()) { remove(); return { kind: "waiting" }; } + await job.save(job.draft); + return done(); + } catch (initialCause) { let cause = initialCause; if (classifyError(cause) === "conflict" && job.matchesServer) { @@ -255,7 +273,7 @@ export function useDurableAutosave(options: DurableAutosaveOptions): Durab const current = optionsRef.current; const target = current.targetKey ?? current.sourceId; const admittedOwner = ownerRef.current ?? readOwner(); - const job: Job = { token: Symbol(), registryKey: JSON.stringify([admittedOwner, target]), resends: 0, id: ++sequence.current, target, owner: admittedOwner, draft, + const job: Job = { token: Symbol(), registryKey: JSON.stringify([admittedOwner, target]), resends: 0, id: ++sequence.current, target, owner: admittedOwner, account: client ? bindAccount(client) : null, draft, save: overrides.save ?? current.save, matchesServer: overrides.matchesServer ?? current.matchesServer, onRetryFailed: overrides.onRetryFailed, storageKey: current.storageKey ?? null, action: current.action ?? "save" }; const previous = jobs.current.get(target); @@ -266,7 +284,7 @@ export function useDurableAutosave(options: DurableAutosaveOptions): Durab generations.current.set(target, job.id); jobs.current.set(target, job); refresh(value => value + 1); return execute(job); - }, [execute, readOwner]); + }, [client, execute, readOwner]); const waitingJobs = [...jobs.current.values()].filter(job => !job.saving).length; const auto = useAutoResend({ sourceId: options.sourceId, active: waitingJobs > 0, pendingCount: waitingJobs, resend: async (): Promise => { @@ -281,7 +299,6 @@ export function useDurableAutosave(options: DurableAutosaveOptions): Durab detachedWork.current = () => { if (!jobs.current.size) return; if (auto.stalled) { for (const job of jobs.current.values()) releaseJob(job); jobs.current.clear(); return; } - const admittedOwner = ownerRef.current; const source = `${optionsRef.current.sourceId}:detached:${instanceId}`; let timer: ReturnType | null = null; let disposed = false, running = false; @@ -298,7 +315,11 @@ export function useDurableAutosave(options: DurableAutosaveOptions): Durab }; const wake = async () => { if (disposed || running) return; - if (mounted.current || readOwner() !== admittedOwner || admittedOwner === undefined) { release(); return; } + if (mounted.current) { release(); return; } + for (const job of jobs.current.values()) if (!owned(job)) { + releaseJob(job); + jobs.current.delete(job.target); + } if (!jobs.current.size) { release(); return; } store.setPendingWrites(source, jobs.current.size); if (store.getSnapshot().status !== "online") return; diff --git a/ui/src/lib/account-scope.ts b/ui/src/lib/account-scope.ts new file mode 100644 index 00000000000..0e7d050e502 --- /dev/null +++ b/ui/src/lib/account-scope.ts @@ -0,0 +1,59 @@ +import type { QueryClient } from "@tanstack/react-query"; +import { queryKeys } from "./queryKeys"; +import { authApi } from "@/api/auth"; +import { classifySendFailure } from "./pending-send"; + +type AccountGeneration = { owner: string | undefined; generation: number }; +const accounts = new WeakMap(); + +function currentOwner(client: QueryClient): string | undefined { + const session = client.getQueryData<{ user?: { id?: string }; session?: { userId?: string } } | null>(queryKeys.auth.session); + return session === undefined ? undefined : session?.user?.id ?? session?.session?.userId ?? "signed-out"; +} + +export function bindAccount(client: QueryClient) { + let account = accounts.get(client); + if (!account) { + account = { owner: currentOwner(client), generation: 0 }; + accounts.set(client, account); + const tracked = account; + client.getQueryCache().subscribe(event => { + const owner = currentOwner(client); + const removed = event.type === "removed" && JSON.stringify(event.query.queryKey) === JSON.stringify(queryKeys.auth.session); + if (owner !== tracked.owner || removed) { + if (removed || tracked.owner !== undefined || tracked.generation !== 0) tracked.generation += 1; + tracked.owner = owner; + } + }); + } + const { owner, generation } = account; + const isCurrentScope = () => account.generation === generation && (owner === undefined ? generation === 0 : account.owner === owner); + const isCurrentAccount = () => account.owner !== undefined && isCurrentScope(); + const whenAccountReady = async () => { + if (account.owner !== undefined || !isCurrentScope()) return; + let stopped = false; + let unsubscribe = () => {}; + const ready = new Promise(resolve => { + unsubscribe = client.getQueryCache().subscribe(() => { + if (account.owner !== undefined || !isCurrentScope()) resolve(); + }); + }); + const lookup = async () => { + for (let attempt = 0; attempt < 3; attempt += 1) { + if (stopped || account.owner !== undefined || !isCurrentScope()) return; + try { + await client.fetchQuery({ queryKey: queryKeys.auth.session, queryFn: () => authApi.getSession(), retry: false }); + return; + } catch (error) { + if (stopped || account.owner !== undefined || !isCurrentScope()) return; + if (attempt === 2 || classifySendFailure(error) !== "pending") throw error; + await new Promise(resolve => setTimeout(resolve, 250 * 2 ** attempt)); + } + } + }; + try { await Promise.race([ready, lookup()]); } + finally { stopped = true; unsubscribe(); } + }; + return { isCurrentAccount, isCurrentScope, whenAccountReady }; +} +