From acaa86e88c0a93d39e98e685efe1158919aa25dd Mon Sep 17 00:00:00 2001 From: Dotta Date: Fri, 30 Oct 2026 16:10:20 -0500 Subject: [PATCH 1/6] fix: release settled deadline work holds during drain Preserve dirty generations with an in-memory resume deadline while allowing settled transaction holds to end during drain. Keep unresolved database intent fenced by the coordinator. Co-Authored-By: Paperclip --- server/src/__tests__/deadline-work.test.ts | 85 +++++++++++++++++++++- server/src/services/deadline-work.ts | 13 ++-- 2 files changed, 92 insertions(+), 6 deletions(-) diff --git a/server/src/__tests__/deadline-work.test.ts b/server/src/__tests__/deadline-work.test.ts index 7b852977345..7657a7e9d99 100644 --- a/server/src/__tests__/deadline-work.test.ts +++ b/server/src/__tests__/deadline-work.test.ts @@ -1,4 +1,5 @@ -import { installDatabaseWorkSignals } from "../../../packages/db/src/work-signals.js"; +import { idleWorkSnapshot } from "../services/task-admission.js"; +import { installDatabaseWorkSignals, signalDatabaseWork, databaseWorkPending } from "../../../packages/db/src/work-signals.js"; import { afterEach, expect, it, vi } from "vitest"; import { createDeliveryWorkCoordinator } from "../services/delivery-work-coordinator.js"; import { registerDeadlineWork } from "../services/deadline-work.js"; @@ -39,3 +40,85 @@ it.each([ expect(run).toHaveBeenCalledTimes(2); expect(onError).not.toHaveBeenCalled(); }); + +it("releases a settled first-write hold during drain and retains a probe-free resume wake", async () => { + vi.useFakeTimers(); + let draining = false; + const owner = installDatabaseWorkSignals({ + execute: vi.fn(async () => [{ status: "committed" }]), + transaction: async (callback: (tx: object) => Promise) => callback({ execute: async () => [{ xid: "1" }] }), + }); + const run = vi.fn(async () => {}), next = vi.fn(async () => null); + coordinator = createDeliveryWorkCoordinator({ owner, canRun: () => !draining, canReconcile: () => true, onError: vi.fn() }); + const worker = registerDeadlineWork(coordinator, DELIVERY_QUEUES.agentTimers, { run, next, canRun: () => !draining, retryMs: 30_000 }); + await worker.ready; + draining = true; + const before = idleWorkSnapshot().active; + await owner.transaction(async tx => { await signalDatabaseWork(tx, DELIVERY_QUEUES.agentTimers); }); + await vi.advanceTimersByTimeAsync(0); + expect(idleWorkSnapshot().active).toBe(before); + expect(coordinator.nextWakeAt()).not.toBeNull(); + await vi.advanceTimersByTimeAsync(3 * 30_000); + expect(run).toHaveBeenCalledOnce(); expect(next).toHaveBeenCalledOnce(); expect(owner.execute).not.toHaveBeenCalled(); + draining = false; + await vi.advanceTimersByTimeAsync(30_000); + expect(run).toHaveBeenCalledTimes(2); expect(coordinator.nextWakeAt()).toBeNull(); +}); + +it("retains failed work through drain and preserves a future domain deadline", async () => { + vi.useFakeTimers(); + let draining = false; + const owner = installDatabaseWorkSignals({ execute: vi.fn(async () => []) }); + const run = vi.fn(async () => { if (run.mock.calls.length === 1) { draining = true; throw new Error("retry debt"); } }); + const deadline = Date.now() + 600_000; + const next = vi.fn(async () => deadline); + const onError = vi.fn(); + coordinator = createDeliveryWorkCoordinator({ owner, canRun: () => !draining, canReconcile: () => true, onError }); + const worker = registerDeadlineWork(coordinator, DELIVERY_QUEUES.agentTimers, { run, next, canRun: () => !draining, retryMs: 30_000 }); + await worker.ready; + await vi.advanceTimersByTimeAsync(90_000); + expect(run).toHaveBeenCalledOnce(); expect(next).not.toHaveBeenCalled(); expect(owner.execute).not.toHaveBeenCalled(); + draining = false; await vi.advanceTimersByTimeAsync(30_000); + expect(run).toHaveBeenCalledTimes(2); expect(coordinator.nextWakeAt()).toBe(deadline); + draining = true; await vi.advanceTimersByTimeAsync(30_000); + expect(coordinator.nextWakeAt()).toBe(deadline); expect(onError).toHaveBeenCalledOnce(); +}); + +it("does no domain probes in warm standby and reconstructs after admission opens", async () => { + vi.useFakeTimers(); + let standby = true; + const owner = installDatabaseWorkSignals({ execute: vi.fn(async () => []) }); + const run = vi.fn(async () => {}), next = vi.fn(async () => null); + coordinator = createDeliveryWorkCoordinator({ owner, canRun: () => !standby, canReconcile: () => !standby, onError: vi.fn() }); + const worker = registerDeadlineWork(coordinator, DELIVERY_QUEUES.agentTimers, { run, next, canRun: () => !standby, retryMs: 30_000 }); + await worker.ready; await vi.advanceTimersByTimeAsync(90_000); + expect(run).not.toHaveBeenCalled(); expect(next).not.toHaveBeenCalled(); expect(owner.execute).not.toHaveBeenCalled(); + standby = false; await vi.advanceTimersByTimeAsync(30_000); + expect(run).toHaveBeenCalledOnce(); expect(coordinator.nextWakeAt()).toBeNull(); +}); + +it("does not release an ambiguous database intent while dispatch is drained", async () => { + vi.useFakeTimers(); + let draining = false, committed = false; + const owner = installDatabaseWorkSignals({ + execute: vi.fn(async () => [{ status: committed ? "committed" : "in progress" }]), + transaction: async (callback: (tx: object) => Promise) => { + await callback({ execute: async () => [{ xid: "2" }] }); + throw new Error("lost acknowledgement"); + }, + }); + coordinator = createDeliveryWorkCoordinator({ owner, canRun: () => !draining, canReconcile: () => true, onError: vi.fn() }); + const run = vi.fn(async () => {}); + const worker = registerDeadlineWork(coordinator, DELIVERY_QUEUES.agentTimers, { run, next: async () => null, canRun: () => !draining, retryMs: 30_000 }); + await worker.ready; draining = true; + const before = idleWorkSnapshot().active; + await expect(owner.transaction(async tx => { await signalDatabaseWork(tx, DELIVERY_QUEUES.agentTimers); })).rejects.toThrow("lost acknowledgement"); + await vi.advanceTimersByTimeAsync(0); + expect(idleWorkSnapshot().active).toBe(before + 1); + expect(databaseWorkPending(owner, DELIVERY_QUEUES.agentTimers)).toBe(true); + committed = true; await vi.advanceTimersByTimeAsync(30_000); + expect(databaseWorkPending(owner, DELIVERY_QUEUES.agentTimers)).toBe(false); + expect(idleWorkSnapshot().active).toBe(before); + draining = false; await vi.advanceTimersByTimeAsync(30_000); + expect(run).toHaveBeenCalledTimes(2); expect(coordinator.nextWakeAt()).toBeNull(); +}); diff --git a/server/src/services/deadline-work.ts b/server/src/services/deadline-work.ts index 306f340d9fc..15ba76abdae 100644 --- a/server/src/services/deadline-work.ts +++ b/server/src/services/deadline-work.ts @@ -26,7 +26,7 @@ export function registerDeadlineWork( pausePassiveMaintenance: true, onSettled: hints => { requested++; task.onSettled?.(hints); }, run: async signal => { - if (!task.canRun()) { requested++; next = null; return; } + if (!task.canRun()) { requested++; return; } if (Date.now() < earliest) { next = earliest; return; } if (Date.now() < coalesceUntil && (next === null || next > Date.now())) { next = next === null ? coalesceUntil : Math.min(next, coalesceUntil); @@ -39,8 +39,6 @@ export function registerDeadlineWork( if (signal.aborted || !task.canRun()) { next = null; return; } const deadline = await task.next(); - - if (!task.canRun()) { next = null; return; } earliest = deadline === null || deadline > Date.now() ? (task.minimumSpacingMs ? Date.now() + task.minimumSpacingMs : 0) @@ -49,8 +47,13 @@ export function registerDeadlineWork( coalesceUntil = Date.now() + (task.notificationCoalesceMs ?? 0); reconciled = generation; }, - hasPending: async () => requested !== reconciled, - nextRunAt: () => next, + // A settled producer must release its admission hold during drain. Keep + // the unreconciled generation and an in-memory resume wake, without asking + // the domain to probe its database while dispatch is closed. + hasPending: async () => task.canRun() && requested !== reconciled, + nextRunAt: () => !task.canRun() && requested !== reconciled + ? Math.min(next ?? Infinity, Date.now() + task.retryMs) + : next, }); return { ...worker, wake: () => { requested++; worker.wake(); } }; } From dbd400ce17d85b88c4b51ff9860e779dc169555b Mon Sep 17 00:00:00 2001 From: Dotta Date: Fri, 30 Oct 2026 17:24:55 -0500 Subject: [PATCH 2/6] fix: back off expired deadlines during domain drain Keep a future in-memory resume wake when domain dispatch is paused independently of coordinator admission. Preserve the saved obligation without rescheduling its expired deadline into an immediate skipped-attempt loop. Co-Authored-By: Paperclip --- server/src/__tests__/deadline-work.test.ts | 47 ++++++++++++++++++++++ server/src/services/deadline-work.ts | 2 +- 2 files changed, 48 insertions(+), 1 deletion(-) diff --git a/server/src/__tests__/deadline-work.test.ts b/server/src/__tests__/deadline-work.test.ts index 7657a7e9d99..c00daeb2f00 100644 --- a/server/src/__tests__/deadline-work.test.ts +++ b/server/src/__tests__/deadline-work.test.ts @@ -122,3 +122,50 @@ it("does not release an ambiguous database intent while dispatch is drained", as draining = false; await vi.advanceTimersByTimeAsync(30_000); expect(run).toHaveBeenCalledTimes(2); expect(coordinator.nextWakeAt()).toBeNull(); }); + +it("backs off an expired deadline when only the domain is drained and resumes promptly", async () => { + vi.useFakeTimers(); + let paused = false; + const owner = installDatabaseWorkSignals({ execute: vi.fn(async () => []) }); + const run = vi.fn(async () => {}); + const domainCanRun = vi.fn(() => !paused); + const deadline = Date.now() + 1_000; + const next = vi.fn(async () => run.mock.calls.length === 1 ? deadline : null); + coordinator = createDeliveryWorkCoordinator({ owner, canRun: () => true, onError: vi.fn() }); + const worker = registerDeadlineWork(coordinator, DELIVERY_QUEUES.agentTimers, { + run, next, canRun: domainCanRun, retryMs: 30_000, + }); + await worker.ready; + paused = true; + const before = idleWorkSnapshot(); + await vi.advanceTimersByTimeAsync(1_000); + expect(coordinator.nextWakeAt()).toBe(Date.now() + 30_000); + const attempts = domainCanRun.mock.calls.length; + await vi.advanceTimersByTimeAsync(90_000); + expect(domainCanRun.mock.calls.length - attempts).toBeLessThanOrEqual(12); + // Four attempts (expiry plus three retries), each begins and ends tracked work. + expect(idleWorkSnapshot().generation - before.generation).toBe(8); + expect(idleWorkSnapshot().active).toBe(before.active); + expect(run).toHaveBeenCalledOnce(); expect(next).toHaveBeenCalledOnce(); expect(owner.execute).not.toHaveBeenCalled(); + paused = false; + await vi.advanceTimersByTimeAsync(30_000); + expect(run).toHaveBeenCalledTimes(2); expect(coordinator.nextWakeAt()).toBeNull(); +}); + +it("retains a prompt paused wake when a new request preempts a far-future deadline", async () => { + vi.useFakeTimers(); + let paused = false; + const owner = installDatabaseWorkSignals({ execute: vi.fn(async () => []) }); + const run = vi.fn(async () => {}); + const deadline = Date.now() + 86_400_000; + coordinator = createDeliveryWorkCoordinator({ owner, canRun: () => true, onError: vi.fn() }); + const worker = registerDeadlineWork(coordinator, DELIVERY_QUEUES.agentTimers, { + run, next: async () => run.mock.calls.length === 1 ? deadline : null, canRun: () => !paused, retryMs: 30_000, + }); + await worker.ready; paused = true; worker.wake(); + await vi.advanceTimersByTimeAsync(0); + expect(coordinator.nextWakeAt()).toBe(Date.now() + 30_000); + expect(owner.execute).not.toHaveBeenCalled(); + paused = false; await vi.advanceTimersByTimeAsync(30_000); + expect(run).toHaveBeenCalledTimes(2); expect(coordinator.nextWakeAt()).toBeNull(); +}); diff --git a/server/src/services/deadline-work.ts b/server/src/services/deadline-work.ts index 15ba76abdae..208b208e051 100644 --- a/server/src/services/deadline-work.ts +++ b/server/src/services/deadline-work.ts @@ -52,7 +52,7 @@ export function registerDeadlineWork( // the domain to probe its database while dispatch is closed. hasPending: async () => task.canRun() && requested !== reconciled, nextRunAt: () => !task.canRun() && requested !== reconciled - ? Math.min(next ?? Infinity, Date.now() + task.retryMs) + ? Date.now() + task.retryMs : next, }); return { ...worker, wake: () => { requested++; worker.wake(); } }; From 4af412eaea9e780e2d867bd930e77865a1c4672b Mon Sep 17 00:00:00 2001 From: Dotta Date: Fri, 30 Oct 2026 17:25:18 -0500 Subject: [PATCH 3/6] refactor: drive status-card evaluation from committed work Move generation sequencing into a feature-owned deadline worker and retain persisted evaluation, eligibility and relative-search obligations. Add transactional producer hints and initial PostgreSQL acceptance coverage; follow-up completes the producer audit and remaining lifecycle gates. Co-Authored-By: Paperclip --- doc/plans/2026-10-30-status-cards-sleep.md | 33 ++++ .../status-card-evaluation-work.test.ts | 180 ++++++++++++++++++ .../status-card-update-engine.test.ts | 23 +++ server/src/__tests__/status-cards.test.ts | 9 +- server/src/index.ts | 35 +--- .../active-run-watchdog/adapters/postgres.ts | 2 + server/src/services/access.ts | 2 + server/src/services/agent-deletion.ts | 2 + server/src/services/companies.ts | 3 + server/src/services/company-deletion.ts | 2 + server/src/services/company-search.ts | 10 +- server/src/services/company-skills.ts | 2 + .../services/delivery-work-notifications.ts | 1 + .../execution-control-notifications.ts | 3 + server/src/services/execution-workspaces.ts | 4 + server/src/services/heartbeat/retries.ts | 2 + server/src/services/instance-settings.ts | 2 + server/src/services/issue-comment-writes.ts | 2 + server/src/services/issue-prefix.ts | 2 + .../issue-review-state-notifications.ts | 2 + server/src/services/issue-title.ts | 2 + server/src/services/issue-tree-control.ts | 2 + server/src/services/issues.ts | 6 + .../native-finalization-reconciler.ts | 2 + .../native-review-participant.ts | 2 + .../native-workspace-export-recovery.ts | 2 + .../native-workspace-export-retry.ts | 2 + .../native-workspace-reconstruction.ts | 2 + .../paperclip-runner-tool-authority.ts | 2 + server/src/services/plugin-workspaces.ts | 2 + .../src/services/provider-fresh-workspace.ts | 2 + server/src/services/routines.ts | 2 + .../services/slack-conversation-lifecycle.ts | 2 + .../src/services/slack-conversation-state.ts | 2 + server/src/services/status-card-evaluation.ts | 90 +++++++++ .../src/services/status-card-finalization.ts | 2 + .../src/services/status-card-notifications.ts | 10 + .../src/services/status-card-update-engine.ts | 40 +++- server/src/services/status-cards.ts | 159 ++++++++++------ 39 files changed, 561 insertions(+), 93 deletions(-) create mode 100644 doc/plans/2026-10-30-status-cards-sleep.md create mode 100644 server/src/__tests__/status-card-evaluation-work.test.ts create mode 100644 server/src/services/status-card-evaluation.ts create mode 100644 server/src/services/status-card-notifications.ts diff --git a/doc/plans/2026-10-30-status-cards-sleep.md b/doc/plans/2026-10-30-status-cards-sleep.md new file mode 100644 index 00000000000..dcd6c5ca226 --- /dev/null +++ b/doc/plans/2026-10-30-status-cards-sleep.md @@ -0,0 +1,33 @@ +# Status-card evaluation deadlines + +Baseline: `76fde891ccc29283b3f5c8dcdce089c5b3d937d1`. + +Status-card ownership covers evaluation policy, card claims, generation, ordinary assignment wakeups, and failed-assignment cancellation. `index.ts` registers a typed capability. Shared delivery/deadline infrastructure owns transaction settlement, coalescing, admission/drain, retries and shutdown. Existing cards, updates and issues remain durable truth; no timer table or sleep-safety exemption is introduced. + +Manual cards have no automatic obligation. Interval cards retain their requested evaluation cadence. Reactive cards evaluate committed source changes, retain the actual debounce/rate/budget/active-hours deadline while relevant changes remain, and disarm once unchanged. Saved relative-search windows retain their actual membership expiry; the duration interpretation stays with company search. Startup reconstructs existing work without advancing business timestamps. + +Scheduled reactive source events can preempt a future evaluation. A feature-owned transaction advisory lock serializes scheduler evaluation across connections while existing card/issue services perform their own committed writes. The lock holder has no card row lock, avoiding self-deadlock. Existing next-evaluation CAS, generating-issue ownership and issue idempotency keys remain required, including for manual refreshes and older server instances that do not use the advisory lock. The persisted five-minute claim remains the failure retry after a scheduled refresh cannot finish. Internal timers do not establish a Cloud wake alarm. + +## Producer audit (in progress) + +| Source | Owning committed boundary | State | +| --- | --- | --- | +| Card create/config/archive/delete, compile, query/summary writeback, evaluation | status-card mutation transactions | wired; ordinary lifecycle exercised | +| Experimental feature flag | instance settings update transaction | wired; disable exercised | +| Company pause/archive/reactivate/delete | company owning transactions | wired; lifecycle acceptance remains | +| Ordinary issue create/update/remove | issue service transactions | wired; create/change exercised | +| Comments create/delete/tombstone | common insert and issue mutation transactions | wired; comment acceptance remains | +| Review state / chat-run milestones | issue-review-state notification boundary | wired; exact caller coverage audit remains | +| Execution-control writes/run settlement | execution-control transaction helpers | wired; exact caller coverage audit remains | +| Agent deletion/membership reassignment, issue tree transitions, title/prefix | owner transaction | wired; representative acceptance remains | +| Workspace selection/binding, plugin/provider workspace, native recovery/reclaim, active watchdog, retry | owner transactions where uncovered by shared notifications | partially wired; remaining per-write audit required | +| Raw workspace cleanup/routes, conversation activation/settlement, queue paths | direct writes and adjacent owner notifications | exact coverage incomplete; not acceptance-ready | +| Imports/independent writers/replicas | supported service writes and startup reconstruction | full ingress contract audit remains | + +Process-local notifications cover writes using the installed database owner. They do not notify an unrelated SQL writer or another server process; those supported paths need explicit ingress or reconstruction coverage. No claim of cross-replica delivery, host suspension, or fleet savings is made by the current tests. + +## Acceptance evidence and remaining gates + +The original status-card PostgreSQL suite passed25/25. New real PostgreSQL tests exercise ordinary card compilation/writeback, source-change debounce, persisted assignment at a controlled provider boundary, result persistence, quiet SQL after completion, earlier/cancelled deadlines, nested outer commit/rollback and drain/resume. The provider boundary is simulated; no model runs. Additional race, restart, failure, active-hour/rate/budget persistence, multi-company and producer-class coverage remain before PR delivery. + +Shared deadline regressions reproduced and corrected a settled-write admission hold during drain. The correction releases only settled holds, retains an in-memory resume wake, and leaves ambiguous transaction intent fenced by the database coordinator. Unit coverage includes first write during drain, failure entering drain, preserved future deadlines, warm standby and ambiguous commit settlement. diff --git a/server/src/__tests__/status-card-evaluation-work.test.ts b/server/src/__tests__/status-card-evaluation-work.test.ts new file mode 100644 index 00000000000..427ad3385eb --- /dev/null +++ b/server/src/__tests__/status-card-evaluation-work.test.ts @@ -0,0 +1,180 @@ +import { randomUUID } from "node:crypto"; +import { eq, sql } from "drizzle-orm"; +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; +import { activityLog, agentWakeupRequests, agents, companies, companyMemberships, createDb, documents, + heartbeatRuns, instanceSettings, issueComments, issues, statusCards, statusCardUpdates, documentRevisions } from "@paperclipai/db"; +import { companySearchQuerySchema, statusCardRefreshPolicySchema } from "@paperclipai/shared"; +import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js"; +import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; +import { createDeliveryWorkCoordinator } from "../services/delivery-work-coordinator.js"; +import { registerStatusCardEvaluationWork, statusCardEvaluationService } from "../services/status-card-evaluation.js"; +import { statusCardService } from "../services/status-cards.js"; +import { issueService } from "../services/issues.js"; +import { instanceSettingsService } from "../services/instance-settings.js"; +import { idleWorkSnapshot } from "../services/task-admission.js"; + +const external = resolveExternalTestDatabaseUrl("PAPERCLIP_STATUS_CARD_EVALUATION_TEST_DATABASE_URL"); +const support = external ? { supported: true } : await getEmbeddedPostgresTestSupport(); +(support.supported ? describe : describe.skip)("status-card evaluation work", () => { + let temporary: Awaited> | undefined; + let db: ReturnType; + let coordinator: ReturnType | undefined; + let draining = false; + let companyId: string, agentId: string; + const errors: unknown[] = []; + const actor = { agentId: null, userId: "board-user" }; + const policy = (mode: "manual" | "reactive" | "interval" = "reactive") => statusCardRefreshPolicySchema.parse({ + mode, debounceSeconds: 60, intervalMinutes: 15, + }); + // Controlled provider boundary: assignment persists a queued wake, and tests + // explicitly run the authorized summarizer writeback. No model/provider runs. + const wakeup = vi.fn(async (id: string, options: { payload?: Record | null; reason?: string | null }) => { + return db.insert(agentWakeupRequests).values({ companyId, agentId: id, source: "assignment", + reason: options.reason, payload: options.payload, status: "queued" }).returning(); + }); + + beforeAll(async () => { + if (!external) temporary = await startEmbeddedPostgresTestDatabase("status-card-evaluation-"); + db = createDb(external ?? temporary!.connectionString, { maxConnections: 6, idleTimeoutSeconds: 0, maxLifetimeSeconds: 0 }); + }, 30_000); + beforeEach(async () => { + companyId = randomUUID(); agentId = randomUUID(); draining = false; errors.length = 0; wakeup.mockClear(); + await db.insert(companies).values({ id: companyId, name: "Status", issuePrefix: `S${companyId.slice(0, 6)}` }); + await db.insert(agents).values({ id: agentId, companyId, name: "Summarizer", role: "general", status: "idle", adapterType: "process", adapterConfig: {} }); + await instanceSettingsService(db).updateExperimental({ enableStatusCards: true }); + await Promise.all(Array.from({ length: 6 }, () => db.execute(sql`select pg_sleep(0.01)`))); + vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"], shouldAdvanceTime: true }); + }); + afterEach(async () => { + await coordinator?.stop(); coordinator = undefined; + vi.useRealTimers(); vi.restoreAllMocks(); + await db.delete(agentWakeupRequests); await db.delete(statusCardUpdates); await db.delete(statusCards); + await db.delete(documentRevisions); await db.delete(documents); await db.delete(activityLog); + await db.delete(issueComments); await db.delete(issues); await db.delete(heartbeatRuns); + await db.delete(instanceSettings); await db.delete(agents); await db.delete(companyMemberships); await db.delete(companies); + }); + afterAll(async () => { await db?.$client.end({ timeout: 1 }); await temporary?.cleanup(); }); + async function start() { + coordinator = createDeliveryWorkCoordinator({ owner: db, canRun: () => !draining, canReconcile: () => true, + onError: error => errors.push(error) }); + await registerStatusCardEvaluationWork(coordinator, statusCardEvaluationService(db, { wakeup }), () => !draining).ready; + } + async function settled() { + await vi.waitFor(async () => { + await vi.advanceTimersByTimeAsync(30_000); + expect(idleWorkSnapshot().active).toBe(0); + }, { timeout: 5000 }); + } + async function quiet() { + await settled(); + expect(coordinator!.nextWakeAt()).toBeNull(); + const reads = vi.spyOn(db, "select"), writes = vi.spyOn(db, "update"), executes = vi.spyOn(db, "execute"); + const calls = wakeup.mock.calls.length; + await vi.advanceTimersByTimeAsync(10 * 60_000); + expect(reads).not.toHaveBeenCalled(); expect(writes).not.toHaveBeenCalled(); expect(executes).not.toHaveBeenCalled(); + expect(wakeup).toHaveBeenCalledTimes(calls); + reads.mockRestore(); writes.mockRestore(); executes.mockRestore(); + } + async function writeback(cardId: string, generationIssueId: string, compile = false) { + const [run] = await db.insert(heartbeatRuns).values({ companyId, agentId, status: "running" }).returning(); + await db.update(issues).set({ checkoutRunId: run!.id }).where(eq(issues.id, generationIssueId)); + const writer = { agentId, runId: run!.id }; + const cards = statusCardService(db); + if (compile) await cards.writeQuery(cardId, { queries: [companySearchQuerySchema.parse({ scope: "issues", status: ["blocked"] })], + title: "Blockers", changeSummary: "Compiled", generationIssueId }, writer); + await cards.writeSummary(cardId, { markdown: "Current blockers", changeSummary: "Updated", generationIssueId }, writer); + await issueService(db).update(generationIssueId, { status: "done" }); + } + async function card(mode: "manual" | "reactive" | "interval" = "reactive") { + const cards = statusCardService(db); + const created = await cards.create(companyId, { interestPrompt: "Blockers", titlePinned: false, agentId, refreshPolicy: policy(mode) }, actor); + const compile = await cards.requestCompile(created.id, actor); + await writeback(created.id, compile.generatingIssue.id, true); + return cards.getById(created.id).then(row => row!); + } + + it("settles empty, disabled, configured manual and unchanged reactive history without recurring SQL", async () => { + await start(); await quiet(); + await card("manual"); await card("reactive"); await quiet(); + await instanceSettingsService(db).updateExperimental({ enableStatusCards: false }); await quiet(); + expect(errors).toEqual([]); + }); + + it("ordinary issue change schedules debounce, assigns one durable wake, writes a result and becomes quiet", async () => { + await start(); const created = await card(); await quiet(); + const changed = await issueService(db).create(companyId, { title: "New blocker", status: "blocked", priority: "medium", createdByUserId: "board-user" }); + await vi.waitFor(async () => { + expect((await statusCardService(db).getById(created.id))?.pendingChangeCount).toBeGreaterThan(0); + }, { timeout: 5000 }); + const pending = await statusCardService(db).getById(created.id); + expect(pending?.pendingChangeCount).toBeGreaterThan(0); + expect(pending?.nextEvalAt).toBeInstanceOf(Date); + expect(wakeup).not.toHaveBeenCalled(); + await vi.advanceTimersByTimeAsync(60_000); await settled(); + const generating = await statusCardService(db).getById(created.id); + expect(generating?.generatingIssueId).toBeTruthy(); + const durable = await db.select().from(agentWakeupRequests); + expect(durable).toHaveLength(1); expect(durable[0]?.payload?.issueId).toBe(generating?.generatingIssueId); + expect(changed.companyId).toBe(companyId); + await writeback(created.id, generating!.generatingIssueId!); await quiet(); + expect(await db.select().from(statusCardUpdates).then(rows => rows.filter(row => row.generationIssueId === generating?.generatingIssueId))) + .toMatchObject([{ status: "ok", finishedAt: expect.any(Date) }]); + expect(errors).toEqual([]); + }); + + it("cancels archived or disabled future work and replaces an earlier interval deadline", async () => { + await start(); let created = await card("interval"); await settled(); + const old = coordinator!.nextWakeAt(); + created = await statusCardService(db).update(created, { refreshPolicy: { ...policy("interval"), intervalMinutes: 1 } }, actor); + await settled(); expect(coordinator!.nextWakeAt()).toBeLessThan(old!); + await statusCardService(db).update(created, { archived: true }, actor); await quiet(); + expect(wakeup).not.toHaveBeenCalled(); expect(errors).toEqual([]); + }); + + it("does not react to rolled-back or nested uncommitted card configuration", async () => { + await start(); const created = await card("manual"); await quiet(); + await expect(db.transaction(async tx => { + await statusCardService(tx as unknown as typeof db).update(created, { refreshPolicy: policy("interval") }, actor); + await vi.advanceTimersByTimeAsync(30_000); expect(wakeup).not.toHaveBeenCalled(); + throw new Error("rollback"); + })).rejects.toThrow("rollback"); + await quiet(); expect((await statusCardService(db).getById(created.id))?.nextEvalAt).toBeNull(); + await db.transaction(async tx => { + await tx.transaction(async nested => { + await statusCardService(nested as unknown as typeof db).update(created, { refreshPolicy: policy("interval") }, actor); + }); + expect(coordinator!.nextWakeAt()).not.toBeNull(); + expect((await statusCardService(db).getById(created.id))?.nextEvalAt).toBeNull(); + }); + await settled(); expect(coordinator!.nextWakeAt()).not.toBeNull(); expect(errors).toEqual([]); + }); + + it("serializes competing connections when one source event preempts a future evaluation", async () => { + const created = await card(); + await db.update(statusCards).set({ nextEvalAt: new Date(Date.now() + 3_600_000) }).where(eq(statusCards.id, created.id)); + await issueService(db).create(companyId, { title: "Concurrent blocker", status: "blocked", priority: "medium", createdByUserId: "board-user" }); + const first = statusCardEvaluationService(db, { wakeup }); + const second = statusCardEvaluationService(db, { wakeup }); + first.changed(new Set([`changes:${companyId}`])); second.changed(new Set([`changes:${companyId}`])); + await Promise.all([first.run(new AbortController().signal), second.run(new AbortController().signal)]); + const pending = await statusCardService(db).getById(created.id); + expect(pending?.pendingChangeCount).toBeGreaterThan(0); + expect(pending?.nextEvalAt?.getTime()).toBeLessThan(Date.now() + 3_600_000); + expect(wakeup).not.toHaveBeenCalled(); + await vi.advanceTimersByTimeAsync(60_000); + await Promise.all([first.run(new AbortController().signal), second.run(new AbortController().signal)]); + const updated = await statusCardService(db).getById(created.id); + expect(updated?.generatingIssueId).toBeTruthy(); expect(wakeup).toHaveBeenCalledOnce(); + expect(await db.select().from(statusCardUpdates).then(rows => rows.filter(row => row.generationIssueId === updated?.generatingIssueId))) + .toHaveLength(1); + }); + + it("preserves the first write during drain until resume", async () => { + await start(); const created = await card("manual"); await quiet(); + draining = true; + await statusCardService(db).update(created, { refreshPolicy: policy("interval") }, actor); + await settled(); expect(wakeup).not.toHaveBeenCalled(); + draining = false; await vi.advanceTimersByTimeAsync(30_000); await settled(); + expect(coordinator!.nextWakeAt()).not.toBeNull(); expect(errors).toEqual([]); + }); +}); diff --git a/server/src/__tests__/status-card-update-engine.test.ts b/server/src/__tests__/status-card-update-engine.test.ts index 09a660e8741..b8309415770 100644 --- a/server/src/__tests__/status-card-update-engine.test.ts +++ b/server/src/__tests__/status-card-update-engine.test.ts @@ -8,6 +8,7 @@ import { filterStatusCardChanges, isWithinStatusCardActiveHours, nextStatusCardEvaluationAt, + nextStatusCardPolicyEligibility, statusCardChangesHash, } from "../services/status-card-update-engine.js"; @@ -78,6 +79,28 @@ describe("status card update engine", () => { expect(evaluateStatusCardPolicy({ policy: hours, now: new Date("2026-07-23T18:00:00.000Z"), lastChangeAt: null, updatesLastHour: 0, tokensToday: 0, manual: false }).action).toBe("pause_hours"); }); + it("disarms unchanged reactive evaluation while retaining exact pending eligibility", () => { + const policy = statusCardRefreshPolicySchema.parse({ mode: "reactive", debounceSeconds: 90, maxUpdatesPerHour: 2, + activeHours: { start: "09:00", end: "17:00", timezone: "UTC" } }); + const now = new Date("2026-07-23T16:59:30Z"); + expect(nextStatusCardEvaluationAt(policy, now)).toBeNull(); + expect(nextStatusCardPolicyEligibility({ policy, now, lastChangeAt: now, completedUpdateTimes: [], tokensToday: 0 })) + .toEqual(new Date("2026-07-24T09:00:00Z")); + const open = new Date("2026-07-23T14:00:00Z"); + expect(nextStatusCardPolicyEligibility({ policy, now: open, lastChangeAt: new Date("2026-07-23T13:00:00Z"), + completedUpdateTimes: [new Date("2026-07-23T13:30:00Z"), new Date("2026-07-23T13:45:00Z")], tokensToday: 0 })) + .toEqual(new Date("2026-07-23T14:30:00.001Z")); + expect(nextStatusCardPolicyEligibility({ policy, now: open, lastChangeAt: open, completedUpdateTimes: [], tokensToday: 100_000 })) + .toEqual(new Date("2026-07-24T09:00:00Z")); + }); + + it("retains empty-window pending changes without a recurring eligibility scan", () => { + const policy = statusCardRefreshPolicySchema.parse({ mode: "reactive", debounceSeconds: 1, + activeHours: { start: "09:00", end: "09:00", timezone: "UTC" } }); + const now = new Date("2026-07-23T14:00:00Z"); + expect(nextStatusCardPolicyEligibility({ policy, now, lastChangeAt: now, completedUpdateTimes: [], tokensToday: 0 })).toBeNull(); + }); + it("selects full rebuilds for bounded drift rules and incremental otherwise", () => { const base = { hasDocument: true, changeCount: 2, queryVersion: 3, lastUpdateQueryVersion: 3, incrementalCount: 2, configurationChanged: false }; expect(chooseStatusCardUpdateKind(base)).toBe("incremental"); diff --git a/server/src/__tests__/status-cards.test.ts b/server/src/__tests__/status-cards.test.ts index e1436666316..0c24b05dd70 100644 --- a/server/src/__tests__/status-cards.test.ts +++ b/server/src/__tests__/status-cards.test.ts @@ -1,3 +1,4 @@ +import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js"; import { randomUUID } from "node:crypto"; import express from "express"; import request from "supertest"; @@ -37,7 +38,8 @@ import { startEmbeddedPostgresTestDatabase, } from "./helpers/embedded-postgres.js"; -const embeddedPostgresSupport = await getEmbeddedPostgresTestSupport(); +const externalDatabaseUrl = resolveExternalTestDatabaseUrl("PAPERCLIP_STATUS_CARDS_TEST_DATABASE_URL"); +const embeddedPostgresSupport = externalDatabaseUrl ? { supported: true } : await getEmbeddedPostgresTestSupport(); const describeEmbeddedPostgres = embeddedPostgresSupport.supported ? describe : describe.skip; type Db = ReturnType; @@ -79,8 +81,8 @@ describeEmbeddedPostgres("status card routes", () => { let tempDb: Awaited> | null = null; beforeAll(async () => { - tempDb = await startEmbeddedPostgresTestDatabase("paperclip-status-cards-"); - db = createDb(tempDb.connectionString); + if (!externalDatabaseUrl) tempDb = await startEmbeddedPostgresTestDatabase("paperclip-status-cards-"); + db = createDb(externalDatabaseUrl ?? tempDb!.connectionString, { maxConnections: 6 }); }, 20_000); afterEach(async () => { @@ -100,6 +102,7 @@ describeEmbeddedPostgres("status card routes", () => { }); afterAll(async () => { + await db?.$client.end({ timeout: 1 }); await tempDb?.cleanup(); }); diff --git a/server/src/index.ts b/server/src/index.ts index ae229f03518..6fb78f1612d 100644 --- a/server/src/index.ts +++ b/server/src/index.ts @@ -145,13 +145,12 @@ import { reconcileCodexLocalManagedHomesOnStartup, reconcilePersistedRuntimeServicesOnStartup, routineService, - statusCardService, toolAccessService, workspaceOperationService, } from "./services/index.js"; import { questionResponseDeliveryService } from "./services/question-response-delivery.js"; import { deliverNativeQuestionResponse } from "./services/native-runtime/native-question-bridge.js"; -import { queueIssueAssignmentWakeup } from "./services/issue-assignment-wakeup.js"; +import { registerStatusCardEvaluationWork, statusCardEvaluationService } from "./services/status-card-evaluation.js"; import { createSecretProposalsService } from "./services/secret-proposals.js"; import { environmentRuntimeService } from "./services/environment-runtime.js"; import { historicalCleanupVerificationService } from "./services/historical-cleanup-verification.js"; @@ -1569,7 +1568,7 @@ async function startServerWithDatabaseTeardown( if (result.settled > 0) logger.info(result, "reconciled routine execution issue status"); }).catch(() => logger.warn("routine execution status reconciliation deferred"))); }; - const statusCards = statusCardService(db as any); + const statusCardEvaluation = statusCardEvaluationService(db as any, heartbeat); const issues = issueService(db as any); const mergedPullRequestConfirmations = issueThreadInteractionService(db as any, { wakeup: heartbeat.wakeup, @@ -1980,6 +1979,8 @@ async function startServerWithDatabaseTeardown( () => !isIdleTaskDrainActive() && !isWarmStandby() && !heartbeatSchedulerStopped).ready); await step("join_request_retention", () => registerJoinRequestRetentionWork(deliveryWork, joinRequestRetentionService(db as any), () => !isIdleTaskDrainActive() && !isWarmStandby() && !heartbeatSchedulerStopped).ready); + await step("status_card_evaluation", () => registerStatusCardEvaluationWork(deliveryWork, statusCardEvaluation, + () => !isIdleTaskDrainActive() && !isWarmStandby() && !heartbeatSchedulerStopped).ready); await step("retention_sweep", runRetentionSweep); }; @@ -2010,33 +2011,7 @@ async function startServerWithDatabaseTeardown( scheduleGitHubConnectionContinuitySweep(); if (isIdleTaskDrainActive() || heartbeatSchedulerStopped) return; - trackHeartbeatSchedulerWork((async () => { - const experimental = await instanceSettingsService(db).getExperimental(); - if (experimental.enableStatusCards !== true) return; - const result = await statusCards.tickDueStatusCards(new Date()); - await Promise.all(result.enqueued.map(async ({ cardId, generatingIssue }) => { - try { - await queueIssueAssignmentWakeup({ - heartbeat, - issue: generatingIssue, - reason: "status_card_update_assigned", - mutation: "status_card.scheduler_update_requested", - contextSource: "status_card_scheduler", - requestedByActorType: "system", - taskKey: `status-card:${cardId}`, - rethrowOnError: true, - }); - } catch (err) { - await issues.update(generatingIssue.id, { status: "cancelled" }); - throw err; - } - })); - if (result.evaluated > 0 || result.enqueued.length > 0) { - logger.info({ evaluated: result.evaluated, enqueued: result.enqueued.length }, "status-card scheduler tick complete"); - } - })().catch((err) => { - logger.error({ err }, "status-card scheduler tick failed"); - })); + if (isIdleTaskDrainActive() || heartbeatSchedulerStopped) return; if (!(await heartbeat.resolveSchedulingSuppression()).suppressed) { diff --git a/server/src/modules/active-run-watchdog/adapters/postgres.ts b/server/src/modules/active-run-watchdog/adapters/postgres.ts index 9531c449260..002415c03a6 100644 --- a/server/src/modules/active-run-watchdog/adapters/postgres.ts +++ b/server/src/modules/active-run-watchdog/adapters/postgres.ts @@ -1,3 +1,4 @@ +import { notifyStatusCardIssueChange } from "../../../services/status-card-notifications.js"; import { DELIVERY_QUEUES, notifyDeliveryWork } from "../../../services/delivery-work-notifications.js"; import { notifyExternalAgentSettlement } from "../../external-agents/index.js"; import { notifyNativeStatusReassessment } from "../../../services/native-status-reassessment-notifications.js"; @@ -385,6 +386,7 @@ export function createPostgresWatchdogAdapter(db: Db): WatchdogRunReader & Watch )); } + await notifyStatusCardIssueChange(tx, companyId); await tx .update(issues) .set({ diff --git a/server/src/services/access.ts b/server/src/services/access.ts index aa0f9dcc828..477c0434c50 100644 --- a/server/src/services/access.ts +++ b/server/src/services/access.ts @@ -1,3 +1,4 @@ +import { notifyStatusCardIssueChange } from "./status-card-notifications.js"; import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js"; import { notifyNativeStatusReassessment } from "./native-status-reassessment-notifications.js"; import { notifyUnsafeWorkspaceExportRecovery } from "./unsafe-workspace-export-notifications.js"; @@ -745,6 +746,7 @@ export function accessService(db: Db) { await notifyNativeStatusReassessment(tx, { companyId }); await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors); await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers); + await notifyStatusCardIssueChange(tx, companyId); const resetInProgress = await tx .update(issues) .set({ diff --git a/server/src/services/agent-deletion.ts b/server/src/services/agent-deletion.ts index 3650c40f1a6..5907d222e1d 100644 --- a/server/src/services/agent-deletion.ts +++ b/server/src/services/agent-deletion.ts @@ -1,3 +1,4 @@ +import { notifyStatusCardIssueChange } from "./status-card-notifications.js"; import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js"; import { notifyExternalConversationSettlement } from "./external-conversation-settlement-notifications.js"; import { notifyUnsafeWorkspaceExportRecovery } from "./unsafe-workspace-export-notifications.js"; @@ -38,6 +39,7 @@ export async function deleteAgentDependencies(tx: Db, companyId: string, id: str await issueThreadInteractionService(tx).cancelPendingForDeletedAddressee(companyId, id); await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors); await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers); + await notifyStatusCardIssueChange(tx, companyId); await tx.update(agents).set({ reportsTo: null }).where(eq(agents.reportsTo, id)); await tx .update(issues) diff --git a/server/src/services/companies.ts b/server/src/services/companies.ts index 0f2ed547a79..8efd6a269fb 100644 --- a/server/src/services/companies.ts +++ b/server/src/services/companies.ts @@ -1,3 +1,4 @@ +import { notifyStatusCardIssueChange } from "./status-card-notifications.js"; import { notifyChatInboundWakeNotices } from "./chat-inbound-wake-notice-notifications.js"; import { notifyCompanyRetentionWork } from "./retention-work-notifications.js"; import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js"; @@ -336,6 +337,7 @@ export function companyService(db: Db, _budgetHooks: BudgetServiceHooks = {}) { await notifyDeliveryWork(tx, DELIVERY_QUEUES.skillSourceSync); await notifyDeliveryWork(tx, DELIVERY_QUEUES.routineSchedules); await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors); + await notifyStatusCardIssueChange(tx, id); await notifyCompanyRetentionWork(tx, id); await notifyChatInboundWakeNotices(tx, { companyId: id }); await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentLifecycle, id); @@ -492,6 +494,7 @@ export function companyService(db: Db, _budgetHooks: BudgetServiceHooks = {}) { await notifyDeliveryWork(tx, DELIVERY_QUEUES.skillSourceSync); await notifyDeliveryWork(tx, DELIVERY_QUEUES.routineSchedules); await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors); + await notifyStatusCardIssueChange(tx, id); await notifyCompanyRetentionWork(tx, id); await notifyChatInboundWakeNotices(tx, { companyId: id }); await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentLifecycle, id); diff --git a/server/src/services/company-deletion.ts b/server/src/services/company-deletion.ts index 7ab88aadfa9..0379638968b 100644 --- a/server/src/services/company-deletion.ts +++ b/server/src/services/company-deletion.ts @@ -1,3 +1,4 @@ +import { notifyStatusCardEvaluation } from "./status-card-notifications.js"; import { notifyUnsafeWorkspaceExportRecovery } from "./unsafe-workspace-export-notifications.js"; import { notifyCancelledWorkspacePreservation } from "./cancelled-workspace-preservation-notifications.js"; import { notifyInstructionCollection } from "./instruction-collection-notifications.js"; @@ -77,6 +78,7 @@ export async function deleteCompany(db: Db, id: string) { await notifyDeliveryWork(tx, DELIVERY_QUEUES.skillPackages); await notifyDeliveryWork(tx, DELIVERY_QUEUES.routineSchedules); await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors); + await notifyStatusCardEvaluation(tx, id); await notifyDeliveryWork(tx, DELIVERY_QUEUES.secretProposalExpiry, id); await notifyToolReviewChange(tx, id); await agentObservabilityService(tx as unknown as Db).removeForCompanyDeletion(id); diff --git a/server/src/services/company-search.ts b/server/src/services/company-search.ts index 220967e362d..bcf2a550d3f 100644 --- a/server/src/services/company-search.ts +++ b/server/src/services/company-search.ts @@ -207,14 +207,20 @@ function priorityRank(priority: string | null | undefined) { return index >= 0 ? index : ISSUE_PRIORITIES.length; } -function updatedWithinStart(value: string | undefined, now = new Date()): Date | null { +/** Duration used by saved search membership and its deadline consumers. */ +export function companySearchUpdatedWithinMs(value: string | undefined): number | null { if (!value) return null; const match = /^(\d+)(h|d|w|m)$/.exec(value); if (!match) return null; const amount = Number.parseInt(match[1]!, 10); const unit = match[2]; const hours = unit === "h" ? amount : unit === "d" ? amount * 24 : unit === "w" ? amount * 24 * 7 : amount * 24 * 30; - return new Date(now.getTime() - hours * 60 * 60 * 1000); + return hours * 60 * 60 * 1000; +} + +function updatedWithinStart(value: string | undefined, now = new Date()): Date | null { + const duration = companySearchUpdatedWithinMs(value); + return duration === null ? null : new Date(now.getTime() - duration); } function issueOnlyFiltersActive(query: CompanySearchQuery) { diff --git a/server/src/services/company-skills.ts b/server/src/services/company-skills.ts index 7ba950039e9..faf54c410a9 100644 --- a/server/src/services/company-skills.ts +++ b/server/src/services/company-skills.ts @@ -1,3 +1,4 @@ +import { notifyStatusCardIssueChange } from "./status-card-notifications.js"; import { notifySkillVersionRetention } from "./skill-version-retention-notifications.js"; import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js"; import { verifyFileManifest } from "./file-manifest-cache.js"; @@ -7450,6 +7451,7 @@ export function companySkillService(db: Db, options: { actor?: AuthorizationActo for (const row of rows) { await db.transaction(async (tx) => { await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers); + await notifyStatusCardIssueChange(tx, companyId); await tx .update(issues) .set({ hiddenAt: now, updatedAt: now }) diff --git a/server/src/services/delivery-work-notifications.ts b/server/src/services/delivery-work-notifications.ts index 9195f1fcb12..fdc4aaadc20 100644 --- a/server/src/services/delivery-work-notifications.ts +++ b/server/src/services/delivery-work-notifications.ts @@ -2,6 +2,7 @@ import { signalDatabaseWork, subscribeDatabaseWork } from "@paperclipai/db"; /** Topics for existing durable queues; register intent before their writes. */ export const DELIVERY_QUEUES = { + statusCardEvaluation: "status-card-evaluation", runEvidenceMigration: "run-evidence-migration", providerTraceRetention: "provider-trace-retention", nativeRunnerEvidenceRetention: "native-runner-evidence-retention", diff --git a/server/src/services/execution-control-notifications.ts b/server/src/services/execution-control-notifications.ts index 22c8913bbcd..c7b26a8bd72 100644 --- a/server/src/services/execution-control-notifications.ts +++ b/server/src/services/execution-control-notifications.ts @@ -1,7 +1,9 @@ +import { notifyStatusCardIssueChange } from "./status-card-notifications.js"; import type { Db } from "@paperclipai/db"; import { DELIVERY_QUEUES, notifyDeliveryWork, type DeliveryQueue } from "./delivery-work-notifications.js"; export async function notifyExecutionRunWork(transaction: object) { + await notifyStatusCardIssueChange(transaction); for (const queue of [DELIVERY_QUEUES.executionStatus, DELIVERY_QUEUES.executionFinalization, DELIVERY_QUEUES.executionReplacement, DELIVERY_QUEUES.executionDisposition, DELIVERY_QUEUES.issueMonitors]) { await notifyDeliveryWork(transaction, queue); @@ -14,6 +16,7 @@ export function writeExecutionControl( write: (tx: Db) => PromiseLike, ): Promise { return db.transaction(async transaction => { + await notifyStatusCardIssueChange(transaction); await notifyDeliveryWork(transaction, DELIVERY_QUEUES.issueMonitors); for (const topic of typeof queue === "string" ? [queue] : queue) await notifyDeliveryWork(transaction, topic); return write(transaction as unknown as Db); diff --git a/server/src/services/execution-workspaces.ts b/server/src/services/execution-workspaces.ts index 4b716916638..caaae186ad8 100644 --- a/server/src/services/execution-workspaces.ts +++ b/server/src/services/execution-workspaces.ts @@ -1,3 +1,4 @@ +import { notifyStatusCardIssueChange } from "./status-card-notifications.js"; import { effectiveProjectWorkspacePolicy } from "./project-workspace-provider.js"; import { inspectPluginWorkspace, removePluginWorkspace, restorePluginWorkspace, type WorkspaceWorkers } from "./plugin-workspaces.js"; import { acquireExecutionWorkspaceLifecycleLock, withExecutionWorkspaceLifecycleLock } from "./execution-workspace-lifecycle-lock.js"; @@ -1590,6 +1591,7 @@ export function executionWorkspaceService(db: Db, opts: ExecutionWorkspaceServic await validateSelection(input, tx); const intent: TaskWorkspaceIntent = { version: 1, request: { key: input.requestKey, expectedBindingRevision: input.expectedBindingRevision }, selection: input.selection, source: input.source ?? "explicit" }; const nextRevision = task.workspaceBindingRevision + 1; + await notifyStatusCardIssueChange(tx, task.companyId); await tx.update(issues).set({ workspaceBindingRevision: nextRevision, workspacePendingSelection: { version: 1, requestKey: input.requestKey, expectedBindingRevision: nextRevision, intent }, updatedAt: new Date() }).where(eq(issues.id, task.id)); @@ -1633,6 +1635,7 @@ export function executionWorkspaceService(db: Db, opts: ExecutionWorkspaceServic } } } + await notifyStatusCardIssueChange(tx, task.companyId); await tx.update(issues).set({ ...patch, assigneeAdapterOverrides, workspaceSelection: pending.intent, workspacePendingSelection: null, workspaceBindingRevision: task.workspaceBindingRevision + 1, updatedAt: new Date() }).where(eq(issues.id, task.id)); return true; @@ -1651,6 +1654,7 @@ export function executionWorkspaceService(db: Db, opts: ExecutionWorkspaceServic const pendingAfterFirstBinding = !task.executionWorkspaceId && pending?.expectedBindingRevision === task.workspaceBindingRevision ? { ...pending, expectedBindingRevision: nextRevision } : pending; + await notifyStatusCardIssueChange(tx, companyId); const [bound] = await tx.update(issues).set({ ...patch, executionWorkspaceId: workspaceId, workspaceBindingRevision: nextRevision, workspacePendingSelection: pendingAfterFirstBinding, updatedAt: new Date() }) .where(eq(issues.id, issueId)).returning(); diff --git a/server/src/services/heartbeat/retries.ts b/server/src/services/heartbeat/retries.ts index 946f5e309f8..de03aebeca0 100644 --- a/server/src/services/heartbeat/retries.ts +++ b/server/src/services/heartbeat/retries.ts @@ -1,3 +1,4 @@ +import { notifyStatusCardIssueChange } from "../status-card-notifications.js"; import { deriveTaskKeyWithHeartbeatFallback } from "./run-state.js"; import { infrastructureRetryDelayMs } from "../system-recovery-state.js"; import { @@ -1178,6 +1179,7 @@ export function createHeartbeatRetries(db: Db, dependencies: HeartbeatRetryDepen } if (issueId) { + await notifyStatusCardIssueChange(tx, run.companyId); await tx .update(issues) .set({ diff --git a/server/src/services/instance-settings.ts b/server/src/services/instance-settings.ts index 0208653a86f..35eddc10660 100644 --- a/server/src/services/instance-settings.ts +++ b/server/src/services/instance-settings.ts @@ -1,3 +1,4 @@ +import { notifyStatusCardEvaluation } from "./status-card-notifications.js"; import { notifySkillVersionRetentionReconstruction } from "./skill-version-retention-notifications.js"; import { notifyNativeEnvironmentSettlement } from "./native-environment-settlement-notifications.js"; import { notifyExternalAgentSettlement } from "../modules/external-agents/index.js"; @@ -625,6 +626,7 @@ export function instanceSettingsService(db: Db, options: InstanceSettingsService } const now = new Date(); const [updated] = await db.transaction(async tx => { + if (patch.enableStatusCards !== undefined) await notifyStatusCardEvaluation(tx); await notifyDeliveryWork(tx, DELIVERY_QUEUES.mcpEvents, "*"); await notifyDeliveryWork(tx, DELIVERY_QUEUES.dotEvents, "*"); await notifyDeliveryWork(tx, DELIVERY_QUEUES.grokDispatch); diff --git a/server/src/services/issue-comment-writes.ts b/server/src/services/issue-comment-writes.ts index e24e78e9855..4d4cf374e5b 100644 --- a/server/src/services/issue-comment-writes.ts +++ b/server/src/services/issue-comment-writes.ts @@ -1,3 +1,4 @@ +import { notifyStatusCardIssueChange } from "./status-card-notifications.js"; import type { PgInsertValue } from "drizzle-orm/pg-core"; import { issueComments, type Db } from "@paperclipai/db"; import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js"; @@ -6,6 +7,7 @@ type WriteDb = Db | Parameters[0]>[0]; export function insertIssueComment(db: WriteDb, input: PgInsertValue & { companyId: string }) { return db.transaction(async tx => { + await notifyStatusCardIssueChange(tx, input.companyId); await notifyDeliveryWork(tx, DELIVERY_QUEUES.dotEvents, `company:${input.companyId}`); return tx.insert(issueComments).values(input).returning(); }); diff --git a/server/src/services/issue-prefix.ts b/server/src/services/issue-prefix.ts index 5e31f9bd7ff..649a4aaae9b 100644 --- a/server/src/services/issue-prefix.ts +++ b/server/src/services/issue-prefix.ts @@ -1,3 +1,4 @@ +import { notifyStatusCardIssueChange } from "./status-card-notifications.js"; import { and, eq, sql } from "drizzle-orm"; import type { PgColumn } from "drizzle-orm/pg-core"; import type { Db } from "@paperclipai/db"; @@ -127,6 +128,7 @@ export async function rekeyCompanyIssueIdentifiers( // overload, which returns NULL for a non-pattern argument. const tailStart = sql`${fromPrefix.length + 1}::int`; + await notifyStatusCardIssueChange(tx, companyId); const rekeyedIssues = await tx .update(issues) .set({ identifier: sql`${toPrefix} || substring(${issues.identifier} from ${tailStart})` }) diff --git a/server/src/services/issue-review-state-notifications.ts b/server/src/services/issue-review-state-notifications.ts index 679b7e37475..d66ca96eb0e 100644 --- a/server/src/services/issue-review-state-notifications.ts +++ b/server/src/services/issue-review-state-notifications.ts @@ -1,9 +1,11 @@ +import { notifyStatusCardIssueChange } from "./status-card-notifications.js"; import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js"; import { notifyExternalConversationSettlement } from "./external-conversation-settlement-notifications.js"; import { notifyNativeReviewResponse } from "./native-review-response-notifications.js"; import { notifyNativeStatusReassessment } from "./native-status-reassessment-notifications.js"; export async function notifyIssueReviewState(tx: object, scope: { companyId: string; issueId?: string; runId?: string; agentId?: string }) { + await notifyStatusCardIssueChange(tx, scope.companyId); await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors); await notifyExternalConversationSettlement(tx, scope); await notifyNativeReviewResponse(tx, scope); diff --git a/server/src/services/issue-title.ts b/server/src/services/issue-title.ts index af6716e3aa4..51e360269d7 100644 --- a/server/src/services/issue-title.ts +++ b/server/src/services/issue-title.ts @@ -1,3 +1,4 @@ +import { notifyStatusCardIssueChange } from "./status-card-notifications.js"; import { and, eq } from "drizzle-orm"; import { heartbeatRuns, issues, type Db } from "@paperclipai/db"; import { setIssueTitleSchema, type SetIssueTitle } from "@paperclipai/shared"; @@ -76,6 +77,7 @@ export async function setIssueTitle( || (issue.title === title && !issue.titleNeedsGeneration)) { return { result: await saveReceipt({ id: issue.id, title: issue.title, titleNeedsGeneration: issue.titleNeedsGeneration, changed: false }), publication: null }; } + await notifyStatusCardIssueChange(tx, companyId); await tx.update(issues).set({ title, titleNeedsGeneration: false, updatedAt: new Date() }).where(predicate); await issueReferenceService(tx).syncIssue(issueId, tx); const { publication } = await persistActivity(tx, { diff --git a/server/src/services/issue-tree-control.ts b/server/src/services/issue-tree-control.ts index 0d7633f846a..a8009559827 100644 --- a/server/src/services/issue-tree-control.ts +++ b/server/src/services/issue-tree-control.ts @@ -1,3 +1,4 @@ +import { notifyStatusCardIssueChange } from "./status-card-notifications.js"; import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js"; import { notifyNativeStatusReassessment } from "./native-status-reassessment-notifications.js"; import { notifyUnsafeWorkspaceExportRecovery } from "./unsafe-workspace-export-notifications.js"; @@ -1011,6 +1012,7 @@ export function issueTreeControlService(db: Db) { await notifyChatResponseAvailability(tx, { companyId, issueId }); if (status === "blocked" || status === "in_review") await notifyUnsafeWorkspaceExportRecovery(tx, { companyId, issueId }); } + await notifyStatusCardIssueChange(tx, companyId); const rows = await tx .update(issues) .set({ diff --git a/server/src/services/issues.ts b/server/src/services/issues.ts index 42c65bb5d02..8787ac867a8 100644 --- a/server/src/services/issues.ts +++ b/server/src/services/issues.ts @@ -1,3 +1,4 @@ +import { notifyStatusCardIssueChange } from "./status-card-notifications.js"; import { notifyExternalConversationSettlement } from "./external-conversation-settlement-notifications.js"; import { notifyNativeRecoveryNotices } from "./native-recovery-notice-notifications.js"; import { notifyNativeRunProjection } from "./native-run-projection-notifications.js"; @@ -10606,6 +10607,7 @@ export function issueService(db: Db) { } if (values.monitorNextCheckAt) await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors); await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers); + await notifyStatusCardIssueChange(tx, companyId); const [issue] = await tx.insert(issues).values(values).returning(); await attachOwnedDraftImages(tx, issue, issue.description, { userId: issueData.createdByUserId, agentId: issueData.createdByAgentId }); if (issue.visibility === "private" && !inheritedPrivateSubtree && issueData.createdByAgentId) { @@ -11486,6 +11488,7 @@ export function issueService(db: Db) { (blockedByIssueIds !== undefined || issueData.unblockDescriptor !== undefined)))) { patch.statusVersion = sql`${issues.statusVersion} + 1` as unknown as number; } + await notifyStatusCardIssueChange(tx, receiptExisting.companyId); if ((patch.status !== undefined && patch.status !== receiptExisting.status) || patch.completedAt !== undefined || patch.cancelledAt !== undefined || issueData.parentId !== undefined || issueData.executionWorkspaceId !== undefined) { @@ -11954,6 +11957,7 @@ export function issueService(db: Db) { if (owner) { await notifyExternalConversationSettlement(tx, { companyId: owner.companyId }); await notifyApprovalWork(tx, owner.companyId); + await notifyStatusCardIssueChange(tx, owner.companyId); } await notifyDecisionTargetIssueRemoval(tx, id); await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors); @@ -12726,6 +12730,7 @@ export function issueService(db: Db) { .from(issueComments).where(eq(issueComments.id, commentId)); if (!source) return null; await notifyExternalConversationSettlement(tx, source); + await notifyStatusCardIssueChange(tx, source.companyId); const [comment] = await tx .delete(issueComments) .where(eq(issueComments.id, commentId)) @@ -12765,6 +12770,7 @@ export function issueService(db: Db) { .for("update"); if (!source) return null; await notifyExternalConversationSettlement(tx, source); + await notifyStatusCardIssueChange(tx, source.companyId); await notifyChatInboundWakeNotices(tx, source); const now = new Date(); const [comment] = await tx diff --git a/server/src/services/native-runtime/native-finalization-reconciler.ts b/server/src/services/native-runtime/native-finalization-reconciler.ts index af828edfb98..c5cf832982e 100644 --- a/server/src/services/native-runtime/native-finalization-reconciler.ts +++ b/server/src/services/native-runtime/native-finalization-reconciler.ts @@ -1,3 +1,4 @@ +import { notifyStatusCardIssueChange } from "../status-card-notifications.js"; import { notifyExternalAgentSettlement } from "../../modules/external-agents/index.js"; import { pendingNativeRunProjectionCondition } from "./native-run-projection.js"; import { nativeStatusReassessmentSourceIsSuperseded, reassessNativeStatusFromSource } from "./native-status-reassessment.js"; @@ -334,6 +335,7 @@ export async function claimNativeSessionResumptions(input: { nativePhaseUpdatedAt: now, updatedAt: now, }).where(eq(heartbeatRuns.id, row.run.id)); + await notifyStatusCardIssueChange(tx, run.companyId); await tx.update(issues).set({ executionRunId: run.id, executionAgentNameKey: agent.name.trim().toLowerCase() || null, diff --git a/server/src/services/native-runtime/native-review-participant.ts b/server/src/services/native-runtime/native-review-participant.ts index 573574dd19d..6391d506d5d 100644 --- a/server/src/services/native-runtime/native-review-participant.ts +++ b/server/src/services/native-runtime/native-review-participant.ts @@ -1,3 +1,4 @@ +import { notifyStatusCardIssueChange } from "../status-card-notifications.js"; import { agents, heartbeatRuns, @@ -240,6 +241,7 @@ export async function claimNativeReviewExecutionLock(db: Db, input: { if (!run || !["queued", "running"].includes(run.status) || !admitted || !expected || admitted.nativeReviewInteractionId !== expected.nativeReviewInteractionId || admitted.nativeReviewDecisionId !== expected.nativeReviewDecisionId) return false; + await notifyStatusCardIssueChange(tx, input.companyId); const claimed = await tx.update(issues).set({ executionRunId: input.runId, executionAgentNameKey: input.agentNameKey, diff --git a/server/src/services/native-runtime/native-workspace-export-recovery.ts b/server/src/services/native-runtime/native-workspace-export-recovery.ts index 41e51f2ddc7..0c3aac34799 100644 --- a/server/src/services/native-runtime/native-workspace-export-recovery.ts +++ b/server/src/services/native-runtime/native-workspace-export-recovery.ts @@ -1,3 +1,4 @@ +import { notifyStatusCardIssueChange } from "../status-card-notifications.js"; import { notifyNativeFinalizationWork } from "../native-finalization-notifications.js"; import { alias } from "drizzle-orm/pg-core"; import { notifyUnsafeWorkspaceExportRecovery, type UnsafeWorkspaceExportScope } from "../unsafe-workspace-export-notifications.js"; @@ -107,6 +108,7 @@ async function recoverCandidate(db: Db, candidate: { runId: string; companyId: s nativePhase: "result_accepted", nativePhaseUpdatedAt: now, resultJson: { ...run.resultJson, finalizationPhase: "result_accepted", failureCode: null, originalFailureCode: null, nextAttemptAt: null }, updatedAt: now, }).where(eq(heartbeatRuns.id, run.id)).returning(); + await notifyStatusCardIssueChange(tx, run.companyId); await tx.update(issues).set({ executionRunId: run.id, updatedAt: now }).where(eq(issues.id, issue.id)); await appendHeartbeatRunEvent(tx as unknown as Db, { companyId: run.companyId, agentId: run.agentId, runId: run.id, eventType: "workspace_export_omitted", stream: "system", level: "info", diff --git a/server/src/services/native-runtime/native-workspace-export-retry.ts b/server/src/services/native-runtime/native-workspace-export-retry.ts index 47de8c73cf4..5a79c9e782e 100644 --- a/server/src/services/native-runtime/native-workspace-export-retry.ts +++ b/server/src/services/native-runtime/native-workspace-export-retry.ts @@ -1,3 +1,4 @@ +import { notifyStatusCardIssueChange } from "../status-card-notifications.js"; import { notifyNativeFinalizationWork } from "../native-finalization-notifications.js"; import { notifyEnvironmentCleanup } from "../environment-cleanup-notifications.js"; import { randomUUID } from "node:crypto"; @@ -141,6 +142,7 @@ export async function retryNativeWorkspaceExport(input: { await tx.update(heartbeatRuns).set({ status: "running", finishedAt: null, error: null, errorCode: null, nativePhase: "result_accepted", nativePhaseUpdatedAt: now, resultJson: { ...current.run.resultJson, workspaceExportRetry: retry, finalizationPhase: "result_accepted", failureCode: null, originalFailureCode: null, nextAttemptAt: null }, updatedAt: now }).where(eq(heartbeatRuns.id, current.run.id)); + await notifyStatusCardIssueChange(tx, current.run.companyId); await tx.update(issues).set({ executionRunId: current.run.id, updatedAt: now }).where(eq(issues.id, current.issue.id)); // Reclaim the same lease only for control-plane copyback. The normal // finalization settlement will stop/retain it again on success or failure. diff --git a/server/src/services/native-runtime/native-workspace-reconstruction.ts b/server/src/services/native-runtime/native-workspace-reconstruction.ts index 5273b30216a..33e37704389 100644 --- a/server/src/services/native-runtime/native-workspace-reconstruction.ts +++ b/server/src/services/native-runtime/native-workspace-reconstruction.ts @@ -1,3 +1,4 @@ +import { notifyStatusCardIssueChange } from "../status-card-notifications.js"; import { DELIVERY_QUEUES, notifyDeliveryWork } from "../delivery-work-notifications.js"; import { notifyUnsafeWorkspaceExportRecovery } from "../unsafe-workspace-export-notifications.js"; import { and, eq, gt, inArray, ne, notInArray, or, sql } from "drizzle-orm"; @@ -187,6 +188,7 @@ export async function reconstructNativeWorkspace(input: { }).where(eq(issueRecoveryActions.id, source.action.id)); await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors); await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers); + await notifyStatusCardIssueChange(tx, input.companyId); await tx.update(issues).set({ status: "todo", unblockDescriptor: null, blockedTransitionAt: null, blockedOwnerNotifiedAt: null, updatedAt: now }).where(eq(issues.id, issue.id)); const publication = await persistActivity(db, { companyId: input.companyId, actorType: "user", actorId: input.actorId, diff --git a/server/src/services/native-runtime/paperclip-runner-tool-authority.ts b/server/src/services/native-runtime/paperclip-runner-tool-authority.ts index aa8970db19f..cf07e70eb7a 100644 --- a/server/src/services/native-runtime/paperclip-runner-tool-authority.ts +++ b/server/src/services/native-runtime/paperclip-runner-tool-authority.ts @@ -1,3 +1,4 @@ +import { notifyStatusCardIssueChange } from "../status-card-notifications.js"; import { requiredQualityReviewSchema } from "@paperclipai/shared"; import { priorRecoveryApiReceipt } from "../recovery-effect-receipts.js"; import { DELIVERY_QUEUES, notifyDeliveryWork } from "../delivery-work-notifications.js"; @@ -2059,6 +2060,7 @@ export class PaperclipRunnerToolAuthority { } const patch = prepareIssueMonitorUpdate(target, input.monitor, actor); await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors); + await notifyStatusCardIssueChange(tx, this.binding.companyId); const [updated] = await tx.update(issues).set({ ...patch, updatedAt: new Date() }) .where(and(eq(issues.id, target.id), eq(issues.companyId, this.binding.companyId))).returning(); if (!updated) throw notFound("Task not found"); diff --git a/server/src/services/plugin-workspaces.ts b/server/src/services/plugin-workspaces.ts index e2383dcfc74..ca04b081e48 100644 --- a/server/src/services/plugin-workspaces.ts +++ b/server/src/services/plugin-workspaces.ts @@ -1,3 +1,4 @@ +import { notifyStatusCardIssueChange } from "./status-card-notifications.js"; import { effectiveProjectWorkspacePolicy, isWorkspaceProvider } from "./project-workspace-provider.js"; import { releaseProviderRecoveryTransfers, stageProviderFreshWorkspace } from "./provider-fresh-workspace.js"; import { randomUUID, createHash } from "node:crypto"; @@ -418,6 +419,7 @@ export async function preparePluginWorkspace( }, }) .returning(); + await notifyStatusCardIssueChange(tx, task.companyId); await tx .update(issues) .set({ diff --git a/server/src/services/provider-fresh-workspace.ts b/server/src/services/provider-fresh-workspace.ts index fcc184e15f4..98a478a375d 100644 --- a/server/src/services/provider-fresh-workspace.ts +++ b/server/src/services/provider-fresh-workspace.ts @@ -1,3 +1,4 @@ +import { notifyStatusCardIssueChange } from "./status-card-notifications.js"; import { assertTaskWorkspaceExecutionAccess } from "./task-workspace-access.js"; import { isDeepStrictEqual } from "node:util"; import { createHash, randomUUID } from "node:crypto"; @@ -152,6 +153,7 @@ export async function reserveFreshPluginTaskWorkspace(input: { providerType: "adapter_managed", cwd: `/workspaces/${input.fresh.attemptId}`, metadata: { runnerTaskFreshWorkspace: input.fresh, runnerTaskWorkspaceOwner: input.owner }, }); + await notifyStatusCardIssueChange(tx, input.companyId); await tx.update(issues).set({ executionWorkspaceId: input.fresh.attemptId, executionWorkspacePreference: "reuse_existing", workspaceBindingRevision: task.workspaceBindingRevision + 1, workspacePendingSelection: task.workspacePendingSelection?.expectedBindingRevision === task.workspaceBindingRevision diff --git a/server/src/services/routines.ts b/server/src/services/routines.ts index 23787b45249..a831a3e489a 100644 --- a/server/src/services/routines.ts +++ b/server/src/services/routines.ts @@ -1,3 +1,4 @@ +import { notifyStatusCardIssueChange } from "./status-card-notifications.js"; import { notifyApprovalWork } from "./approval-work-notifications.js"; import { notifyDecisionTargetIssueRemoval } from "./decision-targets.js"; import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js"; @@ -1949,6 +1950,7 @@ export function routineService( if (createdIssue) { await notifyApprovalWork(txDb, createdIssue.companyId); await notifyDecisionTargetIssueRemoval(txDb, createdIssue.id); + await notifyStatusCardIssueChange(txDb, createdIssue.companyId); await txDb.delete(issues).where(eq(issues.id, createdIssue.id)); } const failureReason = error instanceof Error ? error.message : String(error); diff --git a/server/src/services/slack-conversation-lifecycle.ts b/server/src/services/slack-conversation-lifecycle.ts index 0fd213dc084..9e784ff005d 100644 --- a/server/src/services/slack-conversation-lifecycle.ts +++ b/server/src/services/slack-conversation-lifecycle.ts @@ -1,3 +1,4 @@ +import { notifyStatusCardIssueChange } from "./status-card-notifications.js"; import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js"; import { notifyUnsafeWorkspaceExportRecovery } from "./unsafe-workspace-export-notifications.js"; import { and, asc, desc, eq, gt, inArray, isNull, sql } from "drizzle-orm"; @@ -133,6 +134,7 @@ export async function settleExternalConversation(db: Db, companyId: string, issu await notifyUnsafeWorkspaceExportRecovery(tx, { companyId, issueId }); await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors); await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers); + await notifyStatusCardIssueChange(tx, companyId); await tx.update(issues).set({ status: "in_review", updatedAt: new Date() }) .where(and(eq(issues.id, issueId), eq(issues.companyId, companyId))); activity = (await persistActivity(tx as unknown as Db, { diff --git a/server/src/services/slack-conversation-state.ts b/server/src/services/slack-conversation-state.ts index dedaa8d6676..08f6c9c038b 100644 --- a/server/src/services/slack-conversation-state.ts +++ b/server/src/services/slack-conversation-state.ts @@ -1,3 +1,4 @@ +import { notifyStatusCardIssueChange } from "./status-card-notifications.js"; import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js"; import { notifyExternalConversationSettlement } from "./external-conversation-settlement-notifications.js"; import { chatConversations, issues, type Db } from "@paperclipai/db"; @@ -49,6 +50,7 @@ export async function resumeExternalConversation(tx: Db, companyId: string, issu await notifyExternalConversationSettlement(tx, { companyId, issueId }); // Only the verified idle convention may leave In Review automatically. // A real decision or delivery problem keeps the existing review semantics. + await notifyStatusCardIssueChange(tx, companyId); await tx.update(issues).set({ status: "todo", updatedAt: new Date() }) .where(and(eq(issues.companyId, companyId), eq(issues.id, issueId), sql`${externalConversationStateSql()} = 'waiting'`)); diff --git a/server/src/services/status-card-evaluation.ts b/server/src/services/status-card-evaluation.ts new file mode 100644 index 00000000000..143df32344a --- /dev/null +++ b/server/src/services/status-card-evaluation.ts @@ -0,0 +1,90 @@ +import { and, asc, eq, isNotNull, isNull } from "drizzle-orm"; +import { companies, statusCards, type Db } from "@paperclipai/db"; +import { logger } from "../middleware/logger.js"; +import { registerDeadlineWork } from "./deadline-work.js"; +import type { createDeliveryWorkCoordinator } from "./delivery-work-coordinator.js"; +import { DELIVERY_QUEUES } from "./delivery-work-notifications.js"; +import { instanceSettingsService } from "./instance-settings.js"; +import { issueService } from "./issues.js"; +import { queueIssueAssignmentWakeup, type IssueAssignmentWakeupDeps } from "./issue-assignment-wakeup.js"; +import { statusCardService } from "./status-cards.js"; + +/** Feature-owned generation and assignment. Existing issue services retain + * execution authority, durable wake dedupe and stalled-generation cleanup. */ +export function statusCardEvaluationService(db: Db, heartbeat: IssueAssignmentWakeupDeps) { + const cards = statusCardService(db); + const issues = issueService(db); + const settings = instanceSettingsService(db); + let reconstructReactive = true; + let allChanges = false; + const changes = new Set(); + let enabled = false; + + return { + changed(hints?: ReadonlySet) { + for (const hint of hints ?? ["*"]) { + if (hint === "*" || hint === "changes:*") allChanges = true; + else if (hint.startsWith("changes:")) changes.add(hint.slice("changes:".length)); + } + }, + async run(signal: AbortSignal) { + enabled = (await settings.getExperimental()).enableStatusCards === true; + if (!enabled || signal.aborted) return; + const reconstruct = reconstructReactive; + const evaluateAllChanges = allChanges; + const companiesChanged = new Set(changes); + changes.clear(); + reconstructReactive = false; + allChanges = false; + try { + const result = await cards.tickDueStatusCards(new Date(), evaluateAllChanges ? "changed" : reconstruct && companiesChanged.size === 0 ? "all" : companiesChanged); + const assignments = await Promise.allSettled(result.enqueued.map(async ({ cardId, generatingIssue }) => { + // Once generated, dispatch or cancel even if shutdown begins: leaving + // an unassigned claimed card is not a completed sweep. + try { + await queueIssueAssignmentWakeup({ + heartbeat, issue: generatingIssue, + reason: "status_card_update_assigned", + mutation: "status_card.scheduler_update_requested", + contextSource: "status_card_scheduler", + requestedByActorType: "system", taskKey: `status-card:${cardId}`, + rethrowOnError: true, + }); + } catch (error) { + await issues.update(generatingIssue.id, { status: "cancelled" }); + throw error; + } + })); + const failures = assignments.filter(result => result.status === "rejected"); + if (failures.length) throw new AggregateError(failures.map(result => result.reason), "Status-card assignment failed"); + if (result.evaluated || result.enqueued.length) { + logger.info({ evaluated: result.evaluated, enqueued: result.enqueued.length }, "status-card evaluation complete"); + } + } catch (error) { + reconstructReactive ||= reconstruct; + allChanges ||= evaluateAllChanges; + for (const companyId of companiesChanged) changes.add(companyId); + throw error; + } + }, + async next() { + if (!enabled) return null; + const [next] = await db.select({ at: statusCards.nextEvalAt }).from(statusCards) + .innerJoin(companies, eq(companies.id, statusCards.companyId)) + .where(and(eq(companies.status, "active"), isNull(statusCards.archivedAt), + isNull(statusCards.generatingIssueId), isNotNull(statusCards.nextEvalAt))) + .orderBy(asc(statusCards.nextEvalAt)).limit(1); + return next?.at?.getTime() ?? null; + }, + }; +} + +export function registerStatusCardEvaluationWork( + coordinator: ReturnType, + service: ReturnType, + canRun: () => boolean, +) { + return registerDeadlineWork(coordinator, DELIVERY_QUEUES.statusCardEvaluation, { + run: service.run, next: service.next, onSettled: service.changed, canRun, retryMs: 30_000, + }); +} diff --git a/server/src/services/status-card-finalization.ts b/server/src/services/status-card-finalization.ts index 7c7d260992e..5d7ee62b523 100644 --- a/server/src/services/status-card-finalization.ts +++ b/server/src/services/status-card-finalization.ts @@ -1,3 +1,4 @@ +import { notifyStatusCardEvaluation } from "./status-card-notifications.js"; import { and, eq, isNull } from "drizzle-orm"; import type { Db } from "@paperclipai/db"; import { statusCards, statusCardUpdates } from "@paperclipai/db"; @@ -36,6 +37,7 @@ export async function finalizeStatusCardsForStalledGeneration( ) { if (!STALLED_GENERATION_STATUSES.has(issue.status)) return []; + await notifyStatusCardEvaluation(dbOrTx, issue.companyId); const now = new Date(); const failureReason = failureReasonForIssue(issue); const cards = await dbOrTx diff --git a/server/src/services/status-card-notifications.ts b/server/src/services/status-card-notifications.ts new file mode 100644 index 00000000000..6fc12985847 --- /dev/null +++ b/server/src/services/status-card-notifications.ts @@ -0,0 +1,10 @@ +import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js"; + +/** Card/settings changes reconstruct deadlines; issue changes also evaluate reactive scope. */ +export function notifyStatusCardEvaluation(transaction: object, companyId?: string) { + return notifyDeliveryWork(transaction, DELIVERY_QUEUES.statusCardEvaluation, companyId ? `cards:${companyId}` : "*"); +} + +export function notifyStatusCardIssueChange(transaction: object, companyId?: string) { + return notifyDeliveryWork(transaction, DELIVERY_QUEUES.statusCardEvaluation, companyId ? `changes:${companyId}` : "changes:*"); +} diff --git a/server/src/services/status-card-update-engine.ts b/server/src/services/status-card-update-engine.ts index ff14d35c640..1f734227808 100644 --- a/server/src/services/status-card-update-engine.ts +++ b/server/src/services/status-card-update-engine.ts @@ -128,10 +128,8 @@ export function isWithinStatusCardActiveHours(policy: StatusCardRefreshPolicy, n } export function nextStatusCardEvaluationAt(policy: StatusCardRefreshPolicy, now: Date) { - if (policy.mode === "manual") return null; - const seconds = policy.mode === "interval" - ? (policy.intervalMinutes ?? 15) * 60 - : Math.min(policy.debounceSeconds ?? 60, 60); + if (policy.mode !== "interval") return null; + const seconds = (policy.intervalMinutes ?? 15) * 60; return new Date(now.getTime() + seconds * 1000); } @@ -172,3 +170,37 @@ export function evaluateStatusCardPolicy(input: { } return { action: "run" as const }; } + +/** A retry exists only while filtered changes remain. Use the owning policy's + * actual eligibility boundary instead of repeatedly rediscovering a closed gate. */ +export function nextStatusCardPolicyEligibility(input: { + policy: StatusCardRefreshPolicy; + now: Date; + lastChangeAt: Date; + completedUpdateTimes: Date[]; + tokensToday: number; +}): Date | null { + if (input.policy.mode === "manual") return null; + let at = input.now.getTime(); + if (input.tokensToday >= (input.policy.dailyTokenCap ?? 100_000)) { + const day = new Date(input.now); + day.setUTCHours(24, 0, 0, 0); + at = Math.max(at, day.getTime()); + } + if (input.policy.mode === "reactive") { + at = Math.max(at, input.lastChangeAt.getTime() + (input.policy.debounceSeconds ?? 60) * 1000); + const cap = input.policy.maxUpdatesPerHour ?? 6; + const recent = input.completedUpdateTimes.map(time => time.getTime()) + .filter(time => time > at - 3_600_000).sort((a, b) => a - b); + if (recent.length >= cap) at = Math.max(at, recent[recent.length - cap]! + 3_600_000 + 1); + } + if (isWithinStatusCardActiveHours(input.policy, new Date(at))) return new Date(at); + // Active-hour precision is one local minute. Walking UTC minutes also honors + // skipped/repeated local times at daylight-saving transitions. + const end = at + 48 * 3_600_000; + for (at = Math.floor(at / 60_000) * 60_000 + 60_000; at <= end; at += 60_000) { + if (isWithinStatusCardActiveHours(input.policy, new Date(at))) return new Date(at); + } + // Equal start/end describes an empty window; retain changes, awaiting config. + return null; +} diff --git a/server/src/services/status-cards.ts b/server/src/services/status-cards.ts index 4a7439b6aa5..84bbf7b2cd4 100644 --- a/server/src/services/status-cards.ts +++ b/server/src/services/status-cards.ts @@ -1,7 +1,9 @@ +import { notifyStatusCardEvaluation } from "./status-card-notifications.js"; import { createHash } from "node:crypto"; import { and, desc, eq, gte, inArray, isNotNull, isNull, lte, ne, or, sql, type SQL } from "drizzle-orm"; import { agents, + companies, costEvents, documentRevisions, documents, @@ -23,7 +25,7 @@ import { conflict, forbidden, notFound, unprocessable } from "../errors.js"; import { logger } from "../middleware/logger.js"; import { readBuiltInAgentMarker } from "../lib/built-in-agent-metadata.js"; import { builtInAgentService } from "./built-in-agents.js"; -import { companySearchService } from "./company-search.js"; +import { companySearchService, companySearchUpdatedWithinMs } from "./company-search.js"; import { issueService } from "./issues.js"; import { SUMMARIZER_BUILT_IN_KEY } from "./summary-slots.js"; import { @@ -34,6 +36,7 @@ import { extractIssueMentions, filterStatusCardChanges, nextStatusCardEvaluationAt, + nextStatusCardPolicyEligibility, STATUS_CARD_MAX_MENTIONED_ISSUES, statusCardChangesHash, statusCardFingerprintHash, @@ -44,6 +47,7 @@ import { type StatusCardActor = { agentId: string | null; userId: string | null }; type StatusCardWriter = { agentId: string | null; runId: string | null }; type StatusCardRow = typeof statusCards.$inferSelect; +type WatchedIssue = CompanySearchIssueSummary & { scopeExpiresAt?: Date | null }; type StatusCardReadOptions = { issueReadCondition?: SQL }; const TERMINAL_ISSUE_STATUSES = new Set(["done", "cancelled"]); @@ -151,9 +155,15 @@ export function statusCardService( db: Db, deps: { issuesSvc?: ReturnType } = {}, ) { - const builtIns = builtInAgentService(db); - const issuesSvc = deps.issuesSvc ?? issueService(db); + const issuesSvc = () => deps.issuesSvc ?? issueService(db); const searchSvc = companySearchService(db); + type Transaction = Parameters[0]>[0]; + function mutate(companyId: string | undefined, write: (tx: Transaction) => Promise) { + return db.transaction(async tx => { + await notifyStatusCardEvaluation(tx, companyId); + return write(tx); + }); + } async function readWatchedIssueVisibility(card: StatusCardRow, options: StatusCardReadOptions = {}) { if (card.queries.length === 0 && (card.mentionedIssueIds?.length ?? 0) === 0) { @@ -240,10 +250,10 @@ export function statusCardService( }; const agentId = actor.agentId; if (!agentId) { - return db.insert(statusCards).values(values).returning().then((rows) => rows[0]!); + return mutate(companyId, tx => tx.insert(statusCards).values(values).returning().then((rows) => rows[0]!)); } - return db.transaction(async (tx) => { + return mutate(companyId, async (tx) => { const author = await tx .select({ id: agents.id }) .from(agents) @@ -292,7 +302,7 @@ export function statusCardService( ...(input.refreshPolicy !== undefined ? { refreshPolicy: input.refreshPolicy, - nextEvalAt: card.archivedAt ? null : nextStatusCardEvaluationAt(input.refreshPolicy, now), + nextEvalAt: card.archivedAt ? null : input.refreshPolicy.mode === "reactive" && card.queries.length > 0 ? now : nextStatusCardEvaluationAt(input.refreshPolicy, now), } : {}), ...(archiveChanged && input.archived @@ -309,21 +319,21 @@ export function statusCardService( } : {}), }; - const next = await db.update(statusCards).set({ + const next = await mutate(card.companyId, tx => tx.update(statusCards).set({ ...values, ...(archiveChanged && input.archived ? { generatingIssueId: null, pendingChangeHash: null } : {}), - }).where(eq(statusCards.id, card.id)).returning().then((rows) => rows[0]!); + }).where(eq(statusCards.id, card.id)).returning().then((rows) => rows[0]!)); if (archiveChanged && input.archived && card.generatingIssueId) { const generationIssue = await db.select().from(issues).where(eq(issues.id, card.generatingIssueId)).then((rows) => rows[0] ?? null); if (generationIssue && !TERMINAL_ISSUE_STATUSES.has(generationIssue.status)) { - await issuesSvc.update(generationIssue.id, { status: "cancelled" }); + await issuesSvc().update(generationIssue.id, { status: "cancelled" }); } } return next; } async function remove(id: string) { - return db.delete(statusCards).where(eq(statusCards.id, id)).returning().then((rows) => rows[0] ?? null); + return mutate(undefined, tx => tx.delete(statusCards).where(eq(statusCards.id, id)).returning().then((rows) => rows[0] ?? null)); } async function listUpdates(cardId: string) { @@ -360,7 +370,7 @@ export function statusCardService( .then((rows) => rows[0] ?? null); if (override) return override.id; } - const builtIn = await builtIns.get(card.companyId, SUMMARIZER_BUILT_IN_KEY); + const builtIn = await builtInAgentService(db).get(card.companyId, SUMMARIZER_BUILT_IN_KEY); if (builtIn.status !== "ready" || !builtIn.agentId) { throw unprocessable("Summarizer built-in agent is not configured", { code: "summarizer_not_configured", @@ -391,7 +401,7 @@ export function statusCardService( let deduplicated = false; const createdAt = new Date(); - const created = await issuesSvc.create(card.companyId, { + const created = await issuesSvc().create(card.companyId, { title: `Compile status card: ${card.title ?? card.interestPrompt.slice(0, 80)}`, description: compileDescription(card, null, hash), status: "todo", @@ -409,16 +419,16 @@ export function statusCardService( // covers idempotency-key hits that resolve to a terminal task (done/cancelled) // as well as a `blocked` one that a manual re-kick is reviving. const reopened = deduplicated && (TERMINAL_ISSUE_STATUSES.has(created.status) || created.status === "blocked") - ? await issuesSvc.update(created.id, { status: "todo", assigneeAgentId: summarizerAgentId }) + ? await issuesSvc().update(created.id, { status: "todo", assigneeAgentId: summarizerAgentId }) : created; - const generationIssue = await issuesSvc.update(reopened!.id, { + const generationIssue = await issuesSvc().update(reopened!.id, { description: compileDescription(card, reopened!.id, hash), }); - const [nextCard] = await db + const [nextCard] = await mutate(card.companyId, tx => tx .update(statusCards) .set({ generatingIssueId: generationIssue!.id, state: "compiling", failureReason: null, updatedAt: createdAt }) .where(eq(statusCards.id, card.id)) - .returning(); + .returning()); return { card: nextCard!, generatingIssue: generationIssue!, @@ -464,7 +474,7 @@ export function statusCardService( if (card.archivedAt) throw unprocessable("Archived status cards cannot accept generation writes"); await assertSummarizerWriter(card, input.generationIssueId, actor); const now = new Date(); - return db.transaction(async (tx) => { + return mutate(card.companyId, async (tx) => { const current = await tx.select().from(statusCards).where(eq(statusCards.id, card.id)).then((rows) => rows[0] ?? null); if (!current || current.archivedAt || current.generatingIssueId !== input.generationIssueId) { throw conflict("Status-card compilation was superseded by a newer task"); @@ -581,14 +591,21 @@ export function statusCardService( } async function executeQueries(card: StatusCardRow, options: StatusCardReadOptions = {}) { - const issueMap = new Map(); + const issueMap = new Map(); for (const storedQuery of card.queries) { const query = companySearchQuerySchema.parse(storedQuery); const response = await searchSvc.search(card.companyId, query, { issueReadCondition: options.issueReadCondition, }); for (const result of response.results) { - if (result.type === "issue" && result.issue) issueMap.set(result.issue.id, result.issue); + if (result.type !== "issue" || !result.issue) continue; + const duration = companySearchUpdatedWithinMs(query.updatedWithin); + const expiry = duration === null ? null : new Date(new Date(result.issue.updatedAt).getTime() + duration + 1); + const previous = issueMap.get(result.issue.id); + issueMap.set(result.issue.id, { ...result.issue, scopeExpiresAt: previous + ? previous.scopeExpiresAt == null || expiry === null ? null + : new Date(Math.max(previous.scopeExpiresAt.getTime(), expiry.getTime())) + : expiry }); } } // Issues mentioned in the latest summary join the watched set alongside the @@ -598,7 +615,11 @@ export function statusCardService( (card.mentionedIssueIds ?? []).filter((issueId) => !issueMap.has(issueId)), options, ); - for (const issue of mentioned) issueMap.set(issue.id, issue); + for (const issue of mentioned) issueMap.set(issue.id, { ...issue, scopeExpiresAt: null }); + for (const id of card.mentionedIssueIds ?? []) { + const issue = issueMap.get(id); + if (issue) issue.scopeExpiresAt = null; + } const snapshot = [...issueMap.values()]; if (snapshot.length === 0) return snapshot; const latestHumanComments = await db @@ -622,6 +643,13 @@ export function statusCardService( return snapshot.map((issue) => ({ ...issue, latestHumanCommentAt: commentByIssueId.get(issue.id) ?? null })); } + function nextEvaluationAt(card: StatusCardRow, snapshot: WatchedIssue[], now: Date) { + const interval = nextStatusCardEvaluationAt(card.refreshPolicy, now); + if (card.refreshPolicy.mode !== "reactive" || !card.refreshPolicy.triggers.membershipChanges) return interval; + const expiries = snapshot.flatMap(issue => issue.scopeExpiresAt && issue.scopeExpiresAt > now ? [issue.scopeExpiresAt.getTime()] : []); + return expiries.length ? new Date(Math.min(...expiries)) : null; + } + async function requestRefresh(cardId: string, input: { full?: boolean; trigger?: "manual" | "interval" | "reactive" | "restore"; @@ -648,15 +676,15 @@ export function statusCardService( const changes = filterStatusCardChanges(allChanges, card.refreshPolicy); const trigger = input.trigger ?? "manual"; const forceRun = trigger === "manual" || trigger === "restore"; - const nextEvalAt = nextStatusCardEvaluationAt(card.refreshPolicy, now); + const nextEvalAt = nextEvaluationAt(card, snapshot, now); if (!forceRun && changes.length === 0) { - const [next] = await db.update(statusCards).set({ + const [next] = await mutate(card.companyId, tx => tx.update(statusCards).set({ pendingChangeCount: 0, pendingChangeHash: null, lastChangeAt: null, state: "active", nextEvalAt, - }).where(eq(statusCards.id, card.id)).returning(); + }).where(eq(statusCards.id, card.id)).returning()); return { card: next!, generatingIssue: null, alreadyGenerating: false, enqueued: false }; } @@ -678,13 +706,17 @@ export function statusCardService( manual: forceRun, }); if (decision.action !== "run") { - const [next] = await db.update(statusCards).set({ + const [next] = await mutate(card.companyId, tx => tx.update(statusCards).set({ pendingChangeCount: changes.length, pendingChangeHash, lastChangeAt, state: decision.action === "pause_budget" ? "paused_budget" : decision.action === "pause_hours" ? "paused_hours" : "active", - nextEvalAt: decision.action === "wait" && "dueAt" in decision ? decision.dueAt : nextEvalAt, - }).where(eq(statusCards.id, card.id)).returning(); + nextEvalAt: nextStatusCardPolicyEligibility({ + policy: card.refreshPolicy, now, lastChangeAt, + completedUpdateTimes: recent.filter(row => row.kind !== "compile" && row.finishedAt).map(row => row.startedAt), + tokensToday: Number(daily[0]?.tokens ?? 0), + }), + }).where(eq(statusCards.id, card.id)).returning()); return { card: next!, generatingIssue: null, alreadyGenerating: false, enqueued: false }; } @@ -707,7 +739,7 @@ export function statusCardService( : null; const fingerprintHash = statusCardFingerprintHash(fingerprint); let deduplicated = false; - const created = await issuesSvc.create(card.companyId, { + const created = await issuesSvc().create(card.companyId, { title: `${kind === "full" ? "Rebuild" : "Update"} status card: ${card.title ?? card.interestPrompt.slice(0, 80)}`, description: updateDescription({ card, generationIssueId: null, fingerprint, changes, kind, trigger, previousSummary, snapshot }), status: "todo", @@ -720,15 +752,15 @@ export function statusCardService( onDeduplicated: (reason) => { deduplicated = reason === "idempotency_key"; }, }); const reopened = deduplicated && TERMINAL_ISSUE_STATUSES.has(created.status) - ? await issuesSvc.update(created.id, { status: "todo", assigneeAgentId: summarizerAgentId }) + ? await issuesSvc().update(created.id, { status: "todo", assigneeAgentId: summarizerAgentId }) : created; - const generationIssue = await issuesSvc.update(reopened!.id, { + const generationIssue = await issuesSvc().update(reopened!.id, { description: updateDescription({ card, generationIssueId: reopened!.id, fingerprint, changes, kind, trigger, previousSummary, snapshot }), }); const priorGenerationPredicate = card.generatingIssueId ? eq(statusCards.generatingIssueId, card.generatingIssueId) : isNull(statusCards.generatingIssueId); - const [next] = await db.update(statusCards).set({ + const [next] = await mutate(card.companyId, tx => tx.update(statusCards).set({ generatingIssueId: generationIssue!.id, pendingChangeCount: changes.length, pendingChangeHash, @@ -736,17 +768,17 @@ export function statusCardService( state: "active", nextEvalAt, failureReason: null, - }).where(and(eq(statusCards.id, card.id), isNull(statusCards.archivedAt), or(isNull(statusCards.generatingIssueId), priorGenerationPredicate))).returning(); + }).where(and(eq(statusCards.id, card.id), isNull(statusCards.archivedAt), or(isNull(statusCards.generatingIssueId), priorGenerationPredicate))).returning()); if (!next) { const winner = await getById(card.id); if (!winner?.generatingIssueId) { if (!TERMINAL_ISSUE_STATUSES.has(generationIssue!.status)) { - await issuesSvc.update(generationIssue!.id, { status: "cancelled" }); + await issuesSvc().update(generationIssue!.id, { status: "cancelled" }); } throw conflict("Status-card refresh claim was lost"); } if (generationIssue!.id !== winner.generatingIssueId && !TERMINAL_ISSUE_STATUSES.has(generationIssue!.status)) { - await issuesSvc.update(generationIssue!.id, { status: "cancelled" }); + await issuesSvc().update(generationIssue!.id, { status: "cancelled" }); } const winnerIssue = await db.select().from(issues).where(eq(issues.id, winner.generatingIssueId)).then((rows) => rows[0] ?? null); return { card: winner, generatingIssue: winnerIssue, alreadyGenerating: true, enqueued: false, kind, changes }; @@ -765,26 +797,44 @@ export function statusCardService( return { card: next, generatingIssue: generationIssue!, alreadyGenerating: deduplicated, enqueued: true, kind, changes }; } - async function tickDueStatusCards(now = new Date()) { - const due = await db.select().from(statusCards).where(and(isNull(statusCards.archivedAt), isNull(statusCards.generatingIssueId), isNotNull(statusCards.nextEvalAt), lte(statusCards.nextEvalAt, now))); + async function tickDueStatusCards(now = new Date(), reactiveCompanies?: ReadonlySet | "all" | "changed") { + const reactiveScope = reactiveCompanies === "all" ? isNull(statusCards.nextEvalAt) : reactiveCompanies === "changed" ? sql`true` : reactiveCompanies?.size + ? inArray(statusCards.companyId, [...reactiveCompanies]) : sql`false`; + const due = await db.select({ card: statusCards }).from(statusCards) + .innerJoin(companies, eq(companies.id, statusCards.companyId)) + .where(and(eq(companies.status, "active"), isNull(statusCards.archivedAt), isNull(statusCards.generatingIssueId), + or(and(isNotNull(statusCards.nextEvalAt), lte(statusCards.nextEvalAt, now)), + and(reactiveScope, ne(statusCards.state, "error"), ne(statusCards.state, "compiling"), sql`${statusCards.refreshPolicy}->>'mode' = 'reactive'`, sql`jsonb_array_length(${statusCards.queries}) > 0`)))) + .then(rows => rows.map(row => row.card)); const enqueued: Array<{ cardId: string; generatingIssue: typeof issues.$inferSelect }> = []; let evaluated = 0; for (const candidate of due) { - const claimUntil = new Date(now.getTime() + 5 * 60 * 1000); - const [claimed] = await db.update(statusCards).set({ nextEvalAt: claimUntil }) - .where(and(eq(statusCards.id, candidate.id), isNull(statusCards.generatingIssueId), lte(statusCards.nextEvalAt, now))) - .returning(); - if (!claimed) continue; - evaluated += 1; - try { - const result = await requestRefresh(claimed.id, { trigger: claimed.refreshPolicy.mode === "reactive" ? "reactive" : "interval", now }); - if (result.enqueued && result.generatingIssue) enqueued.push({ cardId: claimed.id, generatingIssue: result.generatingIssue }); - } catch (err) { - logger.warn( - { err, cardId: claimed.id, companyId: claimed.companyId }, - "status card scheduled refresh failed", + // nextEvalAt is both a retry lease and an evaluation deadline. A source + // event may preempt a genuine future deadline, so serialize scheduler + // evaluation across connections without holding a row lock while the + // existing issue/card services commit through their own transactions. + await db.transaction(async lockTx => { + const [lock] = await lockTx.execute<{ acquired: boolean }>( + sql`select pg_try_advisory_xact_lock(hashtextextended(${candidate.id}, 0)) as acquired`, ); - } + if (!lock?.acquired) return; + const claimUntil = new Date(now.getTime() + 5 * 60 * 1000); + const [claimed] = await mutate(candidate.companyId, tx => tx.update(statusCards).set({ nextEvalAt: claimUntil }) + .where(and(eq(statusCards.id, candidate.id), isNull(statusCards.archivedAt), isNull(statusCards.generatingIssueId), + candidate.nextEvalAt ? eq(statusCards.nextEvalAt, candidate.nextEvalAt) : isNull(statusCards.nextEvalAt))) + .returning()); + if (!claimed) return; + evaluated += 1; + try { + const result = await requestRefresh(claimed.id, { trigger: claimed.refreshPolicy.mode === "reactive" ? "reactive" : "interval", now }); + if (result.enqueued && result.generatingIssue) enqueued.push({ cardId: claimed.id, generatingIssue: result.generatingIssue }); + } catch (err) { + logger.warn( + { err, cardId: claimed.id, companyId: claimed.companyId }, + "status card scheduled refresh failed", + ); + } + }); } return { evaluated, enqueued }; } @@ -796,7 +846,7 @@ export function statusCardService( await assertSummarizerWriter(card, input.generationIssueId, actor); if (card.queries.length === 0) throw conflict("Compile the status-card query before writing its summary"); const now = new Date(); - return db.transaction(async (tx) => { + return mutate(card.companyId, async (tx) => { const current = await tx.select().from(statusCards).where(eq(statusCards.id, card.id)).then((rows) => rows[0] ?? null); if (!current || current.archivedAt || current.generatingIssueId !== input.generationIssueId) { throw conflict("Status-card generation was superseded by a newer task"); @@ -816,9 +866,8 @@ export function statusCardService( const mentionedIssueIds = await resolveMentionedIssueIds(current.companyId, input.markdown); // Current watched membership: compiled-query matches plus the issues this // summary mentions. - const watchedNow = buildStatusCardFingerprint( - await executeQueries({ ...current, mentionedIssueIds }), - ); + const watchedSnapshot = await executeQueries({ ...current, mentionedIssueIds }); + const watchedNow = buildStatusCardFingerprint(watchedSnapshot); let snapshot: StatusCardFingerprint; if (payloadFingerprint) { // Keep the generation-time fingerprint as the change baseline: issues @@ -889,7 +938,9 @@ export function statusCardService( pendingChangeCount: 0, pendingChangeHash: null, lastChangeAt: null, - nextEvalAt: nextStatusCardEvaluationAt(current.refreshPolicy, now), + nextEvalAt: current.refreshPolicy.mode === "reactive" && + filterStatusCardChanges(diffStatusCardFingerprint(snapshot, watchedNow), current.refreshPolicy).length > 0 + ? now : nextEvaluationAt(current, watchedSnapshot, now), updatedAt: now, }).where(and(eq(statusCards.id, current.id), eq(statusCards.generatingIssueId, input.generationIssueId))).returning(); if (!next) throw conflict("Status-card generation was superseded by a newer task"); From 0fac69e1679270f48700d98cbae6cce8558d9554 Mon Sep 17 00:00:00 2001 From: Dotta Date: Fri, 30 Oct 2026 17:04:31 -0500 Subject: [PATCH 4/6] fix(server): scope run retention hints to affected companies Co-Authored-By: Paperclip --- server/src/__tests__/execution-control-work.test.ts | 4 ++-- server/src/services/execution-control-notifications.ts | 9 ++++++--- .../src/services/execution-control-reconciliation.ts | 2 +- server/src/services/execution-recovery-resolution.ts | 4 ++-- server/src/services/explicit-native-continuation.ts | 2 +- server/src/services/heartbeat.ts | 2 +- server/src/services/heartbeat/run-lifecycle.ts | 2 +- server/src/services/heartbeat/runtime-execution.ts | 4 ++-- server/src/services/issue-recovery-actions.ts | 10 +++++----- server/src/services/legacy-execution-recovery.ts | 2 +- .../native-runtime/native-failed-workspace-recovery.ts | 2 +- .../services/native-runtime/native-restart-recovery.ts | 2 +- .../services/native-runtime/native-run-finalizer.ts | 4 ++-- .../services/native-runtime/native-run-projection.ts | 2 +- .../services/native-runtime/native-safe-replacement.ts | 4 ++-- .../services/native-runtime/native-session-executor.ts | 6 +++--- .../services/native-runtime/native-superseded-retry.ts | 2 +- server/src/services/system-recovery.ts | 2 +- server/src/services/task-retry.ts | 2 +- 19 files changed, 35 insertions(+), 32 deletions(-) diff --git a/server/src/__tests__/execution-control-work.test.ts b/server/src/__tests__/execution-control-work.test.ts index 0046d8574aa..14cd9456eb5 100644 --- a/server/src/__tests__/execution-control-work.test.ts +++ b/server/src/__tests__/execution-control-work.test.ts @@ -383,11 +383,11 @@ describe.skipIf(!support.supported)("execution-control durable work", () => { const due = new Date(Date.now() + 120_000), earlier = new Date(Date.now() + 60_000); for (const at of [due, earlier]) { await writeExecutionControl(db, DELIVERY_QUEUES.executionFinalization, tx => tx.update(heartbeatRuns) - .set({ executionControlDeadlineAt: at }).where(eq(heartbeatRuns.id, run.id))); + .set({ executionControlDeadlineAt: at }).where(eq(heartbeatRuns.id, run.id)), { companyId }); await settle(); expect(c.nextWakeAt()).toBe(at.getTime()); } await writeExecutionControl(db, DELIVERY_QUEUES.executionFinalization, tx => tx.update(heartbeatRuns) - .set({ executionControlDeadlineAt: null }).where(eq(heartbeatRuns.id, run.id))); + .set({ executionControlDeadlineAt: null }).where(eq(heartbeatRuns.id, run.id)), { companyId }); await quiet(c); }); it("delivers a reconciled continuation when marking and resolving commit together", async () => { diff --git a/server/src/services/execution-control-notifications.ts b/server/src/services/execution-control-notifications.ts index c7b26a8bd72..eb2ea380c0b 100644 --- a/server/src/services/execution-control-notifications.ts +++ b/server/src/services/execution-control-notifications.ts @@ -2,8 +2,10 @@ import { notifyStatusCardIssueChange } from "./status-card-notifications.js"; import type { Db } from "@paperclipai/db"; import { DELIVERY_QUEUES, notifyDeliveryWork, type DeliveryQueue } from "./delivery-work-notifications.js"; -export async function notifyExecutionRunWork(transaction: object) { - await notifyStatusCardIssueChange(transaction); +export type ExecutionWorkScope = { companyId: string; agentId?: string; runId?: string; issueId?: string }; + +export async function notifyExecutionRunWork(transaction: object, scope?: ExecutionWorkScope) { + if (scope) await notifyStatusCardIssueChange(transaction, scope.companyId); for (const queue of [DELIVERY_QUEUES.executionStatus, DELIVERY_QUEUES.executionFinalization, DELIVERY_QUEUES.executionReplacement, DELIVERY_QUEUES.executionDisposition, DELIVERY_QUEUES.issueMonitors]) { await notifyDeliveryWork(transaction, queue); @@ -14,9 +16,10 @@ export function writeExecutionControl( db: Db | Parameters[0]>[0], queue: DeliveryQueue | readonly DeliveryQueue[], write: (tx: Db) => PromiseLike, + scope: ExecutionWorkScope | undefined, ): Promise { return db.transaction(async transaction => { - await notifyStatusCardIssueChange(transaction); + if (scope) await notifyStatusCardIssueChange(transaction, scope.companyId); await notifyDeliveryWork(transaction, DELIVERY_QUEUES.issueMonitors); for (const topic of typeof queue === "string" ? [queue] : queue) await notifyDeliveryWork(transaction, topic); return write(transaction as unknown as Db); diff --git a/server/src/services/execution-control-reconciliation.ts b/server/src/services/execution-control-reconciliation.ts index 2f2a435982d..621fd210cfc 100644 --- a/server/src/services/execution-control-reconciliation.ts +++ b/server/src/services/execution-control-reconciliation.ts @@ -196,7 +196,7 @@ export async function reconcileAbandonedExecutionControl( await notifyNativeEnvironmentSettlement(tx, { companyId: run.companyId, runId: run.id }); await notifyCancelledWorkspacePreservation(tx, { companyId: run.companyId, runId: run.id }); await notifyUnsafeWorkspaceExportRunChange(tx, run); - await notifyExecutionRunWork(tx); + await notifyExecutionRunWork(tx, { companyId: run.companyId }); const [updatedRun] = await tx .update(heartbeatRuns) .set({ diff --git a/server/src/services/execution-recovery-resolution.ts b/server/src/services/execution-recovery-resolution.ts index f3f5a3e4310..d56e7d926f1 100644 --- a/server/src/services/execution-recovery-resolution.ts +++ b/server/src/services/execution-recovery-resolution.ts @@ -546,7 +546,7 @@ export async function settleUnrecoverableExecutions( companyId: run.companyId, issueId: task.id, runId: run.id, }).limit(1).for("update"); if (!hold) return; - await notifyExecutionRunWork(tx); + await notifyExecutionRunWork(tx, { companyId: run.companyId }); const note = "Exact session recovery stopped. The recorded workspace recovery holds this execution without replaying its actions."; await tx.update(issueRecoveryActions).set({ status: "cancelled", outcome: "cancelled", resolvedAt: now, updatedAt: now, wakePolicy: null, monitorPolicy: null, @@ -654,7 +654,7 @@ export async function settleUnrecoverableExecutions( replay: "not_authorized", }, }); - await notifyExecutionRunWork(tx); + await notifyExecutionRunWork(tx, { companyId: run.companyId }); await tx .update(heartbeatRuns) .set({ executionStatusDeliveryId: randomUUID() }) diff --git a/server/src/services/explicit-native-continuation.ts b/server/src/services/explicit-native-continuation.ts index 074ffc3bdf5..084a25e457f 100644 --- a/server/src/services/explicit-native-continuation.ts +++ b/server/src/services/explicit-native-continuation.ts @@ -475,7 +475,7 @@ export async function admitExplicitNativeContinuation(input: { }).where(and(eq(agentWakeupRequests.companyId, companyId), eq(agentWakeupRequests.id, input.queuedCommentInterruptId), eq(agentWakeupRequests.agentId, agentId), eq(agentWakeupRequests.status, "deferred_issue_execution"))); if (cancelledStartupIds.size) await db.transaction(async tx => { - await notifyExecutionRunWork(tx); + await notifyExecutionRunWork(tx, { companyId, agentId }); for (const runId of cancelledStartupIds) { await tx.update(nativeRunFinalizations).set({ phase: "terminal_failure", failureCode: "native_startup_cancelled", nextAttemptAt: null, diff --git a/server/src/services/heartbeat.ts b/server/src/services/heartbeat.ts index eb655fdef25..a4cec7b81b0 100644 --- a/server/src/services/heartbeat.ts +++ b/server/src/services/heartbeat.ts @@ -3724,7 +3724,7 @@ export function heartbeatService( await writeExecutionControl(db, DELIVERY_QUEUES.executionFinalization, tx => tx .update(heartbeatRuns) .set({ executionControlDeadlineAt: null }) - .where(eq(heartbeatRuns.id, run.id))); + .where(eq(heartbeatRuns.id, run.id)), { companyId: run.companyId }); } nativeOwnershipHeld = nativeOwnershipHeld || diff --git a/server/src/services/heartbeat/run-lifecycle.ts b/server/src/services/heartbeat/run-lifecycle.ts index 00a27c1faf2..9b9eaad3061 100644 --- a/server/src/services/heartbeat/run-lifecycle.ts +++ b/server/src/services/heartbeat/run-lifecycle.ts @@ -1044,7 +1044,7 @@ export function createHeartbeatLifecycle(db: Db, dependencies: HeartbeatLifecycl if (previous && isHeartbeatRunTerminalStatus(status)) await notifyRetainedNativeSessionCleanup(tx, { companyId: previous.companyId, runId: previous.id }); if (isHeartbeatRunTerminalStatus(status)) await notifyDeliveryWork(tx, DELIVERY_QUEUES.accounting, companyId); - await notifyExecutionRunWork(tx); + await notifyExecutionRunWork(tx, companyId ? { companyId } : undefined); if (previous) await notifyChatRunMilestones(tx, { companyId: previous.companyId, runId: previous.id, issueId: previous.nativeIssueId ?? previous.issueId ?? readNonEmptyString(parseObject(previous.contextSnapshot).issueId) ?? undefined }); return write(tx); diff --git a/server/src/services/heartbeat/runtime-execution.ts b/server/src/services/heartbeat/runtime-execution.ts index b5b64abc948..bdbbfa07752 100644 --- a/server/src/services/heartbeat/runtime-execution.ts +++ b/server/src/services/heartbeat/runtime-execution.ts @@ -966,7 +966,7 @@ export async function executeHeartbeatRuntime(db: Db, input: HeartbeatRuntimeExe eq(heartbeatRuns.id, run.id), eq(heartbeatRuns.status, "running"), ), - )); + ), { companyId: run.companyId }); const workspaceFinalizeStatus = hasWorkspaceRestoreFailure(adapterResult.resultJson) ? "failed" : "succeeded"; await recordWorkspaceFinalize(workspaceFinalizeStatus); if (adapterResult.nativeFinalization) { @@ -1042,7 +1042,7 @@ export async function executeHeartbeatRuntime(db: Db, input: HeartbeatRuntimeExe eq(heartbeatRuns.id, run.id), eq(heartbeatRuns.status, "running"), ), - )); + ), { companyId: run.companyId }); if ( issueRef && context.resumeSessionGoalHeartbeat === true && diff --git a/server/src/services/issue-recovery-actions.ts b/server/src/services/issue-recovery-actions.ts index 5fb28bcb6d8..48fb001c0b4 100644 --- a/server/src/services/issue-recovery-actions.ts +++ b/server/src/services/issue-recovery-actions.ts @@ -392,7 +392,7 @@ export function issueRecoveryActionService(db: DbOrTransaction) { const [updated] = await writeExecutionControl(db, [DELIVERY_QUEUES.executionDisposition, DELIVERY_QUEUES.executionContinuation], tx => tx.update(issueRecoveryActions).set({ nextAction: input.nextAction, updatedAt: now }) .where(and(eq(issueRecoveryActions.id, existing.id), inArray(issueRecoveryActions.status, [...ACTIVE_RECOVERY_ACTION_STATUSES]), - sql`${issueRecoveryActions.evidence}->'workspaceOwnerStop' is null`)).returning()); + sql`${issueRecoveryActions.evidence}->'workspaceOwnerStop' is null`)).returning(), { companyId: input.companyId }); if (updated) return toReadModel(updated); } return existing; @@ -467,7 +467,7 @@ export function issueRecoveryActionService(db: DbOrTransaction) { ), ) .returning(); - }); + }, { companyId: input.companyId }); if (!exhausted) { return retryUpsertSourceScoped(input, retryCount); } @@ -539,7 +539,7 @@ export function issueRecoveryActionService(db: DbOrTransaction) { ), ) .returning(); - }); + }, { companyId: input.companyId }); if (!updated) { return retryUpsertSourceScoped(input, retryCount); } @@ -554,7 +554,7 @@ export function issueRecoveryActionService(db: DbOrTransaction) { .insert(issueRecoveryActions) .values(buildInsertValues(input, ownerType, now)) .returning(); - }); + }, { companyId: input.companyId }); return toReadModel(created!); } catch (error) { if (!isUniqueRecoveryActionConflict(error)) throw error; @@ -608,7 +608,7 @@ export function issueRecoveryActionService(db: DbOrTransaction) { }) .where(and(...predicates)) .returning(); - }); + }, { companyId: input.companyId }); return updated ? toReadModel(updated) : null; } diff --git a/server/src/services/legacy-execution-recovery.ts b/server/src/services/legacy-execution-recovery.ts index 427704de63f..c462cdf1fca 100644 --- a/server/src/services/legacy-execution-recovery.ts +++ b/server/src/services/legacy-execution-recovery.ts @@ -135,7 +135,7 @@ export async function terminalizeLegacyExecution(input: { }; if (issueId) await notifyUnsafeWorkspaceExportRecovery(tx, { companyId: current.companyId, issueId }); await notifyInstructionCollection(tx, { companyId: current.companyId, runId: current.id }); - await notifyExecutionRunWork(tx); + await notifyExecutionRunWork(tx, { companyId: current.companyId }); let [updated] = await tx .update(heartbeatRuns) .set({ diff --git a/server/src/services/native-runtime/native-failed-workspace-recovery.ts b/server/src/services/native-runtime/native-failed-workspace-recovery.ts index 5c3268f6afc..a1c6c914b33 100644 --- a/server/src/services/native-runtime/native-failed-workspace-recovery.ts +++ b/server/src/services/native-runtime/native-failed-workspace-recovery.ts @@ -65,7 +65,7 @@ export async function preserveNativeFailedWorkspace(db: Db, selected: Pick>'{systemRecovery,revision}' = ${String(previous.revision)}`)) .returning({ id: issueRecoveryActions.id }); return Boolean(updated); - }); + }, { companyId: source.companyId }); } export async function loadSystemRecovery(db: Db, source: WorkspaceRecoverySourceIdentity) { diff --git a/server/src/services/task-retry.ts b/server/src/services/task-retry.ts index e9f0bd4edbf..17075c34eb9 100644 --- a/server/src/services/task-retry.ts +++ b/server/src/services/task-retry.ts @@ -334,7 +334,7 @@ export async function requestTaskRetry(input: { details: { requestId: active?.id ?? id, sourceRunId: latest.id, attachedToExistingWork: Boolean(active || busy) } }); return { requestId: active?.id ?? id, accepted: true, stale: false, observedRunId: latest.id, chatActionId: intent.chatActionId }; - }); + }, { companyId: input.companyId }); } /** Chat owns admission and authorization. The task receipt follows that From 938e22aa21cbd6c887f2451d9a339b2db44d11f4 Mon Sep 17 00:00:00 2001 From: Dotta Date: Fri, 30 Oct 2026 17:47:09 -0500 Subject: [PATCH 5/6] test: prove timer deadlines survive settled drain holds Expect committed producers to release idle admission while retaining a resume deadline, then verify actual dispatch and final quiet after resume. Co-Authored-By: Paperclip --- server/src/__tests__/agent-timer-deadlines.test.ts | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/server/src/__tests__/agent-timer-deadlines.test.ts b/server/src/__tests__/agent-timer-deadlines.test.ts index 4c4c0d324a9..f925fb8dfd7 100644 --- a/server/src/__tests__/agent-timer-deadlines.test.ts +++ b/server/src/__tests__/agent-timer-deadlines.test.ts @@ -597,8 +597,10 @@ describe.skipIf(!support.supported)("agent timer owned deadlines", () => { await agentService(db).update(f.agent.id, { runtimeConfig: { heartbeat: { enabled: true, intervalSec: 600 } } }); await vi.advanceTimersByTimeAsync(2000); - - expect(idleWorkSnapshot().active).toBe(1); + // The producer committed: retain its resume deadline without keeping the + // instance busy throughout the drain. Uncertain writes still hold admission. + expect(idleWorkSnapshot().active).toBe(0); + expect(coordinator!.nextWakeAt()).toBeGreaterThan(Date.now()); const select = vi.spyOn(db, "select"), execute = vi.spyOn(db, "execute"), transaction = vi.spyOn(db, "transaction"); try { await vi.advanceTimersByTimeAsync(60_000); @@ -608,6 +610,11 @@ describe.skipIf(!support.supported)("agent timer owned deadlines", () => { stopTaskDrain(); await vi.advanceTimersByTimeAsync(1000); await vi.waitFor(() => expect(coordinator!.nextWakeAt()).toBe(now.getTime() + 600_000), { timeout: 3000 }); + await vi.advanceTimersByTimeAsync(coordinator!.nextWakeAt()! - Date.now()); + await vi.waitFor(() => expect(f.enqueueWakeup).toHaveBeenCalledOnce()); + expect(f.enqueueWakeup).toHaveBeenCalledWith(f.agent.id, expect.objectContaining({ source: "timer", reason: "heartbeat_timer" })); + await agentService(db).update(f.agent.id, { runtimeConfig: { heartbeat: { enabled: false } } }); + await quiet(); }); it("does not inspect timer rows during startup drain, then reconstructs them on resume", async () => { From a12f68ef134d10b69357ec6d193ccf56b0db971f Mon Sep 17 00:00:00 2001 From: Dotta Date: Fri, 30 Oct 2026 17:50:38 -0500 Subject: [PATCH 6/6] fix: preserve scoped status-card work across mutations and recovery Complete transactional source coverage, cached feature gating, startup scope reconstruction, off-page relative query expiry and busy-lock retry. Verify producer, persistence, failure, budget resume, compatibility and startup paths with isolated PostgreSQL and focused tests. Co-Authored-By: Paperclip --- doc/plans/2026-10-30-status-cards-sleep.md | 113 ++++++++-- .../__tests__/company-search-service.test.ts | 14 +- .../server-startup-feedback-export.test.ts | 30 +++ .../status-card-evaluation-work.test.ts | 209 +++++++++++++++++- .../status-card-notifications.test.ts | 31 +++ .../modules/wake-queue/adapters/postgres.ts | 2 + .../adapters/queued-comment-postgres.ts | 3 + server/src/routes/execution-workspaces.ts | 6 +- server/src/services/agent-conversations.ts | 3 + server/src/services/budgets.ts | 3 + .../telegram/attachment-failure.ts | 2 + server/src/services/company-search.ts | 15 +- server/src/services/documents.ts | 5 + server/src/services/execution-workspaces.ts | 5 +- server/src/services/heartbeat/queue.ts | 3 + server/src/services/heartbeat/scheduling.ts | 6 + .../src/services/issue-thread-interactions.ts | 63 +++--- server/src/services/issues.ts | 34 ++- ...ative-status-reassessment-notifications.ts | 4 +- server/src/services/projects.ts | 3 + server/src/services/status-card-evaluation.ts | 14 +- server/src/services/status-cards.ts | 53 ++--- server/src/services/task-watchdogs.ts | 6 +- server/src/services/workspace-runtime.ts | 22 +- 24 files changed, 524 insertions(+), 125 deletions(-) create mode 100644 server/src/__tests__/status-card-notifications.test.ts diff --git a/doc/plans/2026-10-30-status-cards-sleep.md b/doc/plans/2026-10-30-status-cards-sleep.md index dcd6c5ca226..2af51ff65cf 100644 --- a/doc/plans/2026-10-30-status-cards-sleep.md +++ b/doc/plans/2026-10-30-status-cards-sleep.md @@ -1,33 +1,102 @@ # Status-card evaluation deadlines -Baseline: `76fde891ccc29283b3f5c8dcdce089c5b3d937d1`. +Status-card evaluation uses existing cards, updates and issues as durable truth. +`index.ts` registers a typed worker. The status-card owner controls evaluation, +generation claims, assignment through the ordinary issue wake API, and failed +assignment cancellation. Shared delivery infrastructure controls transaction +settlement, coalescing, retries, admission drain, standby and shutdown. -Status-card ownership covers evaluation policy, card claims, generation, ordinary assignment wakeups, and failed-assignment cancellation. `index.ts` registers a typed capability. Shared delivery/deadline infrastructure owns transaction settlement, coalescing, admission/drain, retries and shutdown. Existing cards, updates and issues remain durable truth; no timer table or sleep-safety exemption is introduced. +Manual cards have no automatic obligation. Interval cards retain the requested +evaluation cadence. Reactive cards evaluate committed source changes and retain +only an actual debounce, hourly rate, daily budget, active-hour or saved-query +membership deadline. Unchanged cards disarm. A changed experimental setting +invalidates the worker's cached feature gate. Other source writes do not reread +settings. Startup reconstructs every disarmed reactive scope together with any +concurrent company-scoped hint, without moving saved eligibility/retry deadlines. -Manual cards have no automatic obligation. Interval cards retain their requested evaluation cadence. Reactive cards evaluate committed source changes, retain the actual debounce/rate/budget/active-hours deadline while relevant changes remain, and disarm once unchanged. Saved relative-search windows retain their actual membership expiry; the duration interpretation stays with company search. Startup reconstructs existing work without advancing business timestamps. +Company search reports the earliest matching `updatedWithin` expiry through an +optional internal callback in its existing aggregate query. It includes rows +outside the returned page: an expiring priority-ranked row ahead of an offset +can change the watched set. The callback exposes only a normalized deadline; +ordinary search consumers and API responses retain their existing contract. +Overlapping query scopes can require an evaluation when one query expires even +if another still retains the same issue; no generation occurs without a filtered +card change. No recurring scan replaces the expired query obligation. -Scheduled reactive source events can preempt a future evaluation. A feature-owned transaction advisory lock serializes scheduler evaluation across connections while existing card/issue services perform their own committed writes. The lock holder has no card row lock, avoiding self-deadlock. Existing next-evaluation CAS, generating-issue ownership and issue idempotency keys remain required, including for manual refreshes and older server instances that do not use the advisory lock. The persisted five-minute claim remains the failure retry after a scheduled refresh cannot finish. Internal timers do not establish a Cloud wake alarm. +## Claims and recovery -## Producer audit (in progress) +A feature-owned PostgreSQL transaction advisory lock serializes scheduled +per-card evaluation across connections. Its transaction holds no card row lock +while existing issue/card services commit through their own transactions. The +lock remains held through actual refresh settlement. Existing card deadline CAS, +generating-issue ownership and issue idempotency keys remain in force. Manual +refresh and older instances continue to use those existing guards. A busy lock +retains a company-scoped retry; consuming a hint is not successful evaluation. +A failed scheduled refresh retains the existing five-minute durable claim. -| Source | Owning committed boundary | State | -| --- | --- | --- | -| Card create/config/archive/delete, compile, query/summary writeback, evaluation | status-card mutation transactions | wired; ordinary lifecycle exercised | -| Experimental feature flag | instance settings update transaction | wired; disable exercised | -| Company pause/archive/reactivate/delete | company owning transactions | wired; lifecycle acceptance remains | -| Ordinary issue create/update/remove | issue service transactions | wired; create/change exercised | -| Comments create/delete/tombstone | common insert and issue mutation transactions | wired; comment acceptance remains | -| Review state / chat-run milestones | issue-review-state notification boundary | wired; exact caller coverage audit remains | -| Execution-control writes/run settlement | execution-control transaction helpers | wired; exact caller coverage audit remains | -| Agent deletion/membership reassignment, issue tree transitions, title/prefix | owner transaction | wired; representative acceptance remains | -| Workspace selection/binding, plugin/provider workspace, native recovery/reclaim, active watchdog, retry | owner transactions where uncovered by shared notifications | partially wired; remaining per-write audit required | -| Raw workspace cleanup/routes, conversation activation/settlement, queue paths | direct writes and adjacent owner notifications | exact coverage incomplete; not acceptance-ready | -| Imports/independent writers/replicas | supported service writes and startup reconstruction | full ingress contract audit remains | +Once a generation exists, the normal issue assignment authority remains its +owner. Assignment failure cancels that issue and finalizes the card's failed +ledger entry. The user's ordinary refresh action can retry it. Restart recovery +must not repeat an already completed generation. No new timer table, automatic +write replay, execution-authority exemption or sleep-safety relaxation is added. -Process-local notifications cover writes using the installed database owner. They do not notify an unrelated SQL writer or another server process; those supported paths need explicit ingress or reconstruction coverage. No claim of cross-replica delivery, host suspension, or fleet savings is made by the current tests. +## Supported producer coverage -## Acceptance evidence and remaining gates +All hints are registered before the relevant write within its owning transaction. +Nested service writes share outer commit ownership. Company-scoped hints batch +reactive reconciliation; ordinary execution writes without known scope retain +all existing execution consumers without sending an instance-wide card hint. -The original status-card PostgreSQL suite passed25/25. New real PostgreSQL tests exercise ordinary card compilation/writeback, source-change debounce, persisted assignment at a controlled provider boundary, result persistence, quiet SQL after completion, earlier/cancelled deadlines, nested outer commit/rollback and drain/resume. The provider boundary is simulated; no model runs. Additional race, restart, failure, active-hour/rate/budget persistence, multi-company and producer-class coverage remain before PR delivery. +| Producer | Notification boundary and coverage | +| --- | --- | +| Card create, configuration, archive/restore, delete, compile, query/summary result, evaluation claim | Status-card mutation transactions; stalled issue finalization signals before releasing the card claim | +| Experimental status-card setting | Instance experimental-settings transaction; full reconstruction is appropriate for this instance-wide setting | +| Company pause/archive/reactivate/delete | Company lifecycle/deletion transactions and budget-owned company hard-stop/resume transactions | +| Issue create/update/delete and label changes | Ordinary issue transactions; label deletion signals its known company before cascades; title and prefix changes signal their existing owner transactions | +| Issue checkout/adoption/release/reset and environment-selection recency | Existing issue transaction, or a transaction around the previously standalone write; company comes from the existing issue/run read | +| Comment create, edit, delete, tombstone, import and recency | Common comment insert; outer root comment transaction; queued-comment and Telegram edit transactions; existing metadata-upgrade path; remove/tombstone transactions | +| Issue import, comment import, document import | One notification per affected company in the existing batch transaction; no per-row notification lookup | +| Document create/update/restore/delete and search membership | Document owner transactions; only the existing company is passed, without copying search predicates into producers | +| Review, interaction and execution transitions | Existing issue-review-state and native-status-reassessment boundaries; scoped execution-control helpers; interaction recency touches own a transaction when called after settlement | +| Run admission, retries, monitors, watchdog/recovery and native settlement/reclaim | Existing scoped review/milestone/status/execution-control hooks; explicit notifications at the uncovered issue execution bind, legacy claim, monitor claim/release and review-participant recovery transaction | +| Agent deletion/access reassignment and project deletion/privacy cascades | Existing company-scoped owner transactions before foreign-key or recursive issue changes | +| Workspace selection/binding, plugin/provider workspace and cleanup/audit touches | Existing owner transactions, including standalone cleanup and audit recency writes now enclosed with their intent | +| Skill-test harness expiry and task-watchdog fingerprint touch | Existing owner mutation transaction; no feature policy is moved into the writer | -Shared deadline regressions reproduced and corrected a settled-write admission hold during drain. The correction releases only settled holds, retains an in-memory resume wake, and leaves ambiguous transaction intent fenced by the database coordinator. Unit coverage includes first write during drain, failure entering drain, preserved future deadlines, warm standby and ambiguous commit settlement. +The audit includes direct issue/comment mutations, raw recursive privacy updates, +bulk insert helpers, label/project/agent cascades, linked document search content +and timestamp-dependent search membership. Metadata-only writes that do not +change the fingerprint, matching fields or visibility do not create an evaluation +obligation: examples include run identity linkage, workspace IDs without a +recency change, document locks and source-trust receipts without searched content +changes. Routine and status-summary documents that have no issue-document link +do not change issue search membership. + +This follows the existing [single-active-writer contract](../idle-sleep-safety.md). +Supported services, import/recovery writers, caller-owned transactions and startup +reconstruction participate. Independent SQL writers or another active server +process require explicit wake ingress; process-local hints do not establish +cross-replica notification delivery. The advisory lock supplies competing +scheduler exclusion, not cross-process event transport. Internal deadlines do +not establish a Cloud wake alarm or prove host/database suspension. + +## Verification + +Real PostgreSQL acceptance uses ordinary card create/compile and authorized +summarizer writeback. The assignment boundary is controlled: it persists a wake, +and the test explicitly performs the authorized result write; no model runs. +Checks cover empty/manual/reactive/disabled quiet SQL, first source change, +debounce and durable wake/result, earlier/cancelled deadlines, outer commit and +rollback, drain/resume, competing connections, busy locks, restart eligibility, +active hours/rate/budget gates, multiple companies, budget-owned company resume, +ignored activity, document/comment membership, relative query expiry including +an off-page candidate, failed assignment cleanup/manual retry and the persisted +five-minute internal-failure retry across restart/drain. + +Shared coordinator regressions also cover warm standby, ambiguous transaction +intent, failure entering drain and different domain/coordinator gates. The paused +worker releases only settled holds and retains a future in-memory resume wake. +An expired saved deadline cannot schedule an immediate skipped-attempt loop. +Existing startup, status-card, company-search, execution and monitor suites cover +the production wiring and unchanged consumers. Exact commands and results belong +in the PR verification record; this document does not claim deployed savings. diff --git a/server/src/__tests__/company-search-service.test.ts b/server/src/__tests__/company-search-service.test.ts index 5d9e53260fb..7dffe3d6a90 100644 --- a/server/src/__tests__/company-search-service.test.ts +++ b/server/src/__tests__/company-search-service.test.ts @@ -1,3 +1,4 @@ +import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js"; import { randomUUID } from "node:crypto"; import { sql } from "drizzle-orm"; import { afterAll, afterEach, beforeAll, describe, expect, it } from "vitest"; @@ -28,12 +29,13 @@ import { import { issueReadSqlCondition, projectReadSqlCondition } from "../services/authorization.js"; import { issueService } from "../services/issues.js"; -const embeddedPostgresSupport = await getEmbeddedPostgresTestSupport(); +const externalDatabaseUrl = resolveExternalTestDatabaseUrl("PAPERCLIP_COMPANY_SEARCH_TEST_DATABASE_URL"); +const embeddedPostgresSupport = externalDatabaseUrl ? { supported: true } : await getEmbeddedPostgresTestSupport(); const describeEmbeddedPostgres = embeddedPostgresSupport.supported ? describe : describe.skip; if (!embeddedPostgresSupport.supported) { console.warn( - `Skipping embedded Postgres company search tests on this host: ${embeddedPostgresSupport.reason ?? "unsupported environment"}`, + `Skipping embedded Postgres company search tests on this host: ${("reason" in embeddedPostgresSupport ? embeddedPostgresSupport.reason : "unsupported environment")}`, ); } @@ -80,8 +82,11 @@ describeEmbeddedPostgres("companySearchService", () => { let tempDb: Awaited> | null = null; beforeAll(async () => { - tempDb = await startEmbeddedPostgresTestDatabase("paperclip-company-search-"); - db = createDb(tempDb.connectionString); + if (!externalDatabaseUrl) tempDb = await startEmbeddedPostgresTestDatabase("paperclip-company-search-"); + db = createDb(externalDatabaseUrl ?? tempDb!.connectionString, { maxConnections: 6 }); + // External suites can reuse this dedicated database sequentially. Match + // the fresh embedded fixture before this suite creates its own rows. + if (externalDatabaseUrl) await db.execute(sql`TRUNCATE TABLE companies CASCADE`); svc = companySearchService(db); await db.execute(sql.raw("CREATE EXTENSION IF NOT EXISTS pg_trgm")); }, 20_000); @@ -100,6 +105,7 @@ describeEmbeddedPostgres("companySearchService", () => { }); afterAll(async () => { + await db?.$client.end({ timeout: 1 }); await tempDb?.cleanup(); }); diff --git a/server/src/__tests__/server-startup-feedback-export.test.ts b/server/src/__tests__/server-startup-feedback-export.test.ts index bbb020c865d..3008ee21f9c 100644 --- a/server/src/__tests__/server-startup-feedback-export.test.ts +++ b/server/src/__tests__/server-startup-feedback-export.test.ts @@ -1,3 +1,4 @@ +import { statusCardEvaluationService, registerStatusCardEvaluationWork } from "../services/status-card-evaluation.js"; import { nativeHarnessBackupExpirationService } from "../services/native-runtime/native-harness-backup-expiration.js"; import { skillVersionRetentionService } from "../services/skill-version-retention.js"; import { approvalQueueSeedingService } from "../services/approval-queue-seeding-work.js"; @@ -527,6 +528,13 @@ vi.mock("../services/native-runtime/native-question-bridge.js", () => ({ validateNativeQuestionResponseInput: vi.fn(), })); +vi.mock("../services/status-card-evaluation.js", () => ({ + statusCardEvaluationService: vi.fn(() => ({ run: vi.fn(async () => {}), next: vi.fn(async () => null), changed: vi.fn() })), + registerStatusCardEvaluationWork: vi.fn((_coordinator, service, canRun) => ({ + ready: canRun() ? service.run(new AbortController().signal) : Promise.resolve(), wake: vi.fn(), stop: vi.fn(async () => {}), + })), +})); + vi.mock("../services/secret-proposals.js", () => ({ createSecretProposalsService: vi.fn(() => ({ expiryDeadlines: vi.fn(async () => []), @@ -790,6 +798,28 @@ describe("startServer feedback export wiring", () => { expect(deliveryWorkErrors.filter(error => error.queue === DELIVERY_QUEUES.skillVersionRetention)).toEqual([]); }); + it.each([false, true])("registers status-card evaluation only with maintenance and never ticks it from the heartbeat loop (%s)", async enabled => { + loadConfigMock.mockReturnValue(buildTestConfig({ heartbeatSchedulerEnabled: enabled, heartbeatSchedulerIntervalMs: 1000 })); + let tick: (() => void) | undefined; + const timer = vi.spyOn(globalThis, "setInterval").mockImplementation(((callback: () => void, interval: number) => { + if (interval === 1000) tick = callback; + return 1 as unknown as ReturnType; + }) as typeof setInterval); + try { + await startServer(); await settleStartupHousekeeping(); + expect(registerStatusCardEvaluationWork).toHaveBeenCalledTimes(enabled ? 1 : 0); + expect(statusCardEvaluationService).toHaveBeenCalledTimes(enabled ? 1 : 0); + if (enabled) { + const service = vi.mocked(statusCardEvaluationService).mock.results.at(-1)!.value; + expect(service.run).toHaveBeenCalledOnce(); + expect(tick).toBeDefined(); + for (let i = 0; i < 3; i++) { tick!(); await settleStartupHousekeeping(); } + expect(service.run).toHaveBeenCalledOnce(); + } + expect(deliveryWorkErrors.filter(error => error.queue === DELIVERY_QUEUES.statusCardEvaluation)).toEqual([]); + } finally { timer.mockRestore(); } + }); + it.each([false, true])("registers response recovery and its shutdown subscription only when maintenance is enabled (%s)", async enabled => { loadConfigMock.mockReturnValue(buildTestConfig({ heartbeatSchedulerEnabled: enabled })); await startServer(); await settleStartupHousekeeping(); diff --git a/server/src/__tests__/status-card-evaluation-work.test.ts b/server/src/__tests__/status-card-evaluation-work.test.ts index 427ad3385eb..4f369cd76e9 100644 --- a/server/src/__tests__/status-card-evaluation-work.test.ts +++ b/server/src/__tests__/status-card-evaluation-work.test.ts @@ -1,7 +1,9 @@ +import { budgetService } from "../services/budgets.js"; +import { documentService } from "../services/documents.js"; import { randomUUID } from "node:crypto"; import { eq, sql } from "drizzle-orm"; import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; -import { activityLog, agentWakeupRequests, agents, companies, companyMemberships, createDb, documents, +import { activityLog, agentWakeupRequests, agents, approvals, budgetIncidents, budgetPolicies, costEvents, companies, companyMemberships, createDb, documents, heartbeatRuns, instanceSettings, issueComments, issues, statusCards, statusCardUpdates, documentRevisions } from "@paperclipai/db"; import { companySearchQuerySchema, statusCardRefreshPolicySchema } from "@paperclipai/shared"; import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js"; @@ -22,6 +24,7 @@ const support = external ? { supported: true } : await getEmbeddedPostgresTestSu let draining = false; let companyId: string, agentId: string; const errors: unknown[] = []; + let compiledQuery = companySearchQuerySchema.parse({ scope: "issues", status: ["blocked"] }); const actor = { agentId: null, userId: "board-user" }; const policy = (mode: "manual" | "reactive" | "interval" = "reactive") => statusCardRefreshPolicySchema.parse({ mode, debounceSeconds: 60, intervalMinutes: 15, @@ -29,7 +32,8 @@ const support = external ? { supported: true } : await getEmbeddedPostgresTestSu // Controlled provider boundary: assignment persists a queued wake, and tests // explicitly run the authorized summarizer writeback. No model/provider runs. const wakeup = vi.fn(async (id: string, options: { payload?: Record | null; reason?: string | null }) => { - return db.insert(agentWakeupRequests).values({ companyId, agentId: id, source: "assignment", + const [agent] = await db.select({ companyId: agents.companyId }).from(agents).where(eq(agents.id, id)); + return db.insert(agentWakeupRequests).values({ companyId: agent!.companyId, agentId: id, source: "assignment", reason: options.reason, payload: options.payload, status: "queued" }).returning(); }); @@ -38,7 +42,8 @@ const support = external ? { supported: true } : await getEmbeddedPostgresTestSu db = createDb(external ?? temporary!.connectionString, { maxConnections: 6, idleTimeoutSeconds: 0, maxLifetimeSeconds: 0 }); }, 30_000); beforeEach(async () => { - companyId = randomUUID(); agentId = randomUUID(); draining = false; errors.length = 0; wakeup.mockClear(); + companyId = randomUUID(); agentId = randomUUID(); draining = false; errors.length = 0; wakeup.mockReset(); + compiledQuery = companySearchQuerySchema.parse({ scope: "issues", status: ["blocked"] }); await db.insert(companies).values({ id: companyId, name: "Status", issuePrefix: `S${companyId.slice(0, 6)}` }); await db.insert(agents).values({ id: agentId, companyId, name: "Summarizer", role: "general", status: "idle", adapterType: "process", adapterConfig: {} }); await instanceSettingsService(db).updateExperimental({ enableStatusCards: true }); @@ -48,6 +53,7 @@ const support = external ? { supported: true } : await getEmbeddedPostgresTestSu afterEach(async () => { await coordinator?.stop(); coordinator = undefined; vi.useRealTimers(); vi.restoreAllMocks(); + await db.delete(budgetIncidents); await db.delete(budgetPolicies); await db.delete(approvals); await db.delete(costEvents); await db.delete(agentWakeupRequests); await db.delete(statusCardUpdates); await db.delete(statusCards); await db.delete(documentRevisions); await db.delete(documents); await db.delete(activityLog); await db.delete(issueComments); await db.delete(issues); await db.delete(heartbeatRuns); @@ -80,14 +86,14 @@ const support = external ? { supported: true } : await getEmbeddedPostgresTestSu await db.update(issues).set({ checkoutRunId: run!.id }).where(eq(issues.id, generationIssueId)); const writer = { agentId, runId: run!.id }; const cards = statusCardService(db); - if (compile) await cards.writeQuery(cardId, { queries: [companySearchQuerySchema.parse({ scope: "issues", status: ["blocked"] })], + if (compile) await cards.writeQuery(cardId, { queries: [compiledQuery], title: "Blockers", changeSummary: "Compiled", generationIssueId }, writer); await cards.writeSummary(cardId, { markdown: "Current blockers", changeSummary: "Updated", generationIssueId }, writer); await issueService(db).update(generationIssueId, { status: "done" }); } - async function card(mode: "manual" | "reactive" | "interval" = "reactive") { + async function card(mode: "manual" | "reactive" | "interval" = "reactive", refreshPolicy = policy(mode)) { const cards = statusCardService(db); - const created = await cards.create(companyId, { interestPrompt: "Blockers", titlePinned: false, agentId, refreshPolicy: policy(mode) }, actor); + const created = await cards.create(companyId, { interestPrompt: "Blockers", titlePinned: false, agentId, refreshPolicy }, actor); const compile = await cards.requestCompile(created.id, actor); await writeback(created.id, compile.generatingIssue.id, true); return cards.getById(created.id).then(row => row!); @@ -131,6 +137,19 @@ const support = external ? { supported: true } : await getEmbeddedPostgresTestSu expect(wakeup).not.toHaveBeenCalled(); expect(errors).toEqual([]); }); + it("restores overdue debt after the feature is disabled across its saved deadline", async () => { + await start(); const created = await card(); await quiet(); + await issueService(db).create(companyId, { title: "Disabled blocker", status: "blocked", priority: "medium" }); + await vi.waitFor(async () => expect((await statusCardService(db).getById(created.id))?.pendingChangeCount).toBeGreaterThan(0)); + const pending = (await statusCardService(db).getById(created.id))!; + await instanceSettingsService(db).updateExperimental({ enableStatusCards: false }); await quiet(); + expect(wakeup).not.toHaveBeenCalled(); + expect((await statusCardService(db).getById(created.id))!.nextEvalAt).toEqual(pending.nextEvalAt); + await instanceSettingsService(db).updateExperimental({ enableStatusCards: true }); + await vi.waitFor(() => expect(wakeup).toHaveBeenCalledOnce()); + await writeback(created.id, (await statusCardService(db).getById(created.id))!.generatingIssueId!); await quiet(); + }); + it("does not react to rolled-back or nested uncommitted card configuration", async () => { await start(); const created = await card("manual"); await quiet(); await expect(db.transaction(async tx => { @@ -169,6 +188,184 @@ const support = external ? { supported: true } : await getEmbeddedPostgresTestSu .toHaveLength(1); }); + it.each(["debounce", "hours", "rate", "budget"] as const)("restores pending %s eligibility after restart without advancing it or acting early", async gate => { + vi.setSystemTime(new Date("2026-11-02T18:00:00Z")); + const refreshPolicy = { ...policy(), ...(gate === "hours" ? { activeHours: { start: "09:00", end: "17:00", timezone: "UTC" } } : {}), + ...(gate === "rate" ? { maxUpdatesPerHour: 1 } : {}), ...(gate === "budget" ? { dailyTokenCap: 100 } : {}) }; + const created = await card("reactive", refreshPolicy); + if (gate === "rate" || gate === "budget") await db.insert(statusCardUpdates).values({ cardId: created.id, kind: "full", trigger: "reactive", + status: "ok", startedAt: new Date(Date.now() - (gate === "rate" ? 30 * 60_000 : 60_000)), finishedAt: new Date(), inputTokens: gate === "budget" ? 100 : 0 }); + await issueService(db).create(companyId, { title: "Saved blocker", status: "blocked", priority: "medium", createdByUserId: "board-user" }); + const service = statusCardEvaluationService(db, { wakeup }); + await service.run(new AbortController().signal); + const pending = (await statusCardService(db).getById(created.id))!; + expect(pending.pendingChangeCount).toBeGreaterThan(0); expect(pending.nextEvalAt!.getTime()).toBeGreaterThan(Date.now()); + const restarted = statusCardEvaluationService(db, { wakeup }); + await restarted.run(new AbortController().signal); + expect(await restarted.next()).toBe(pending.nextEvalAt!.getTime()); + expect((await statusCardService(db).getById(created.id))!.nextEvalAt).toEqual(pending.nextEvalAt); + vi.setSystemTime(pending.nextEvalAt!.getTime() - 1); await restarted.run(new AbortController().signal); + expect(wakeup).not.toHaveBeenCalled(); + vi.setSystemTime(pending.nextEvalAt!); await restarted.run(new AbortController().signal); + expect(wakeup).toHaveBeenCalledOnce(); + }); + + it("unions startup reconstruction with a scoped source change in another company", async () => { + const firstCompany = companyId; + const first = await card(); + await issueService(db).create(companyId, { title: "First blocker", status: "blocked", priority: "medium" }); + companyId = randomUUID(); agentId = randomUUID(); + await db.insert(companies).values({ id: companyId, name: "Other", issuePrefix: `X${companyId.slice(0, 6)}` }); + await db.insert(agents).values({ id: agentId, companyId, name: "Other summarizer", role: "general", status: "idle", adapterType: "process", adapterConfig: {} }); + const second = await card(); + await issueService(db).create(companyId, { title: "Second blocker", status: "blocked", priority: "medium" }); + const service = statusCardEvaluationService(db, { wakeup }); + service.changed(new Set([`changes:${companyId}`])); + await service.run(new AbortController().signal); + const cards = statusCardService(db); + expect((await cards.getById(first.id))?.companyId).toBe(firstCompany); + expect((await cards.getById(first.id))?.pendingChangeCount).toBeGreaterThan(0); + expect((await cards.getById(second.id))?.pendingChangeCount).toBeGreaterThan(0); + await vi.advanceTimersByTimeAsync(60_000); await service.run(new AbortController().signal); + expect(wakeup).toHaveBeenCalledTimes(2); + expect(new Set(await db.select().from(agentWakeupRequests).then(rows => rows.map(row => row.companyId)))) + .toEqual(new Set([firstCompany, companyId])); + }); + + it("keeps ignored issue activity quiet and reacts to human comments and document search membership", async () => { + await start(); await card(); await quiet(); + await issueService(db).create(companyId, { title: "Ignored todo", status: "todo", priority: "medium" }); + await quiet(); expect(wakeup).not.toHaveBeenCalled(); + compiledQuery = companySearchQuerySchema.parse({ scope: "all", q: "launch" }); + const searchCard = await card(); await quiet(); + const target = await issueService(db).create(companyId, { title: "Ordinary task", status: "todo", priority: "medium" }); + await quiet(); + await documentService(db).upsertIssueDocument({ issueId: target.id, key: "plan", format: "markdown", body: "launch evidence", createdByUserId: "board-user" }); + await vi.waitFor(async () => expect((await statusCardService(db).getById(searchCard.id))?.pendingChangeCount).toBeGreaterThan(0)); + await vi.advanceTimersByTimeAsync(60_000); await settled(); expect(wakeup).toHaveBeenCalledOnce(); + await writeback(searchCard.id, (await statusCardService(db).getById(searchCard.id))!.generatingIssueId!); await quiet(); + await issueService(db).addComment(target.id, "A human launch update", { userId: "board-user" }); + await vi.waitFor(async () => expect((await statusCardService(db).getById(searchCard.id))?.pendingChangeCount).toBeGreaterThan(0)); + }); + + it("retains relative query membership expiry across restart and emits the eventual removal", async () => { + const now = new Date(); + const target = await issueService(db).create(companyId, { title: "Expiring blocker", status: "blocked", priority: "medium" }); + await db.update(issues).set({ updatedAt: now }).where(eq(issues.id, target.id)); + compiledQuery = companySearchQuerySchema.parse({ scope: "issues", status: ["blocked"], updatedWithin: "1h" }); + const created = await card(); + const expires = now.getTime() + 3_600_001; + expect(created.nextEvalAt?.getTime()).toBe(expires); + const restarted = statusCardEvaluationService(db, { wakeup }); + await restarted.run(new AbortController().signal); + expect(await restarted.next()).toBe(expires); + vi.setSystemTime(expires - 1); await restarted.run(new AbortController().signal); expect(wakeup).not.toHaveBeenCalled(); + vi.setSystemTime(expires); await restarted.run(new AbortController().signal); + const pending = (await statusCardService(db).getById(created.id))!; + expect(pending.pendingChangeCount).toBe(1); expect(pending.nextEvalAt!.getTime()).toBe(expires + 60_000); + vi.setSystemTime(pending.nextEvalAt!); await restarted.run(new AbortController().signal); + expect(wakeup).toHaveBeenCalledOnce(); + }); + + it("uses expiry of an off-page candidate to refresh offset-ranked query membership", async () => { + const now = new Date(); + const tasks = await Promise.all((["critical", "high", "medium"] as const).map(priority => issueService(db).create(companyId, { + title: `${priority} blocker`, priority, status: "blocked", + }))); + await db.update(issues).set({ updatedAt: new Date(now.getTime() - 59 * 60_000) }).where(eq(issues.id, tasks[0]!.id)); + for (const task of tasks.slice(1)) await db.update(issues).set({ updatedAt: now }).where(eq(issues.id, task.id)); + compiledQuery = companySearchQuerySchema.parse({ scope: "issues", status: ["blocked"], updatedWithin: "1h", sort: "priority", limit: 1, offset: 1 }); + const created = await card(); + expect(Object.keys(created.fingerprint!)).toEqual([tasks[1]!.id]); + const expires = now.getTime() + 60_001; + expect(created.nextEvalAt?.getTime()).toBe(expires); + const service = statusCardEvaluationService(db, { wakeup }); + vi.setSystemTime(expires); await service.run(new AbortController().signal); + const pending = (await statusCardService(db).getById(created.id))!; + expect(pending.pendingChangeCount).toBe(2); expect(wakeup).not.toHaveBeenCalled(); + vi.setSystemTime(pending.nextEvalAt!); await service.run(new AbortController().signal); + expect(wakeup).toHaveBeenCalledOnce(); + const [update] = await db.select().from(statusCardUpdates).where(eq(statusCardUpdates.generationIssueId, (await statusCardService(db).getById(created.id))!.generatingIssueId!)); + expect(new Set(update!.changes.map(change => change.issueId))).toEqual(new Set([tasks[1]!.id, tasks[2]!.id])); + }); + + it("retains a scoped retry while another evaluator owns the card lock", async () => { + const created = await card(); + await issueService(db).create(companyId, { title: "Locked blocker", status: "blocked", priority: "medium" }); + const acquired = Promise.withResolvers(), release = Promise.withResolvers(); + const holder = db.transaction(async tx => { + await tx.execute(sql`select pg_advisory_xact_lock(hashtextextended(${created.id}, 0))`); + acquired.resolve(); await release.promise; + }); + await acquired.promise; + const service = statusCardEvaluationService(db, { wakeup }); + service.changed(new Set([`changes:${companyId}`])); + try { + await service.run(new AbortController().signal); + expect(await service.next()).toBeGreaterThan(Date.now()); + expect((await statusCardService(db).getById(created.id))!.nextEvalAt).toBeNull(); + expect(wakeup).not.toHaveBeenCalled(); + } finally { release.resolve(); await holder; } + await vi.advanceTimersByTimeAsync(30_000); await service.run(new AbortController().signal); + const pending = (await statusCardService(db).getById(created.id))!; + expect(pending.pendingChangeCount).toBeGreaterThan(0); + vi.setSystemTime(pending.nextEvalAt!); await service.run(new AbortController().signal); + expect(wakeup).toHaveBeenCalledOnce(); + }); + + it("retains the durable five-minute retry after evaluation failure across restart and drain", async () => { + const created = await card(); + await db.update(statusCards).set({ queries: [{ scope: "invalid" } as never], nextEvalAt: new Date() }).where(eq(statusCards.id, created.id)); + const service = statusCardEvaluationService(db, { wakeup }); + await service.run(new AbortController().signal); + const failed = (await statusCardService(db).getById(created.id))!; + const retryAt = failed.nextEvalAt!.getTime(); + expect(retryAt).toBeGreaterThan(Date.now() + 299_000); expect(wakeup).not.toHaveBeenCalled(); + await start(); expect(coordinator!.nextWakeAt()).toBe(retryAt); + draining = true; await vi.advanceTimersByTimeAsync(5 * 60_000); await settled(); expect(wakeup).not.toHaveBeenCalled(); + // Repair the controlled malformed stored-query fault; no tested result is precreated. + await db.update(statusCards).set({ queries: [compiledQuery] }).where(eq(statusCards.id, created.id)); + draining = false; await vi.advanceTimersByTimeAsync(30_000); await quiet(); + expect((await statusCardService(db).getById(created.id))!.nextEvalAt).toBeNull(); + }); + + it("cancels failed assignments without rerunning a completed generation and settles after retry", async () => { + await start(); const created = await card(); await quiet(); + wakeup.mockRejectedValueOnce(new Error("assignment unavailable")); + await issueService(db).create(companyId, { title: "Assignment blocker", status: "blocked", priority: "medium" }); + await vi.waitFor(async () => expect((await statusCardService(db).getById(created.id))?.pendingChangeCount).toBeGreaterThan(0)); + await vi.advanceTimersByTimeAsync(60_000); await settled(); + const failed = (await statusCardService(db).getById(created.id))!; + expect(failed).toMatchObject({ state: "error", generatingIssueId: null, nextEvalAt: null }); + expect(errors.length).toBeGreaterThan(0); + const history = await db.select().from(statusCardUpdates).where(eq(statusCardUpdates.cardId, created.id)); + const ledger = history.find(row => row.status === "failed")!; + expect(ledger.finishedAt).toBeInstanceOf(Date); + expect((await db.select().from(issues).where(eq(issues.id, ledger.generationIssueId!)))[0]?.status).toBe("cancelled"); + const count = wakeup.mock.calls.length; await quiet(); expect(wakeup).toHaveBeenCalledTimes(count); + // An explicit ordinary refresh is still the user's supported retry action. + const retry = await statusCardService(db).requestRefresh(created.id, { trigger: "manual", actor }); + expect(retry.generatingIssue).toBeTruthy(); await writeback(created.id, retry.generatingIssue!.id); await quiet(); + expect((await statusCardService(db).getById(created.id))!.state).toBe("active"); + }); + + it("budget-owned company resume reactivates reactive work committed while paused", async () => { + await start(); const created = await card(); await quiet(); + const budgets = budgetService(db); + await budgets.upsertPolicy(companyId, { scopeType: "company", scopeId: companyId, amount: 100, hardStopEnabled: true, notifyEnabled: false }, "board-user"); + const [cost] = await db.insert(costEvents).values({ companyId, agentId, provider: "openai", biller: "openai", billingType: "metered_api", model: "test", + costCents: 200, occurredAt: new Date() }).returning(); + await budgets.evaluateCostEvent(cost!); + expect((await db.select().from(companies).where(eq(companies.id, companyId)))[0]?.status).toBe("paused"); + await quiet(); + await issueService(db).create(companyId, { title: "Paused blocker", status: "blocked", priority: "medium" }); await quiet(); + expect(wakeup).not.toHaveBeenCalled(); + await budgets.upsertPolicy(companyId, { scopeType: "company", scopeId: companyId, amount: 1000 }, "board-user"); + expect((await db.select().from(companies).where(eq(companies.id, companyId)))[0]?.status).toBe("active"); + await vi.waitFor(async () => expect((await statusCardService(db).getById(created.id))?.pendingChangeCount).toBeGreaterThan(0)); + await vi.advanceTimersByTimeAsync(60_000); await settled(); expect(wakeup).toHaveBeenCalledOnce(); + }); + it("preserves the first write during drain until resume", async () => { await start(); const created = await card("manual"); await quiet(); draining = true; diff --git a/server/src/__tests__/status-card-notifications.test.ts b/server/src/__tests__/status-card-notifications.test.ts new file mode 100644 index 00000000000..0566ebdd394 --- /dev/null +++ b/server/src/__tests__/status-card-notifications.test.ts @@ -0,0 +1,31 @@ +import { expect, it, vi } from "vitest"; +import { installDatabaseWorkSignals, subscribeDatabaseWork } from "../../../packages/db/src/work-signals.js"; +import { DELIVERY_QUEUES } from "../services/delivery-work-notifications.js"; +import { notifyExecutionRunWork, writeExecutionControl } from "../services/execution-control-notifications.js"; + +it.each([false, true])("keeps execution queue consumers intact and emits only a known status-card company scope (%s)", async scoped => { + const owner = installDatabaseWorkSignals({ execute: vi.fn(async () => []), transaction: async (fn: (tx: object) => Promise) => + fn({ execute: async () => [{ xid: "1" }] }) }); + const queues = [DELIVERY_QUEUES.executionStatus, DELIVERY_QUEUES.executionFinalization, DELIVERY_QUEUES.executionReplacement, + DELIVERY_QUEUES.executionDisposition, DELIVERY_QUEUES.issueMonitors]; + const settled: Array<{ queue: string; hints?: string[] }> = []; + const unsubscribe = [...queues, DELIVERY_QUEUES.statusCardEvaluation, DELIVERY_QUEUES.workspaceRecovery].map(queue => subscribeDatabaseWork(owner, queue, (event, hints) => { + if (event === "settled") settled.push({ queue, hints: hints ? [...hints] : undefined }); + })); + try { + const scope = scoped ? { companyId: "changed-company", runId: "changed-run" } : undefined; + await owner.transaction(async tx => { + await notifyExecutionRunWork(tx, scope); + expect(settled).toEqual([]); + }); + expect(new Set(settled.map(row => row.queue))).toEqual(new Set([...queues, ...(scoped ? [DELIVERY_QUEUES.statusCardEvaluation] : [])])); + expect(settled.find(row => row.queue === DELIVERY_QUEUES.statusCardEvaluation)?.hints) + .toEqual(scoped ? ["changes:changed-company"] : undefined); + settled.length = 0; + const write = vi.fn(async () => "written"); + expect(await writeExecutionControl(owner as never, DELIVERY_QUEUES.workspaceRecovery, write, scope)).toBe("written"); + expect(write).toHaveBeenCalledOnce(); + expect(new Set(settled.map(row => row.queue))).toEqual(new Set([DELIVERY_QUEUES.issueMonitors, DELIVERY_QUEUES.workspaceRecovery, + ...(scoped ? [DELIVERY_QUEUES.statusCardEvaluation] : [])])); + } finally { for (const stop of unsubscribe) stop(); } +}); diff --git a/server/src/modules/wake-queue/adapters/postgres.ts b/server/src/modules/wake-queue/adapters/postgres.ts index b888ee51fb3..2b7f0a16e26 100644 --- a/server/src/modules/wake-queue/adapters/postgres.ts +++ b/server/src/modules/wake-queue/adapters/postgres.ts @@ -1,3 +1,4 @@ +import { notifyStatusCardIssueChange } from "../../../services/status-card-notifications.js"; import { notifyExternalConversationSettlement } from "../../../services/external-conversation-settlement-notifications.js"; import { notifyNativeStatusReassessment } from "../../../services/native-status-reassessment-notifications.js"; import { notifyUnsafeWorkspaceExportRecovery } from "../../../services/unsafe-workspace-export-notifications.js"; @@ -610,6 +611,7 @@ function buildTransaction(tx: Db, deps: WakeQueuePostgresAdapterDeps, db: Db, ru }, async queueReviewParticipantRecoveryRun({ companyId, issue, finishingRun, recoveryAgent, sessionBefore, now }) { + await notifyStatusCardIssueChange(tx, companyId); const executionState = parseIssueExecutionState(issue.executionState); const wakeupRequest = await tx .insert(agentWakeupRequests) diff --git a/server/src/modules/wake-queue/adapters/queued-comment-postgres.ts b/server/src/modules/wake-queue/adapters/queued-comment-postgres.ts index 184c799f82d..a10419f0c41 100644 --- a/server/src/modules/wake-queue/adapters/queued-comment-postgres.ts +++ b/server/src/modules/wake-queue/adapters/queued-comment-postgres.ts @@ -1,3 +1,4 @@ +import { notifyStatusCardIssueChange } from "../../../services/status-card-notifications.js"; import { notifyExternalAgentSettlement } from "../../external-agents/index.js"; import { notifyNativeStatusReassessment } from "../../../services/native-status-reassessment-notifications.js"; import { notifyUnsafeWorkspaceExportRecovery } from "../../../services/unsafe-workspace-export-notifications.js"; @@ -55,6 +56,7 @@ export type QueuedCommentQueuePostgresAdapterDeps = { function buildTransaction(tx: Db, companyId: string, issueId: string, deps: QueuedCommentQueuePostgresAdapterDeps): QueuedCommentQueueTransaction { return { async updateCommentBody({ issueId, commentId, body, updatedAt }) { + await notifyStatusCardIssueChange(tx, companyId); const updated = await tx .update(issueComments) .set({ body, updatedAt }) @@ -65,6 +67,7 @@ function buildTransaction(tx: Db, companyId: string, issueId: string, deps: Queu }, async touchIssueUpdatedAt({ issueId, updatedAt }) { + await notifyStatusCardIssueChange(tx, companyId); await tx.update(issues).set({ updatedAt }).where(and(eq(issues.id, issueId), eq(issues.companyId, companyId))); }, diff --git a/server/src/routes/execution-workspaces.ts b/server/src/routes/execution-workspaces.ts index e87fbbeea6b..7be8e505a94 100644 --- a/server/src/routes/execution-workspaces.ts +++ b/server/src/routes/execution-workspaces.ts @@ -1,3 +1,4 @@ +import { notifyStatusCardIssueChange } from "../services/status-card-notifications.js"; import { removePluginWorkspace } from "../services/plugin-workspaces.js"; import { canActorReadWorkspaceOperation } from "../services/heartbeat-run-privacy.js"; import { canActorReadExecutionWorkspace, executionWorkspaceReadSqlCondition } from "../services/authorization.js"; @@ -1245,7 +1246,9 @@ export function executionWorkspaceRoutes(db: Db, opts: { pluginWorkerManager?: P await runtimeLeases.release({ executionWorkspaceId: existing.id, force: true }); if (existing.mode === "shared_workspace") { - await db + await db.transaction(async tx => { + await notifyStatusCardIssueChange(tx, existing.companyId); + await tx .update(issues) .set({ executionWorkspaceId: null, @@ -1257,6 +1260,7 @@ export function executionWorkspaceRoutes(db: Db, opts: { pluginWorkerManager?: P eq(issues.executionWorkspaceId, existing.id), ), ); + }); } try { diff --git a/server/src/services/agent-conversations.ts b/server/src/services/agent-conversations.ts index af44a0355d8..6dc783568e2 100644 --- a/server/src/services/agent-conversations.ts +++ b/server/src/services/agent-conversations.ts @@ -1,3 +1,4 @@ +import { notifyStatusCardIssueChange } from "./status-card-notifications.js"; import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js"; import { notifyUnsafeWorkspaceExportRecovery } from "./unsafe-workspace-export-notifications.js"; import { FAST_RESPONSE_AGENT_GUIDANCE, fastResponseHistoryBody } from "@paperclipai/shared"; @@ -269,6 +270,7 @@ export async function prepareConversationTurn( context.conversationReplayThroughCommentId = replayThrough?.id ?? commentId; } } + await notifyStatusCardIssueChange(tx, issue.companyId); await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors); await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers); await tx @@ -389,6 +391,7 @@ export async function settleConversationTurn( issue.conversationState === conversationState ) return true; + await notifyStatusCardIssueChange(tx, issue.companyId); await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors); await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers); if (status === "in_review") await notifyUnsafeWorkspaceExportRecovery(tx, { companyId: issue.companyId, issueId: issue.id }); diff --git a/server/src/services/budgets.ts b/server/src/services/budgets.ts index 6777124ee90..baf667c43c9 100644 --- a/server/src/services/budgets.ts +++ b/server/src/services/budgets.ts @@ -1,3 +1,4 @@ +import { notifyStatusCardIssueChange } from "./status-card-notifications.js"; import { notifyApprovalWork } from "./approval-work-notifications.js"; import { notifyCompanyRetentionWork } from "./retention-work-notifications.js"; import { addCents, compareCents, normalizeCents } from "@paperclipai/shared"; @@ -301,6 +302,7 @@ export function budgetServiceInTransaction(db: Db, publications: ActivityPublica const scope = await resolveScopeRecord(db, "company", policy.scopeId); if (scope.paused) return; await notifyCompanyRetentionWork(db, policy.companyId); + await notifyStatusCardIssueChange(db, policy.companyId); await notifyDeliveryWork(db, DELIVERY_QUEUES.agentTimers); await notifyDeliveryWork(db, DELIVERY_QUEUES.routineSchedules); await notifyDeliveryWork(db, DELIVERY_QUEUES.issueMonitors); @@ -361,6 +363,7 @@ export function budgetServiceInTransaction(db: Db, publications: ActivityPublica const scope = await resolveScopeRecord(db, "company", policy.scopeId); if (scope.pauseReason !== "budget") return; await notifyCompanyRetentionWork(db, policy.companyId); + await notifyStatusCardIssueChange(db, policy.companyId); await notifyDeliveryWork(db, DELIVERY_QUEUES.agentTimers); await notifyDeliveryWork(db, DELIVERY_QUEUES.routineSchedules); await notifyDeliveryWork(db, DELIVERY_QUEUES.issueMonitors); diff --git a/server/src/services/chat-channels/telegram/attachment-failure.ts b/server/src/services/chat-channels/telegram/attachment-failure.ts index 58d5d4a2677..49e2497a055 100644 --- a/server/src/services/chat-channels/telegram/attachment-failure.ts +++ b/server/src/services/chat-channels/telegram/attachment-failure.ts @@ -1,3 +1,4 @@ +import { notifyStatusCardIssueChange } from "../../status-card-notifications.js"; import { createProviderEffects, type ProviderEffectTarget } from "../provider-effects.js"; import type { EndpointRow, RuntimeContext, InboundRuntimeContext } from "../runtime-types.js"; @@ -82,6 +83,7 @@ export function createTelegramAttachmentFailure(db: Db, options: TelegramAttachm // Paperclip-visible failure. The message link stays authoritative for // exact provider redelivery, while no agent wakeup is enqueued for an // empty turn that contains no usable bytes. + await notifyStatusCardIssueChange(tx, input.endpoint.companyId); await tx .update(issueComments) .set({ body: visibleFailure, updatedAt: new Date() }) diff --git a/server/src/services/company-search.ts b/server/src/services/company-search.ts index bcf2a550d3f..e6595beebd8 100644 --- a/server/src/services/company-search.ts +++ b/server/src/services/company-search.ts @@ -548,7 +548,7 @@ export function companySearchService(db: Db) { search: async ( companyId: string, query: CompanySearchQuery, - options?: { issueReadCondition?: SQL; projectReadCondition?: SQL }, + options?: { issueReadCondition?: SQL; projectReadCondition?: SQL; onNextMembershipChange?: (at: Date | null) => void }, ): Promise => { const taskSearch = parseTaskSearch(query.q); const normalizedQuery = taskSearch.normalizedQuery; @@ -559,6 +559,7 @@ export function companySearchService(db: Db) { const limit = query.limit; const offset = query.offset; if (!hasSearchText && !issueOnlyFiltersActive(query)) { + options?.onNextMembershipChange?.(null); return { query: query.q, normalizedQuery, @@ -727,6 +728,13 @@ export function companySearchService(db: Db) { } } + const membershipWindow = companySearchUpdatedWithinMs(query.updatedWithin); + if (options?.onNextMembershipChange && membershipWindow !== null && scopeIncludesIssues(scope)) { + // Includes rows outside the visible page: their expiry can move an + // offset/priority-ranked result into the watched set without a write. + branches.push(sql`SELECT 'membership-expiry' AS kind, min(m.updated_at)::text AS value, + 0::int AS count ${countTail} FROM matched m ${branchWhere([scopeCond, ...facetsAll])}`); + } const resultRows = await db.execute(sql` ${taskSearchCtes(companyId, taskSearch, scope !== "issues", and(...issueFilters), options?.issueReadCondition)} ${sql.join(branches, sql` UNION ALL `)} @@ -738,6 +746,7 @@ export function companySearchService(db: Db) { totals: { current: 0, unfiltered: 0, omit: {} }, }; const issueRowsRaw: IssueSearchRow[] = []; + let nextMembershipChange: Date | null = null; for (const row of resultRows) { if (row.kind === "result") { issueRowsRaw.push({ @@ -762,6 +771,9 @@ export function companySearchService(db: Db) { }); continue; } + if (row.kind === "membership-expiry" && row.value && membershipWindow !== null) { + nextMembershipChange = new Date(new Date(row.value).getTime() + membershipWindow + 1); + } const count = Number(row.count ?? 0); if (row.kind === "type:issue") aggregates.typeCounts.issue = count; else if (row.kind === "type:comment") aggregates.typeCounts.comment = count; @@ -782,6 +794,7 @@ export function companySearchService(db: Db) { aggregates.totals.omit[row.kind.slice("total:omit:".length) as CompanySearchIssueFilterKey] = count; } } + options?.onNextMembershipChange?.(nextMembershipChange); return { rows: await enrichIssueSnippets(issueRowsRaw), aggregates }; } diff --git a/server/src/services/documents.ts b/server/src/services/documents.ts index 493c495e007..79e8e14309c 100644 --- a/server/src/services/documents.ts +++ b/server/src/services/documents.ts @@ -1,3 +1,4 @@ +import { notifyStatusCardIssueChange } from "./status-card-notifications.js"; import { randomUUID } from "node:crypto"; import { and, asc, desc, eq, sql } from "drizzle-orm"; import type { Db } from "@paperclipai/db"; @@ -238,6 +239,7 @@ export function documentService(db: Db) { for (let attempt = 0; attempt < maxAttempts; attempt += 1) { try { return await db.transaction(async (tx) => { + await notifyStatusCardIssueChange(tx, issue.companyId); const now = new Date(); const existing = await tx .select({ @@ -592,6 +594,7 @@ export function documentService(db: Db) { }); } await db.transaction(async (tx) => { + for (const companyId of new Set(rows.map(row => row.companyId))) await notifyStatusCardIssueChange(tx, companyId); await insertRowsInChunks(tx, documents, documentRows); await insertRowsInChunks(tx, documentRevisions, revisionRows); await insertRowsInChunks(tx, issueDocuments, linkRows); @@ -662,6 +665,7 @@ export function documentService(db: Db) { }) .returning(); + await notifyStatusCardIssueChange(tx, existing.companyId); await tx .update(documents) .set({ @@ -818,6 +822,7 @@ export function documentService(db: Db) { } await tx.delete(issueDocuments).where(eq(issueDocuments.documentId, existing.id)); + await notifyStatusCardIssueChange(tx, existing.companyId); await tx.delete(documents).where(eq(documents.id, existing.id)); return { diff --git a/server/src/services/execution-workspaces.ts b/server/src/services/execution-workspaces.ts index caaae186ad8..15ff32f292e 100644 --- a/server/src/services/execution-workspaces.ts +++ b/server/src/services/execution-workspaces.ts @@ -2530,13 +2530,16 @@ export function executionWorkspaceService(db: Db, opts: ExecutionWorkspaceServic forceWorktreeRemoval: false, }); if (cleanup.cleaned && workspace.mode === "shared_workspace") { - await db + await db.transaction(async tx => { + await notifyStatusCardIssueChange(tx, workspace.companyId); + await tx .update(issues) .set({ executionWorkspaceId: null, updatedAt: now() }) .where(and( eq(issues.companyId, workspace.companyId), eq(issues.executionWorkspaceId, workspace.id), )); + }); } const cleanupReason = [ISSUE_TERMINAL_WORKSPACE_CLEANUP_REASON, ...cleanup.warnings].join(" | "); if (!cleanup.cleaned || cleanup.warnings.length > 0) { diff --git a/server/src/services/heartbeat/queue.ts b/server/src/services/heartbeat/queue.ts index a41b486b647..4d1e423d014 100644 --- a/server/src/services/heartbeat/queue.ts +++ b/server/src/services/heartbeat/queue.ts @@ -1,3 +1,4 @@ +import { notifyStatusCardIssueChange } from "../status-card-notifications.js"; import { DELIVERY_QUEUES, notifyDeliveryWork } from "../delivery-work-notifications.js"; import { notifyExternalAgentSettlement } from "../../modules/external-agents/index.js"; import { notifyNativeStatusReassessment } from "../native-status-reassessment-notifications.js"; @@ -1267,6 +1268,7 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende } async function bindClaimedIssueExecution(tx: Db, ownsIssue: boolean, claimedRun: typeof heartbeatRuns.$inferSelect | null | undefined) { if (!claimedRun || !issueId || !ownsIssue) return; + await notifyStatusCardIssueChange(tx, run.companyId); await tx.update(issues).set({ executionRunId: claimedRun.id, executionAgentNameKey: normalizeAgentNameKey(agent.name), @@ -3758,6 +3760,7 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende .from(agents) .where(eq(agents.id, legacyRun.agentId)) .then((rows) => rows[0] ?? null); + await notifyStatusCardIssueChange(tx, issue.companyId); await tx .update(issues) .set({ diff --git a/server/src/services/heartbeat/scheduling.ts b/server/src/services/heartbeat/scheduling.ts index dece8b41ae9..b0e7f308297 100644 --- a/server/src/services/heartbeat/scheduling.ts +++ b/server/src/services/heartbeat/scheduling.ts @@ -1,3 +1,4 @@ +import { notifyStatusCardIssueChange } from "../status-card-notifications.js"; import { DELIVERY_QUEUES, notifyDeliveryWork } from "../delivery-work-notifications.js"; import { notifyIssueReviewState } from "../issue-review-state-notifications.js"; import { insertIssueComment } from "../issue-comment-writes.js"; @@ -595,6 +596,7 @@ export function createHeartbeatScheduling( try { await db.transaction(async (tx) => { await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors); + await notifyStatusCardIssueChange(tx, claimed.companyId); const released = await tx.update(issues) .set({ monitorWakeRequestedAt: null, updatedAt: input.now }) .where(issueMonitorClaimCondition(claimed)).returning({ id: issues.id }); @@ -628,6 +630,7 @@ export function createHeartbeatScheduling( // failed, and never clobber a monitor that was replaced in the meantime. await db.transaction(async tx => { await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors); + await notifyStatusCardIssueChange(tx, claimed.companyId); await tx.update(issues).set({ monitorWakeRequestedAt: null, updatedAt: input.now }) .where(issueMonitorClaimCondition(claimed)); }); @@ -827,6 +830,7 @@ export function createHeartbeatScheduling( } const consumed = await db.transaction(async tx => { + await notifyStatusCardIssueChange(tx, claimed.companyId); await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors); return tx.update(issues) .set({ @@ -977,6 +981,7 @@ export function createHeartbeatScheduling( const staleClaimThreshold = new Date(now.getTime() - 5 * 60 * 1000); const claimed = await db.transaction(async (tx) => { + await notifyStatusCardIssueChange(tx, issue.companyId); await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors); const [updated] = await tx .update(issues) @@ -1050,6 +1055,7 @@ export function createHeartbeatScheduling( for (const due of dueMonitors) { if (!canContinue()) break; const claimed = await db.transaction(async (tx) => { + await notifyStatusCardIssueChange(tx, due.companyId); await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors); const [updated] = await tx .update(issues) diff --git a/server/src/services/issue-thread-interactions.ts b/server/src/services/issue-thread-interactions.ts index 59c14618342..f4d5abbc5e2 100644 --- a/server/src/services/issue-thread-interactions.ts +++ b/server/src/services/issue-thread-interactions.ts @@ -1,3 +1,4 @@ +import { notifyStatusCardIssueChange } from "./status-card-notifications.js"; import { notifyNativeStatusReassessment } from "./native-status-reassessment-notifications.js"; import { notifyIssueReviewState } from "./issue-review-state-notifications.js"; import { notifyChatConfirmationResolution } from "./chat-confirmation-notifications.js"; @@ -376,7 +377,7 @@ type ResolvedInteractionResult = { }; type IssueThreadInteractionRow = typeof issueThreadInteractions.$inferSelect; -type IssueTouchDb = Pick; +type IssueTouchDb = Pick; function isNativeCompletionReview( row: Pick, @@ -801,11 +802,11 @@ function hydrateInteraction( } } -async function touchIssue(db: IssueTouchDb, issueId: string) { - await db - .update(issues) - .set({ updatedAt: new Date() }) - .where(eq(issues.id, issueId)); +async function touchIssue(db: IssueTouchDb, issueId: string, companyId: string) { + await db.transaction(async tx => { + await notifyStatusCardIssueChange(tx, companyId); + await tx.update(issues).set({ updatedAt: new Date() }).where(eq(issues.id, issueId)); + }); } function isTerminalIssueStatus(status: string) { @@ -2013,7 +2014,7 @@ async function expireStaleRequestConfirmationTarget( "Interaction has already been resolved", ); } - await touchIssue(tx, args.row.issueId); + await touchIssue(tx, args.row.issueId, args.row.companyId); const expired = hydrateInteraction(updated); await enqueueTerminalIssueInteractionChatPublications(tx, expired); return expired; @@ -2468,7 +2469,7 @@ export function issueThreadInteractionService( }; } } else { - await touchIssue(tx, args.issue.id); + await touchIssue(tx, args.issue.id, args.issue.companyId); } // Snapshot the continuation target only after every lifecycle mutation @@ -2682,7 +2683,7 @@ export function issueThreadInteractionService( tx, ); } else { - await touchIssue(tx, args.issue.id); + await touchIssue(tx, args.issue.id, args.issue.companyId); } await enqueueTerminalIssueInteractionChatPublications( tx as unknown as Db, @@ -2864,7 +2865,7 @@ export function issueThreadInteractionService( await enqueueTerminalIssueInteractionChatPublications(tx as unknown as Db, hydrateInteraction(previous)); } } - await touchIssue(tx, issue.id); + await touchIssue(tx, issue.id, issue.companyId); inserted = true; await enqueueIssueInteractionChatPublications(tx as unknown as Db, hydrateInteraction(row)); await notifyDeliveryWork(tx, DELIVERY_QUEUES.connection); @@ -2908,7 +2909,7 @@ export function issueThreadInteractionService( ) .returning(); if (!updated) throw interactionAlreadyResolvedError(); - await touchIssue(db, issue.id); + await touchIssue(db, issue.id, issue.companyId); return hydrateInteraction(updated) as ConnectionIntentInteraction; }, resolveConnectionIntent: async ( @@ -2972,7 +2973,7 @@ export function issueThreadInteractionService( return row; }); if (!updated) throw interactionAlreadyResolvedError(); - await touchIssue(db, issue.id); + await touchIssue(db, issue.id, issue.companyId); const interaction = hydrateInteraction( updated, ) as ConnectionIntentInteraction; @@ -3351,7 +3352,7 @@ export function issueThreadInteractionService( }) .where(eq(issueThreadInteractions.id, current.id)) .returning(); - await touchIssue(tx, issue.id); + await touchIssue(tx, issue.id, issue.companyId); return row; }); return hydrateInteraction(updated); @@ -3414,10 +3415,8 @@ export function issueThreadInteractionService( if (updated) cancelled.push(updated); } - for (const issueId of new Set( - cancelled.map((interaction) => interaction.issueId), - )) { - await touchIssue(db, issueId); + for (const [issueId, companyId] of new Map(cancelled.map(interaction => [interaction.issueId, interaction.companyId]))) { + await touchIssue(db, issueId, companyId); } await emitResolvedInteractionsTelemetry(db, cancelled); return cancelled; @@ -3506,10 +3505,8 @@ export function issueThreadInteractionService( } if (expired.length > 0) { - for (const issueId of new Set( - expired.map((interaction) => interaction.issueId), - )) { - await touchIssue(db, issueId); + for (const [issueId, companyId] of new Map(expired.map(interaction => [interaction.issueId, interaction.companyId]))) { + await touchIssue(db, issueId, companyId); } await emitResolvedInteractionsTelemetry(db, expired); } @@ -4052,7 +4049,7 @@ export function issueThreadInteractionService( return interaction; } - await touchIssue(db, issue.id); + await touchIssue(db, issue.id, issue.companyId); if (superseded.length > 0) { await emitResolvedInteractionsTelemetry( db, @@ -4357,7 +4354,7 @@ export function issueThreadInteractionService( .where(eq(issueThreadInteractions.id, interactionId)) .returning(); - await touchIssue(tx, issue.id); + await touchIssue(tx, issue.id, issue.companyId); await mutationOptions.afterResolveInTransaction?.(tx, hydrateInteraction(updated)); await enqueueTerminalIssueInteractionChatPublications(tx as unknown as Db, hydrateInteraction(updated)); current.status = updated.status; @@ -4549,7 +4546,7 @@ export function issueThreadInteractionService( throw interactionAlreadyResolvedError(); } - await touchIssue(tx, issue.id); + await touchIssue(tx, issue.id, issue.companyId); await mutationOptions.afterResolveInTransaction?.(tx, hydrateInteraction(updated)); if (complete) await enqueueTerminalIssueInteractionChatPublications(tx as unknown as Db, hydrateInteraction(updated)); return { @@ -4622,7 +4619,7 @@ export function issueThreadInteractionService( throw interactionAlreadyResolvedError(); } - await touchIssue(db, issue.id); + await touchIssue(db, issue.id, issue.companyId); const rejected = hydrateInteraction(updated); await emitInteractionResolvedTelemetry(db, rejected); return rejected; @@ -4720,7 +4717,7 @@ export function issueThreadInteractionService( } if (expired.length > 0) { - await touchIssue(db, issue.id); + await touchIssue(db, issue.id, issue.companyId); await emitResolvedInteractionsTelemetry(db, expired); } return expired; @@ -4952,7 +4949,7 @@ export function issueThreadInteractionService( } if (expired.length > 0) { - await touchIssue(db, issue.id); + await touchIssue(db, issue.id, issue.companyId); await emitResolvedInteractionsTelemetry(db, expired); } return expired; @@ -5049,7 +5046,7 @@ export function issueThreadInteractionService( } if (expired.length > 0) { - await touchIssue(db, issue.id); + await touchIssue(db, issue.id, issue.companyId); await emitResolvedInteractionsTelemetry(db, expired); } return expired; @@ -5189,7 +5186,7 @@ export function issueThreadInteractionService( if (updated) expired.push(hydrateInteraction(updated)); } if (expired.length > 0) { - await touchIssue(db, issue.id); + await touchIssue(db, issue.id, issue.companyId); await emitResolvedInteractionsTelemetry(db, expired); } return expired; @@ -5294,7 +5291,7 @@ export function issueThreadInteractionService( return row; }); - await touchIssue(db, issue.id); + await touchIssue(db, issue.id, issue.companyId); const withdrawn = hydrateInteraction(updated); await emitInteractionResolvedTelemetry(db, withdrawn); return withdrawn; @@ -5417,7 +5414,7 @@ export function issueThreadInteractionService( return row; }); - await touchIssue(db, issue.id); + await touchIssue(db, issue.id, issue.companyId); const answered = hydrateInteraction(updated); await emitInteractionResolvedTelemetry(db, answered); return answered; @@ -5553,7 +5550,7 @@ export function issueThreadInteractionService( return row; }); - await touchIssue(db, issue.id); + await touchIssue(db, issue.id, issue.companyId); const skipped = hydrateInteraction(updated); await emitInteractionResolvedTelemetry(db, skipped); return skipped; @@ -5630,7 +5627,7 @@ export function issueThreadInteractionService( return row; }); - await touchIssue(db, issue.id); + await touchIssue(db, issue.id, issue.companyId); const cancelled = hydrateInteraction(updated); await emitInteractionResolvedTelemetry(db, cancelled); return cancelled; diff --git a/server/src/services/issues.ts b/server/src/services/issues.ts index 8787ac867a8..f58d0e62789 100644 --- a/server/src/services/issues.ts +++ b/server/src/services/issues.ts @@ -7730,6 +7730,7 @@ export function issueService(db: Db) { } const now = new Date(); + await notifyStatusCardIssueChange(tx, lockedIssue.companyId); const adopted = await tx .update(issues) .set({ @@ -7783,13 +7784,14 @@ export function issueService(db: Db) { sql`select ${heartbeatRuns.id} from ${heartbeatRuns} where ${heartbeatRuns.id} = ${input.actorRunId} for update`, ); const actorRun = await tx - .select({ status: heartbeatRuns.status }) + .select({ status: heartbeatRuns.status, companyId: heartbeatRuns.companyId }) .from(heartbeatRuns) .where(eq(heartbeatRuns.id, input.actorRunId)) .then((rows) => rows[0] ?? null); if (!actorRun || TERMINAL_HEARTBEAT_RUN_STATUSES.has(actorRun.status)) return null; + await notifyStatusCardIssueChange(tx, actorRun.companyId); const now = new Date(); const adopted = await tx .update(issues) @@ -10892,6 +10894,7 @@ export function issueService(db: Db) { } } + await notifyStatusCardIssueChange(tx, companyId); await insertRowsInChunks(tx, issues, issueRows); await insertRowsInChunks(tx, issueLabels, labelRows); }); @@ -10908,6 +10911,7 @@ export function issueService(db: Db) { const censorUsernameInLogs = (await instanceSettings.getGeneral()) .censorUsernameInLogs; await db.transaction(async (tx) => { + for (const companyId of new Set(rows.map(row => row.companyId))) await notifyStatusCardIssueChange(tx, companyId); const commentRows = rows.map((row) => { const createdAt = row.createdAt ? new Date(row.createdAt) : null; return { @@ -11922,7 +11926,9 @@ export function issueService(db: Db) { ); if (settings?.environmentId !== environmentId) continue; - await db + await db.transaction(async tx => { + await notifyStatusCardIssueChange(tx, companyId); + await tx .update(issues) .set({ executionWorkspaceSettings: { @@ -11932,6 +11938,7 @@ export function issueService(db: Db) { updatedAt: new Date(), }) .where(eq(issues.id, row.id)); + }); cleared += 1; } @@ -12088,6 +12095,7 @@ export function issueService(db: Db) { ) : isNull(issues.executionRunId); const updated = await db.transaction(async tx => { + await notifyStatusCardIssueChange(tx, issueCompany.companyId); await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors); await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers); return tx @@ -12140,7 +12148,9 @@ export function issueService(db: Db) { current.executionRunId === checkoutRunId) && checkoutRunId ) { - const adopted = await db + const adopted = await db.transaction(async tx => { + await notifyStatusCardIssueChange(tx, issueCompany.companyId); + return tx .update(issues) .set({ checkoutRunId, @@ -12161,6 +12171,7 @@ export function issueService(db: Db) { ) .returning() .then((rows) => rows[0] ?? null); + }); if (adopted) return adopted; } @@ -12217,6 +12228,7 @@ export function issueService(db: Db) { } const staleExecutionRunId = current.executionRunId; const adopted = await db.transaction(async tx => { + await notifyStatusCardIssueChange(tx, issueCompany.companyId); await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors); await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers); return tx @@ -12524,6 +12536,7 @@ export function issueService(db: Db) { patch.assigneeAgentId = null; } + await notifyStatusCardIssueChange(tx, existing.companyId); await notifyTerminalWorkspaceWork(tx, `issue:${id}`); if (existing.checkoutRunId || existing.executionRunId || options.clearAssignee) { await notifyExternalConversationSettlement(tx, { companyId: existing.companyId, issueId: id }); @@ -12581,12 +12594,12 @@ export function issueService(db: Db) { return created; }, - deleteLabel: async (id: string) => - db - .delete(labels) - .where(eq(labels.id, id)) - .returning() - .then((rows) => rows[0] ?? null), + deleteLabel: async (id: string) => db.transaction(async tx => { + const [label] = await tx.select({ companyId: labels.companyId }).from(labels).where(eq(labels.id, id)); + if (!label) return null; + await notifyStatusCardIssueChange(tx, label.companyId); + return tx.delete(labels).where(eq(labels.id, id)).returning().then(rows => rows[0] ?? null); + }), listComments: async ( issueId: string, @@ -12838,7 +12851,7 @@ export function issueService(db: Db) { }, dbOrTx: any = db, ): Promise { - if (dbOrTx === db && (actor.runId || actor.userId)) { + if (dbOrTx === db) { const append = () => measureRunTransaction(db, "response", async (tx) => { // Serialize run-authored comments on the issue so a provider retry @@ -13081,6 +13094,7 @@ export function issueService(db: Db) { currentUserRedactionOptions.enabled, ); } + await notifyStatusCardIssueChange(dbOrTx, issue.companyId); comment = shouldUpgradeExternalAuthorization || shouldUpgradeAttachmentAuthorization diff --git a/server/src/services/native-status-reassessment-notifications.ts b/server/src/services/native-status-reassessment-notifications.ts index fb845523bb4..33cf199736f 100644 --- a/server/src/services/native-status-reassessment-notifications.ts +++ b/server/src/services/native-status-reassessment-notifications.ts @@ -1,8 +1,10 @@ +import { notifyStatusCardIssueChange } from "./status-card-notifications.js"; import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js"; export type NativeStatusReassessmentScope = { companyId: string; issueId?: string; runId?: string; agentId?: string }; -export function notifyNativeStatusReassessment(tx: object, scope: NativeStatusReassessmentScope) { +export async function notifyNativeStatusReassessment(tx: object, scope: NativeStatusReassessmentScope) { + await notifyStatusCardIssueChange(tx, scope.companyId); return notifyDeliveryWork(tx, DELIVERY_QUEUES.nativeStatusReassessment, JSON.stringify({ companyId: scope.companyId, issueId: scope.issueId, runId: scope.runId, agentId: scope.agentId })); } diff --git a/server/src/services/projects.ts b/server/src/services/projects.ts index 28cae661e9b..45891a218a6 100644 --- a/server/src/services/projects.ts +++ b/server/src/services/projects.ts @@ -1,3 +1,4 @@ +import { notifyStatusCardIssueChange } from "./status-card-notifications.js"; import { notifyChatInboundWakeNotices } from "./chat-inbound-wake-notice-notifications.js"; import { listProjectWorkspaceProviders, resolveProjectWorkspaceProvider } from "./project-workspace-provider.js"; import { retireRemovedPluginWorkspaceDependents } from "./plugin-workspaces.js"; @@ -1013,6 +1014,7 @@ export function projectService(db: Db, options: { captureWorkspaceUpdates?: bool const row = await db.transaction(async (tx) => { if (updates.visibility !== undefined) { + await notifyStatusCardIssueChange(tx, existingProject.companyId); await notifyChatInboundWakeNotices(tx, { companyId: existingProject.companyId }); await tx.execute(sql`select pg_advisory_xact_lock(hashtextextended(${`issue-privacy-tree:${existingProject.companyId}`}, 0))`); } @@ -1086,6 +1088,7 @@ export function projectService(db: Db, options: { captureWorkspaceUpdates?: bool await tx.select({ id: projectWorkspaces.id }).from(projectWorkspaces).where(eq(projectWorkspaces.projectId, id)).for("update"); await recordProjectLifecycleEvent(tx as unknown as Db, project.companyId, id, "delete"); await notifyDeliveryWork(tx, DELIVERY_QUEUES.skillSourceSync); + await notifyStatusCardIssueChange(tx, project.companyId); const [row] = await tx.delete(projects).where(eq(projects.id, id)).returning(); return row ? { ...row, urlKey: deriveProjectUrlKey(row.name, row.id) } : null; }), diff --git a/server/src/services/status-card-evaluation.ts b/server/src/services/status-card-evaluation.ts index 143df32344a..3e33c0d0885 100644 --- a/server/src/services/status-card-evaluation.ts +++ b/server/src/services/status-card-evaluation.ts @@ -18,17 +18,18 @@ export function statusCardEvaluationService(db: Db, heartbeat: IssueAssignmentWa let reconstructReactive = true; let allChanges = false; const changes = new Set(); - let enabled = false; + let enabled: boolean | null = null; + let retryAt: number | null = null; return { changed(hints?: ReadonlySet) { for (const hint of hints ?? ["*"]) { - if (hint === "*" || hint === "changes:*") allChanges = true; + if (hint === "*" || hint === "changes:*") { allChanges = true; if (hint === "*") enabled = null; } else if (hint.startsWith("changes:")) changes.add(hint.slice("changes:".length)); } }, async run(signal: AbortSignal) { - enabled = (await settings.getExperimental()).enableStatusCards === true; + if (enabled === null) enabled = (await settings.getExperimental()).enableStatusCards === true; if (!enabled || signal.aborted) return; const reconstruct = reconstructReactive; const evaluateAllChanges = allChanges; @@ -37,7 +38,9 @@ export function statusCardEvaluationService(db: Db, heartbeat: IssueAssignmentWa reconstructReactive = false; allChanges = false; try { - const result = await cards.tickDueStatusCards(new Date(), evaluateAllChanges ? "changed" : reconstruct && companiesChanged.size === 0 ? "all" : companiesChanged); + const result = await cards.tickDueStatusCards(new Date(), evaluateAllChanges ? "changed" : companiesChanged, reconstruct); + retryAt = result.deferredCompanyIds.size ? Date.now() + 30_000 : null; + for (const companyId of result.deferredCompanyIds) changes.add(companyId); const assignments = await Promise.allSettled(result.enqueued.map(async ({ cardId, generatingIssue }) => { // Once generated, dispatch or cancel even if shutdown begins: leaving // an unassigned claimed card is not a completed sweep. @@ -74,7 +77,8 @@ export function statusCardEvaluationService(db: Db, heartbeat: IssueAssignmentWa .where(and(eq(companies.status, "active"), isNull(statusCards.archivedAt), isNull(statusCards.generatingIssueId), isNotNull(statusCards.nextEvalAt))) .orderBy(asc(statusCards.nextEvalAt)).limit(1); - return next?.at?.getTime() ?? null; + const deadline = next?.at?.getTime() ?? null; + return retryAt === null ? deadline : Math.min(deadline ?? Infinity, retryAt); }, }; } diff --git a/server/src/services/status-cards.ts b/server/src/services/status-cards.ts index 84bbf7b2cd4..19e7ac75140 100644 --- a/server/src/services/status-cards.ts +++ b/server/src/services/status-cards.ts @@ -25,7 +25,7 @@ import { conflict, forbidden, notFound, unprocessable } from "../errors.js"; import { logger } from "../middleware/logger.js"; import { readBuiltInAgentMarker } from "../lib/built-in-agent-metadata.js"; import { builtInAgentService } from "./built-in-agents.js"; -import { companySearchService, companySearchUpdatedWithinMs } from "./company-search.js"; +import { companySearchService } from "./company-search.js"; import { issueService } from "./issues.js"; import { SUMMARIZER_BUILT_IN_KEY } from "./summary-slots.js"; import { @@ -47,7 +47,6 @@ import { type StatusCardActor = { agentId: string | null; userId: string | null }; type StatusCardWriter = { agentId: string | null; runId: string | null }; type StatusCardRow = typeof statusCards.$inferSelect; -type WatchedIssue = CompanySearchIssueSummary & { scopeExpiresAt?: Date | null }; type StatusCardReadOptions = { issueReadCondition?: SQL }; const TERMINAL_ISSUE_STATUSES = new Set(["done", "cancelled"]); @@ -590,22 +589,15 @@ export function statusCardService( return loadIssueSummaries(card.companyId, card.mentionedIssueIds ?? [], options); } - async function executeQueries(card: StatusCardRow, options: StatusCardReadOptions = {}) { - const issueMap = new Map(); + async function executeQueries(card: StatusCardRow, options: StatusCardReadOptions = {}, onNextMembershipChange?: (at: Date | null) => void) { + const issueMap = new Map(); for (const storedQuery of card.queries) { const query = companySearchQuerySchema.parse(storedQuery); const response = await searchSvc.search(card.companyId, query, { - issueReadCondition: options.issueReadCondition, + issueReadCondition: options.issueReadCondition, onNextMembershipChange, }); for (const result of response.results) { - if (result.type !== "issue" || !result.issue) continue; - const duration = companySearchUpdatedWithinMs(query.updatedWithin); - const expiry = duration === null ? null : new Date(new Date(result.issue.updatedAt).getTime() + duration + 1); - const previous = issueMap.get(result.issue.id); - issueMap.set(result.issue.id, { ...result.issue, scopeExpiresAt: previous - ? previous.scopeExpiresAt == null || expiry === null ? null - : new Date(Math.max(previous.scopeExpiresAt.getTime(), expiry.getTime())) - : expiry }); + if (result.type === "issue" && result.issue) issueMap.set(result.issue.id, result.issue); } } // Issues mentioned in the latest summary join the watched set alongside the @@ -615,11 +607,7 @@ export function statusCardService( (card.mentionedIssueIds ?? []).filter((issueId) => !issueMap.has(issueId)), options, ); - for (const issue of mentioned) issueMap.set(issue.id, { ...issue, scopeExpiresAt: null }); - for (const id of card.mentionedIssueIds ?? []) { - const issue = issueMap.get(id); - if (issue) issue.scopeExpiresAt = null; - } + for (const issue of mentioned) issueMap.set(issue.id, issue); const snapshot = [...issueMap.values()]; if (snapshot.length === 0) return snapshot; const latestHumanComments = await db @@ -643,11 +631,10 @@ export function statusCardService( return snapshot.map((issue) => ({ ...issue, latestHumanCommentAt: commentByIssueId.get(issue.id) ?? null })); } - function nextEvaluationAt(card: StatusCardRow, snapshot: WatchedIssue[], now: Date) { + function nextEvaluationAt(card: StatusCardRow, membershipAt: Date | null, now: Date) { const interval = nextStatusCardEvaluationAt(card.refreshPolicy, now); - if (card.refreshPolicy.mode !== "reactive" || !card.refreshPolicy.triggers.membershipChanges) return interval; - const expiries = snapshot.flatMap(issue => issue.scopeExpiresAt && issue.scopeExpiresAt > now ? [issue.scopeExpiresAt.getTime()] : []); - return expiries.length ? new Date(Math.min(...expiries)) : null; + return card.refreshPolicy.mode === "reactive" && (card.refreshPolicy.triggers.membershipChanges || card.refreshPolicy.triggers.anyUpdate) + ? membershipAt : interval; } async function requestRefresh(cardId: string, input: { @@ -670,13 +657,14 @@ export function statusCardService( } const now = input.now ?? new Date(); - const snapshot = await executeQueries(card); + let membershipAt: Date | null = null; + const snapshot = await executeQueries(card, {}, at => { if (at && (!membershipAt || at < membershipAt)) membershipAt = at; }); const fingerprint = buildStatusCardFingerprint(snapshot); const allChanges = diffStatusCardFingerprint(card.fingerprint as StatusCardFingerprint | null, fingerprint); const changes = filterStatusCardChanges(allChanges, card.refreshPolicy); const trigger = input.trigger ?? "manual"; const forceRun = trigger === "manual" || trigger === "restore"; - const nextEvalAt = nextEvaluationAt(card, snapshot, now); + const nextEvalAt = nextEvaluationAt(card, membershipAt, now); if (!forceRun && changes.length === 0) { const [next] = await mutate(card.companyId, tx => tx.update(statusCards).set({ pendingChangeCount: 0, @@ -797,9 +785,12 @@ export function statusCardService( return { card: next, generatingIssue: generationIssue!, alreadyGenerating: deduplicated, enqueued: true, kind, changes }; } - async function tickDueStatusCards(now = new Date(), reactiveCompanies?: ReadonlySet | "all" | "changed") { - const reactiveScope = reactiveCompanies === "all" ? isNull(statusCards.nextEvalAt) : reactiveCompanies === "changed" ? sql`true` : reactiveCompanies?.size + async function tickDueStatusCards(now = new Date(), reactiveCompanies?: ReadonlySet | "changed", reconstruct = false) { + const changedScope = reactiveCompanies === "changed" ? sql`true` : reactiveCompanies?.size ? inArray(statusCards.companyId, [...reactiveCompanies]) : sql`false`; + // Startup restores every disarmed reactive scope. A concurrent committed + // source hint can additionally preempt its company's saved future deadline. + const reactiveScope = or(reconstruct ? isNull(statusCards.nextEvalAt) : undefined, changedScope); const due = await db.select({ card: statusCards }).from(statusCards) .innerJoin(companies, eq(companies.id, statusCards.companyId)) .where(and(eq(companies.status, "active"), isNull(statusCards.archivedAt), isNull(statusCards.generatingIssueId), @@ -808,6 +799,7 @@ export function statusCardService( .then(rows => rows.map(row => row.card)); const enqueued: Array<{ cardId: string; generatingIssue: typeof issues.$inferSelect }> = []; let evaluated = 0; + const deferredCompanyIds = new Set(); for (const candidate of due) { // nextEvalAt is both a retry lease and an evaluation deadline. A source // event may preempt a genuine future deadline, so serialize scheduler @@ -817,7 +809,7 @@ export function statusCardService( const [lock] = await lockTx.execute<{ acquired: boolean }>( sql`select pg_try_advisory_xact_lock(hashtextextended(${candidate.id}, 0)) as acquired`, ); - if (!lock?.acquired) return; + if (!lock?.acquired) { deferredCompanyIds.add(candidate.companyId); return; } const claimUntil = new Date(now.getTime() + 5 * 60 * 1000); const [claimed] = await mutate(candidate.companyId, tx => tx.update(statusCards).set({ nextEvalAt: claimUntil }) .where(and(eq(statusCards.id, candidate.id), isNull(statusCards.archivedAt), isNull(statusCards.generatingIssueId), @@ -836,7 +828,7 @@ export function statusCardService( } }); } - return { evaluated, enqueued }; + return { evaluated, enqueued, deferredCompanyIds }; } async function writeSummary(cardId: string, input: WriteStatusCardSummary, actor: StatusCardWriter) { @@ -866,7 +858,8 @@ export function statusCardService( const mentionedIssueIds = await resolveMentionedIssueIds(current.companyId, input.markdown); // Current watched membership: compiled-query matches plus the issues this // summary mentions. - const watchedSnapshot = await executeQueries({ ...current, mentionedIssueIds }); + let membershipAt: Date | null = null; + const watchedSnapshot = await executeQueries({ ...current, mentionedIssueIds }, {}, at => { if (at && (!membershipAt || at < membershipAt)) membershipAt = at; }); const watchedNow = buildStatusCardFingerprint(watchedSnapshot); let snapshot: StatusCardFingerprint; if (payloadFingerprint) { @@ -940,7 +933,7 @@ export function statusCardService( lastChangeAt: null, nextEvalAt: current.refreshPolicy.mode === "reactive" && filterStatusCardChanges(diffStatusCardFingerprint(snapshot, watchedNow), current.refreshPolicy).length > 0 - ? now : nextEvaluationAt(current, watchedSnapshot, now), + ? now : nextEvaluationAt(current, membershipAt, now), updatedAt: now, }).where(and(eq(statusCards.id, current.id), eq(statusCards.generatingIssueId, input.generationIssueId))).returning(); if (!next) throw conflict("Status-card generation was superseded by a newer task"); diff --git a/server/src/services/task-watchdogs.ts b/server/src/services/task-watchdogs.ts index cf8d92d39df..84c7f92a8a6 100644 --- a/server/src/services/task-watchdogs.ts +++ b/server/src/services/task-watchdogs.ts @@ -1,3 +1,4 @@ +import { notifyStatusCardIssueChange } from "./status-card-notifications.js"; import { createHash } from "node:crypto"; import { and, asc, eq, gte, inArray, isNull, or, sql } from "drizzle-orm"; import type { Db } from "@paperclipai/db"; @@ -1398,10 +1399,13 @@ export function taskWatchdogService(db: Db, deps: TaskWatchdogServiceDeps = {}) }) ?? fallback : fallback; if (!shouldReopen && watchdogIssue.originFingerprint !== input.classification.stopFingerprint) { - await db + await db.transaction(async tx => { + await notifyStatusCardIssueChange(tx, input.watchdog.companyId); + await tx .update(issues) .set({ originFingerprint: input.classification.stopFingerprint, updatedAt: new Date() }) .where(and(eq(issues.companyId, input.watchdog.companyId), eq(issues.id, watchdogIssue.id))); + }); watchdogIssue.originFingerprint = input.classification.stopFingerprint; } await issuesSvc.addComment( diff --git a/server/src/services/workspace-runtime.ts b/server/src/services/workspace-runtime.ts index 600d5e32a42..2bb9239eaf3 100644 --- a/server/src/services/workspace-runtime.ts +++ b/server/src/services/workspace-runtime.ts @@ -1,3 +1,4 @@ +import { notifyStatusCardIssueChange } from "./status-card-notifications.js"; import { insertIssueComment } from "./issue-comment-writes.js"; import { restoreGitWorkspaceContents } from "./git-workspace-contents.js"; import { savedGitWorkspaceRestoration, ensureSavedGitCommit, GitCacheRestorationFailure } from "./git-workspace-restoration.js"; @@ -1636,10 +1637,11 @@ async function writeDirtyQuarantineAuditComments(input: { let sourceAuditCommentId: string | null = null; let claimantAuditCommentId: string | null = null; const now = new Date(); - if (input.evidence.sourceIssueId) { + const sourceIssueId = input.evidence.sourceIssueId; + if (sourceIssueId) { const [sourceComment] = await insertIssueComment(input.db, { companyId: input.companyId, - issueId: input.evidence.sourceIssueId, + issueId: sourceIssueId, authorAgentId: null, authorUserId: null, authorType: "system", @@ -1647,10 +1649,10 @@ async function writeDirtyQuarantineAuditComments(input: { body, }); sourceAuditCommentId = sourceComment?.id ?? null; - await input.db - .update(issues) - .set({ updatedAt: now }) - .where(eq(issues.id, input.evidence.sourceIssueId)); + await input.db.transaction(async tx => { + await notifyStatusCardIssueChange(tx, input.companyId); + await tx.update(issues).set({ updatedAt: now }).where(eq(issues.id, sourceIssueId)); + }); } const claimantIssueId = input.evidence.contention?.claimedByIssueId ?? null; @@ -1665,10 +1667,10 @@ async function writeDirtyQuarantineAuditComments(input: { body, }); claimantAuditCommentId = claimantComment?.id ?? null; - await input.db - .update(issues) - .set({ updatedAt: now }) - .where(eq(issues.id, claimantIssueId)); + await input.db.transaction(async tx => { + await notifyStatusCardIssueChange(tx, input.companyId); + await tx.update(issues).set({ updatedAt: now }).where(eq(issues.id, claimantIssueId)); + }); } return { sourceAuditCommentId, claimantAuditCommentId };