From dce383c733bb1ca31cd34d7dbb82bda597ad7418 Mon Sep 17 00:00:00 2001 From: Dotta <34892728+cryppadotta@users.noreply.github.com> Date: Sat, 31 Oct 2026 12:28:40 -0500 Subject: [PATCH] perf(server): stop recurring attention retention scans (#18388) Reviewed and verified at a61af9888e2e8cea782434568b5e7463db254d1c. Co-Authored-By: Paperclip --- .../2026-10-30-attention-retention-sleep.md | 66 + .../attention-retention-work.test.ts | 1088 +++++++++++++++++ .../src/__tests__/attention-service.test.ts | 2 +- ...cker-workspace-history-performance.test.ts | 103 ++ .../src/__tests__/company-portability.test.ts | 1 + .../decision-effect-recovery-work.test.ts | 4 +- ...retention-materialization-security.test.ts | 187 +++ .../src/__tests__/decisions-service.test.ts | 2 +- .../git-restore-point-ledger.test.ts | 2 +- .../__tests__/issue-blocker-attention.test.ts | 12 +- server/src/__tests__/issues-service.test.ts | 2 +- .../server-startup-feedback-export.test.ts | 18 +- server/src/__tests__/work-products.test.ts | 28 +- server/src/index.ts | 25 +- .../__tests__/lifecycle.test.ts | 2 +- server/src/services/accounting-integrity.ts | 3 + server/src/services/agent-configuration.ts | 2 + server/src/services/agent-deletion.ts | 2 + server/src/services/agent-hiring.ts | 2 + server/src/services/agent-lifecycle.ts | 2 + server/src/services/agent-project-access.ts | 2 + server/src/services/agents.ts | 6 +- server/src/services/attention-retention.ts | 324 +++++ .../services/attention-source-eligibility.ts | 118 ++ server/src/services/attention.ts | 119 +- server/src/services/budgets.ts | 23 +- server/src/services/company-portability.ts | 2 + server/src/services/decision-expiry.ts | 2 +- server/src/services/decision-retention.ts | 107 +- .../services/decision-work-notifications.ts | 13 +- server/src/services/decisions.ts | 25 +- .../services/delivery-work-notifications.ts | 1 + .../execution-control-notifications.ts | 3 + .../services/grok-bot/workspace-inspection.ts | 2 + server/src/services/heartbeat-run-events.ts | 9 +- server/src/services/heartbeat/agent-status.ts | 3 + server/src/services/heartbeat/retries.ts | 2 + server/src/services/issue-recovery-actions.ts | 2 + .../issue-thread-interactions.test.ts | 19 +- server/src/services/issue-title.ts | 2 + server/src/services/issue-visibility.ts | 8 +- server/src/services/issues.ts | 492 ++++---- .../native-finalization-notifications.ts | 2 + .../native-provider-capacity-retry.ts | 2 + .../native-review-participant.ts | 2 + .../native-session-executor.test.ts | 11 + .../native-workspace-reconstruction.ts | 2 + .../paperclip-runner-tool-authority.ts | 2 + .../superseded-external-review-requests.ts | 2 + ...ative-status-reassessment-notifications.ts | 2 + server/src/services/plugin-managed-agents.ts | 2 + .../services/retention-work-notifications.ts | 77 +- .../services/slack-conversation-lifecycle.ts | 2 + .../src/services/slack-conversation-state.ts | 2 + server/src/services/workspace-operations.ts | 9 +- 55 files changed, 2532 insertions(+), 422 deletions(-) create mode 100644 doc/plans/2026-10-30-attention-retention-sleep.md create mode 100644 server/src/__tests__/attention-retention-work.test.ts create mode 100644 server/src/__tests__/blocker-workspace-history-performance.test.ts create mode 100644 server/src/__tests__/decision-retention-materialization-security.test.ts create mode 100644 server/src/services/attention-retention.ts create mode 100644 server/src/services/attention-source-eligibility.ts diff --git a/doc/plans/2026-10-30-attention-retention-sleep.md b/doc/plans/2026-10-30-attention-retention-sleep.md new file mode 100644 index 00000000000..bf4c0d26d9a --- /dev/null +++ b/doc/plans/2026-10-30-attention-retention-sleep.md @@ -0,0 +1,66 @@ +# Archive attention sources without repeated feed scans + +Attention retention now reconstructs source deadlines at startup and after committed source changes. It no longer rebuilds every active company's full feed on each heartbeat tick. The archive age remains 90 days. Keep and manual archive state remain authoritative, and automatic archive still commits its notification outbox atomically. + +This document records source ownership, producer coverage, and the evidence needed to review the replacement. It makes no claim about host suspension or external wake alarms. + +## Existing source rows own eligibility and deadlines + +`attention-retention.ts` selects at most 100 unfinished source candidates per reconstruction page. It excludes inactive companies and kept or archived retention states. The scoped coordinator retains eligible deadlines, retries concrete failures, and cancels entries that leave attention. It does not schedule another scan after settlement unless a producer changes a relevant source. + +| Source | Eligible durable state | Activity owner | +| --- | --- | --- | +| Decision | Open decision with a visible execution issue | Decision updated time | +| Interaction | Pending board-visible interaction; newest ordinary confirmation or independent quality gate | Interaction updated time | +| Recovery action | Active or escalated action owned by a user or board, with a visible source issue | Recovery updated time | +| Blocker attention | Board unblock descriptor or non-live terminal blocker selected by the issue owner | Blocked transition or terminal issue updated time | +| Review | Human participant, user assignee, pending approval, or stalled review selected by the issue owner | Issue updated time | +| Failed run | Failed or timed-out run with an exhaustion event, eligible agent and issue, and no newer matching run | Finished time, falling back to updated or created time | +| Budget alert | Open hard incident, or soft incident meeting the existing 85 percent rounded threshold | Incident updated time | +| Agent error | Agent in error status | Agent updated time | + +The feed and worker share confirmation collapse, newer-run probes, issue visibility, review eligibility, and terminal-blocker predicates. Issue graph and agent invokability logic remain with their existing owners. Blocker entries retain both the root used to reconstruct the graph and the terminal subject that owns retention state. PostgreSQL deadlines round submillisecond timestamps upward. The worker locks the source and terminal subject before archival. + +`productivity_review` generates no feed items and owes no retention work. Queue retention-day overrides control shelf presentation, which is computed on reads. They do not change the fixed automatic archive age. Approval retention, join-request retention, and archive notification delivery keep their existing workers. + +## Existing mutation boundaries reactivate work + +Committed hints identify a company, source kind, source ID or changed issue where those values are known. Routine issue writers pass identity to the retention domain helper before their mutation. The helper captures old and new parent paths, reverse blocker ancestors, run-linked issues and liveness-origin references; issue-scoped source queries then select reviews, blockers, interactions, decisions, recovery actions and failed runs by their issue identity. Budget incidents and agent errors remain tied to their own source changes. Ordinary issue writes do not reconstruct the unrelated company feed. Opaque liveness and stranded-recovery origin references resolve through company-scoped issue rows before entering UUID queries, so valid none sentinels do not roll back writes and moved recovery tasks still invalidate their original review. Queue keys explicitly pick supported identity fields. Retention's own writes use exact source hints; blocker retention changes invalidate the company/source because multiple blocked roots can project one terminal subject. Reconstruction pages and due batches remain bounded. Review attention and interaction invokability are computed in company batches per candidate page; a full 100-candidate review page uses at most 20 statements and 100 confirmations use at most six. Decisions, recovery source links, runs, eligible agents, issue visibility, newer-run probes and blocker graphs also load in company batches per page. A 100-source decision or recovery page uses at most three statements; 100 failed runs use at most six; 100 blocker roots use at most 30 graph/source statements plus retention-state reads for their emitted entries. These evidence maps exist only for one load. The blocker owner shares fetched node and edge facts while tracking depth, node count, expanded nodes and live-work reachability independently for each root. The shared graph fetch stores at most 2,000 nodes and 4,000 edges, and bounds each frontier query before materializing rows. Positive live-work evidence from the bounded returned rows is retained independently of node/edge admission, including running status, active runs and pending confirmations. Filling the graph with explicit blockers cannot hide a subsequently returned active child. An internal batch result identifies roots whose evaluation is incomplete; the legacy map API exposes no extra transport fields. Retention schedules only those roots for an immediate fresh single-root check, which disarms or saves the actual source deadline. It never archives a partial intermediary or adds a recurring incomplete-root poll. Fully evaluated roots preserve their exact classification. All later graph queries, including recovery source IDs, agent IDs and workspace-finalize inputs, use bounded chunks. A SQL-generated 70,000-node graph proves PostgreSQL parameter limits are respected at the driver boundary. Archive checks still lock and freshly re-read sources, terminal subjects and related eligibility. Ordinary execution, decision and budget mutations pass their affected company; exact decision sources are included. + +Producer coverage includes decision creation and changes; interaction creation, resolution, expiry and supersession; issue creation, updates, blocker relations, checkout, adoption and release; agent hire, configuration, lifecycle, error clearing and deletion; budget incidents and company pause/resume; exhausted-run events and retry-run creation; native status, review, finalization and reconstruction; external conversation transitions; and imported blocker relations. Existing service transactions retain outer commit ownership. Savepoint release does not publish work before outer commit. + +The writer contract follows the existing delivery workers: serving processes use the transaction-aware service boundaries, and each serving process registers its workers. Independent raw SQL writers must supply equivalent ingress or restart reconstruction. Restores reconstruct work at startup. The change adds no permanent listener, independent timer, table, or Cloud wake mechanism. Existing sleep-safety gates remain in force. + +## Real PostgreSQL evidence proves quiet and reactivation + +The new acceptance suite exercises all eight sources, including terminal blocker identity and date, newest confirmations, independent quality gates, unhealthy reporting chains, conversation visibility, maintained reviews and pending approvals, newer-run suppression, ordinary task and interaction producers, ordinary exhaustion-event ingress, and ordinary error clearing. Additional ordinary producer controls cover child creation with a parent-blocking relation, budget incident creation and resolution, and repeated company-scoped execution-control writes. + +It also checks keep/revive, moved and cancelled deadlines, company activation and isolation, retained terminal history, restart, nested outer commit and rollback, first writes during drain, retry across drain, atomic archive/outbox rollback, and notifications during reconstruction. A 225-item backlog uses three startup pages and source-specific self-invalidations. SQL is counted at the PostgreSQL driver boundary. + +The old aggregate path issued 12 statements per tick in each of three configured-idle samples. The replacement issued zero statements during 300 seconds after settlement. One transaction-status read after an injected rollback is classified as uncertainty reconciliation; the following 300-second drain window must remain quiet. + +The focused suite supports `PAPERCLIP_ATTENTION_RETENTION_TEST_DATABASE_URL`, validates its dedicated `_test` database name, applies pending migrations, and prewarms a six-connection pool before advancing fake time. Providers are not involved in retention selection; the existing archive outbox delivery tests verify the separate controlled notification boundary. Fake time proves scheduling and SQL quiet, not database or host suspension. + +## Review and operational limits remain explicit + +Local embedded PostgreSQL could not start on the test host because system semaphores were exhausted. The accepted runs use isolated PostgreSQL 17 databases. A mixed consumer run also exposed fixture contamination from a lifecycle suite that retains native finalization rows; isolated reruns are required and preserve the failed run in the review evidence. Broad Linux CI remains the required full-suite gate. + +Review the source parity fixtures and producer hints together. The intended outcome is unchanged archive state and notifications with no recurring retention SQL when the sources remain unchanged. No merge or deployment is part of this change. + +Issue producer invalidation has its own finite discovery budget. Reverse parent/blocker traversal returns at most 101 distinct UUIDs in one recursive query, with each relation adjacency capped at 101 unique edges before materialization. At most 100 affected issues produce one sorted canonical batch hint; an overflow sentinel produces one company hint and the existing complete paged reconstruction. UNION deduplication terminates cycles. A PostgreSQL EXPLAIN ANALYZE on 2,100 reverse edges proves the outer limit stops at 101 CTE rows while the indexed relation branch returns only 100 rows. Repeated identical writes coalesce by canonical key. The shared scheduler is unchanged: a broader company hint does not cancel already pending narrower scopes; those remain finite batches of at most 100 IDs, processed at the existing 25-scope pass limit. This permits redundant bounded reconstruction instead of discarding invalidations or introducing cursor/subsumption policy across other owners. A 151-node cyclic chain proves overflow still registers the far ancestor's exact archive deadline and settles quietly afterward. + +The ordinary feed retains conservative `needs_attention` entries for roots outside the shared graph budget. The batch finalization visits every input root; a root without explored edges points attention at itself. A real PostgreSQL regression with 2,001 blocked roots verifies the nonempty deferred set, bounded graph size, ordinary issue-list metadata, and all 2,001 attention-feed subject IDs. This is a consumer parity proof; it does not increase graph traversal or change archive eligibility. + +## Completed blocker history and incomplete graph checks + +Explicit blocker queries apply the dependency owner's pending-finalization predicate before the graph row limit. A done blocker with no operation debt is excluded; attributed operations take precedence over legacy unattributed workspace operations, and a later successful export clears the barrier. The readiness helper and bounded attention query share this predicate, including the existing optional workspace/operation locks. Each relation resolves its globally unique issue identity before tenant and eligibility checks, avoiding repeated company-index scans. + +A fresh archive check consumes the private graph completeness result. An unexpanded frontier cannot establish absence of live work, so that check disarms without archiving. Supported issue, relation, run and workspace changes reactivate the root; restart reconstructs it. Completed-only history remains fully classified and can archive normally. No incomplete-root polling is added. + +The ordinary workspace-operation recorder also signals no-run finalization in its owning transaction. Export with a workspace identity invalidates that company's blocker source, including every root sharing the exported workspace; only a known issue without a workspace uses the existing reverse-issue hint. This rare source change uses bounded company/source paging and does not rebuild unrelated retention kinds. Run-backed finalization retains its existing native-status fanout. This lets a parked incomplete tree reactivate when export clears its done blockers, without an incomplete-root retry timer. Recorder rollback leaves the operation barrier and attention state unchanged. + +## First manual recovery-action materialization + +Creating a recovery-action retention row through Archive, Revive or Keep requires the action to be currently open and human-owned, with a visible source issue readable by the actor. The feed and manual materializer share the same recovery eligibility SQL. The materializer retains its action-row lock, same-company source join and canonical issue visibility/read authority. Existing retention rows retain their manual controls; TTL archival and other source kinds are unchanged. + +The real-owner regression creates an agent-owned action, attempts pre-archive, then exhausts that same recovery identity through the ordinary recovery service. The parent materializer hides the resulting board request; the repaired path rejects pre-archive and leaves the request visible. System ownership, resolved actions, hidden/harness/conversation sources and foreign-company records cannot create their first row. Existing-row Archive/Revive/Keep and aged legacy agent-owned TTL archival remain covered. The clean parent-module control fails on the lost visible board request; the repaired full materialization, decision-service and attention-service cohort passes 110 cases. A conversation fixture identity omission was caught and corrected locally before the clean causal control and final qualification; it is not counted as product failure evidence. diff --git a/server/src/__tests__/attention-retention-work.test.ts b/server/src/__tests__/attention-retention-work.test.ts new file mode 100644 index 00000000000..b722c75f395 --- /dev/null +++ b/server/src/__tests__/attention-retention-work.test.ts @@ -0,0 +1,1088 @@ +import { randomUUID } from "node:crypto"; +import { and, eq, sql, type SQL } from "drizzle-orm"; +import { PgDialect } from "drizzle-orm/pg-core"; +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; +import { subscribeDatabaseWork, authUsers, companyMemberships, agents, approvals, budgetIncidents, budgetPolicies, companies, createDb, applyPendingMigrations, decisionArchiveNotificationOutbox, + decisionRetention, decisions, costEvents, heartbeatRunEvents, heartbeatRuns, issueApprovals, issueRecoveryActions, + issueRelations, issueThreadInteractions, issues, type Db } from "@paperclipai/db"; +import { attentionRetentionService, registerAttentionRetentionWork } from "../services/attention-retention.js"; +import { workspaceOperationService } from "../services/workspace-operations.js"; +import * as workspaceOperationLogs from "../services/workspace-operation-log-store.js"; +import { budgetService } from "../services/budgets.js"; +import { writeExecutionControl } from "../services/execution-control-notifications.js"; +import { DELIVERY_QUEUES } from "../services/delivery-work-notifications.js"; +import { attentionService } from "../services/attention.js"; +import { companyService } from "../services/companies.js"; +import { createDeliveryWorkCoordinator } from "../services/delivery-work-coordinator.js"; +import { decisionRetentionService, DEFAULT_DECISION_ARCHIVE_DAYS } from "../services/decision-retention.js"; +import { appendHeartbeatRunEvent } from "../services/heartbeat-run-events.js"; +import { buildIssueGraphLivenessIncidentKey, RECOVERY_ORIGIN_KINDS } from "../services/recovery/origins.js"; +import { issueThreadInteractionService } from "../services/issue-thread-interactions.js"; +import { createAgentLifecycle } from "../services/agent-lifecycle.js"; +import { BLOCKER_ATTENTION_MAX_DEPTH, BLOCKER_ATTENTION_MAX_NODES, issueService } from "../services/issues.js"; +import { notifyAttentionRetentionWork, notifyIssueAttentionRetentionWork } from "../services/retention-work-notifications.js"; +import { idleWorkSnapshot } from "../services/task-admission.js"; +import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; + +import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js"; + +const externalUrl = resolveExternalTestDatabaseUrl("PAPERCLIP_ATTENTION_RETENTION_TEST_DATABASE_URL"); +const support = externalUrl ? { supported: true } : await getEmbeddedPostgresTestSupport(); +const delay = DEFAULT_DECISION_ARCHIVE_DAYS * 86_400_000; +(support.supported ? describe : describe.skip)("remaining attention retention", () => { + let temporary: Awaited> | undefined; + let db: ReturnType; + let companyId: string, agentId: string, draining = false; + let coordinator: ReturnType | undefined; + const errors: unknown[] = []; + const statements: string[] = []; + const parameterCounts: number[] = []; + let captureBlockerPlan = false; + let blockerPlanQuery: { query: string; parameters: unknown[] } | undefined; + beforeAll(async () => { + if (externalUrl) await applyPendingMigrations(externalUrl); + else temporary = await startEmbeddedPostgresTestDatabase("attention-retention-"); + db = createDb(externalUrl ?? temporary!.connectionString, { maxConnections: 6, idleTimeoutSeconds: 0, maxLifetimeSeconds: 0 }); + db.$client.options.onnotice = () => {}; + db.$client.options.debug = (_connection, query, parameters) => { + statements.push(query); parameterCounts.push(parameters.length); + if (captureBlockerPlan && !blockerPlanQuery && query.includes("latest_blocker_operation") && query.includes("issue_relations")) + blockerPlanQuery = { query, parameters: [...parameters] }; + }; + }, 30_000); + beforeEach(async () => { + companyId = randomUUID(); agentId = randomUUID(); draining = false; errors.length = 0; + captureBlockerPlan = false; blockerPlanQuery = undefined; + await db.insert(companies).values({ id: companyId, name: "Retention", issuePrefix: `A${companyId.slice(0, 6)}` }); + await db.insert(authUsers).values({ id: "board", name: "Board", email: "board@retention.test", createdAt: new Date(), updatedAt: new Date() }).onConflictDoNothing(); + await db.insert(companyMemberships).values({ companyId, principalType: "user", principalId: "board", membershipRole: "owner", status: "active" }); + await db.insert(agents).values({ id: agentId, companyId, name: "Origin", role: "engineer", status: "active" }); + await Promise.all(Array.from({ length: 6 }, () => db.execute(sql`select pg_sleep(0.01)`))); + vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout", "setInterval", "clearInterval"], shouldAdvanceTime: true }); + vi.setSystemTime(new Date("2026-11-01T12:00:00Z")); + }); + afterEach(async () => { + await coordinator?.stop(); coordinator = undefined; + vi.useRealTimers(); vi.restoreAllMocks(); + await db.transaction(async tx => { + await tx.execute(sql`set local client_min_messages = warning`); + await tx.execute(sql`drop trigger if exists attention_retention_fail on decision_retention`); + await tx.execute(sql`drop function if exists attention_retention_fail()`); + await tx.execute(sql`truncate companies cascade`); + }); + expect(errors).toEqual([]); + }); + afterAll(async () => { await db?.$client.end({ timeout: 1 }); await temporary?.cleanup(); }); + const old = () => new Date(Date.now() - delay - 600_000); + async function insertIssue(status = "todo", values: Partial = {}, database: Db = db) { + const [row] = await database.insert(issues).values({ companyId, title: randomUUID(), status, + createdByAgentId: agentId, createdAt: old(), updatedAt: old(), ...values }).returning(); + return row; + } + async function change(write: (tx: Db) => PromiseLike, scope = { companyId }) { + return db.transaction(async tx => { await notifyAttentionRetentionWork(tx, scope); return write(tx as unknown as Db); }); + } + const state = (kind: string, id: string) => db.select().from(decisionRetention).where(and( + eq(decisionRetention.companyId, companyId), eq(decisionRetention.sourceKind, kind), eq(decisionRetention.sourceId, id))) + .then(rows => rows[0] ?? null); + async function settle() { + await vi.waitFor(async () => { await vi.advanceTimersByTimeAsync(1000); expect(idleWorkSnapshot().active).toBe(0); }, { timeout: 15_000 }); + } + async function quiet() { + await settle(); expect(coordinator!.nextWakeAt()).toBeNull(); + const select = vi.spyOn(db, "select"), execute = vi.spyOn(db, "execute"), transaction = vi.spyOn(db, "transaction"); + statements.length = 0; + await vi.advanceTimersByTimeAsync(300_000); + expect(statements).toEqual([]); + expect(select).not.toHaveBeenCalled(); expect(execute).not.toHaveBeenCalled(); expect(transaction).not.toHaveBeenCalled(); + select.mockRestore(); execute.mockRestore(); transaction.mockRestore(); + } + async function start(service = attentionRetentionService(db)) { + coordinator = createDeliveryWorkCoordinator({ owner: db, canRun: () => !draining, canReconcile: () => true, onError: error => errors.push(error) }); + await registerAttentionRetentionWork(coordinator, service, () => !draining).ready; + return service; + } + async function archived(kind: string, id: string) { + await vi.waitFor(async () => { await vi.advanceTimersByTimeAsync(1000); expect((await state(kind, id))?.archivedAt).toBeInstanceOf(Date); }, { timeout: 15_000 }); + } + async function loadAll(service = attentionRetentionService(db)) { + const entries = []; + for (;;) { const page = await service.load({ companyId }); entries.push(...page.rows ?? []); if (page.complete) return entries; } + } + const mutation = (sourceKind: "review", sourceId: string) => ({ companyId, sourceKind, sourceId, + authActor: { type: "board" as const, source: "local_implicit" as const, userId: "board", companyIds: [companyId], isInstanceAdmin: true }, + actor: { actorType: "user" as const, actorId: "board", userId: "board", agentId: null, runId: null, agentApiKeyId: null, responsibleUserId: "board" } }); + + it("records the aggregate baseline SQL across old ticks and eliminates it after settlement", async () => { + await insertIssue("done"); + const ticks: number[] = []; + for (let tick = 0; tick < 3; tick++) { + statements.length = 0; + const feed = await attentionService(db).list(companyId, { includeDismissed: true, excludeSources: ["approval", "join_request"], all: true, allowUnscopedAll: true }); + await decisionRetentionService(db).autoArchive({ companyId, items: feed.items }); + ticks.push(statements.length); + await vi.advanceTimersByTimeAsync(30_000); + } + expect(ticks.every(count => count > 0)).toBe(true); + process.stdout.write(JSON.stringify({ baselineStatementsPer30sTick: ticks, treatmentIdle300s: 0 }) + "\n"); + await start(); await quiet(); + }); + + it("has no recurring SQL when empty or when terminal source history remains", async () => { + await insertIssue("done"); await insertIssue("cancelled"); + await start(); await quiet(); + }); + + it("wakes from the ordinary task producer, waits until due, archives with an outbox, and settles", async () => { + await start(); await quiet(); + const row = await issueService(db).create(companyId, { title: "Review a delivered result", status: "in_review", assigneeUserId: "board", createdByAgentId: agentId }); + await settle(); + const [saved] = await db.select({ at: sql`ceil(extract(epoch from ${issues.updatedAt}) * 1000)`.mapWith(Number) }).from(issues).where(eq(issues.id, row.id)); + const due = saved.at + delay; + expect(coordinator!.nextWakeAt()).toBe(due); expect(await state("review", row.id)).toBeNull(); + const select = vi.spyOn(db, "select"), execute = vi.spyOn(db, "execute"); + await vi.advanceTimersByTimeAsync(due - Date.now() - 1000); + expect(select).not.toHaveBeenCalled(); expect(execute).not.toHaveBeenCalled(); select.mockRestore(); execute.mockRestore(); + await vi.advanceTimersByTimeAsync(2000); await archived("review", row.id); await quiet(); + const outbox = await db.select().from(decisionArchiveNotificationOutbox); + expect(outbox).toHaveLength(1); expect(outbox[0]).toMatchObject({ companyId, sourceKind: "review", sourceId: row.id, originAgentId: agentId, originIssueId: row.id }); + expect((await db.select().from(issues).where(eq(issues.id, row.id)))[0].status).toBe("in_review"); + await coordinator!.stop(); coordinator = undefined; await start(); await quiet(); + }); + + it("registers the ordinary createChild blocker path inside its relation transaction", async () => { + const parent = await issueService(db).create(companyId, { title: "Parent", status: "in_progress", assigneeAgentId: agentId, createdByAgentId: agentId }); + await start(); await quiet(); + const { issue: child, parentBlockerAdded } = await issueService(db).createChild(parent.id, { + title: "Child needs review", status: "in_review", assigneeUserId: "board", createdByAgentId: agentId, + blockParentUntilDone: true, + }); + expect(parentBlockerAdded).toBe(true); + expect(await db.select().from(issueRelations).where(and(eq(issueRelations.issueId, child.id), + eq(issueRelations.relatedIssueId, parent.id)))).toHaveLength(1); + await settle(); expect(coordinator!.nextWakeAt()).not.toBeNull(); + await vi.advanceTimersByTimeAsync(delay + 2000); await archived("review", child.id); await quiet(); + expect(await db.select().from(decisionArchiveNotificationOutbox).where(eq(decisionArchiveNotificationOutbox.sourceId, child.id))).toHaveLength(1); + }); + + it("batches a full page of maintained reviews and confirmations by company", async () => { + const rows = await db.insert(issues).values(Array.from({ length: 100 }, () => ({ companyId, + title: randomUUID(), status: "in_review", assigneeAgentId: agentId, updatedAt: old() }))).returning(); + await db.insert(issueThreadInteractions).values(rows.map(row => ({ companyId, issueId: row.id, + kind: "request_confirmation", addresseeAgentId: agentId, payload: { version: 1, prompt: "Agent verdict" } }))); + statements.length = 0; + const page = await attentionRetentionService(db).load({ companyId, sourceKind: "review" }); + expect(page.rows).toEqual([]); expect(page.complete).toBe(false); + expect(statements.length).toBeLessThanOrEqual(20); + expect(statements.filter(query => query.includes('from "agents"')).length).toBeLessThanOrEqual(1); + await db.update(agents).set({ status: "paused" }).where(eq(agents.id, agentId)); + statements.length = 0; + const interactions = await attentionRetentionService(db).load({ companyId, sourceKind: "issue_thread_interaction" }); + expect(interactions.rows).toHaveLength(100); expect(interactions.complete).toBe(false); + expect(statements.length).toBeLessThanOrEqual(6); + expect(statements.filter(query => query.includes('from "agents"'))).toHaveLength(1); + }); + + it("batches decision and recovery source links and rechecks a late hidden origin", async () => { + const origins = await db.insert(issues).values(Array.from({ length: 100 }, () => ({ companyId, + title: randomUUID(), status: "todo", updatedAt: old() }))).returning(); + const [run] = await db.insert(heartbeatRuns).values({ companyId, agentId, status: "succeeded" }).returning(); + const decisionRows = await db.insert(decisions).values(origins.map(origin => ({ companyId, + originAgentId: agentId, originIssueId: origin.id, originRunId: run.id, title: "Decision", body: "Choose", + options: [], signedSpec: "fixture", targetSnapshots: {}, expiresAt: new Date(Date.now() + 86_400_000), updatedAt: old() }))).returning(); + const recoveryRows = await db.insert(issueRecoveryActions).values(origins.map(origin => ({ companyId, + sourceIssueId: origin.id, kind: "execution", ownerType: "board", cause: "fixture", fingerprint: "fixture", + nextAction: "Choose", updatedAt: old() }))).returning(); + const service = attentionRetentionService(db); + const pages = []; + for (const sourceKind of ["decision", "recovery_action"] as const) { + statements.length = 0; + const page = await service.load({ companyId, sourceKind }); + expect(page.rows).toHaveLength(100); expect(page.complete).toBe(false); + expect(statements.length).toBeLessThanOrEqual(3); + pages.push(page.rows!); + } + await db.update(issues).set({ hiddenAt: new Date() }).where(eq(issues.id, origins[99].id)); + const hiddenDecision = pages[0].find(entry => entry.sourceId === decisionRows[99].id)!; + const hiddenRecovery = pages[1].find(entry => entry.sourceId === recoveryRows[99].id)!; + expect(await service.check(hiddenDecision)).toBeNull(); expect(await service.check(hiddenRecovery)).toBeNull(); + expect(await state("decision", hiddenDecision.sourceId)).toBeNull(); expect(await state("recovery_action", hiddenRecovery.sourceId)).toBeNull(); + expect(await db.select().from(decisionArchiveNotificationOutbox)).toHaveLength(0); + }); + + it("batches 100 failed runs and rechecks late visibility and supersession before archive", async () => { + const origins = await db.insert(issues).values(Array.from({ length: 100 }, () => ({ companyId, + title: randomUUID(), status: "todo", updatedAt: old() }))).returning(); + const runs = await db.insert(heartbeatRuns).values(origins.map(origin => ({ companyId, agentId, + status: "failed", contextSnapshot: { issueId: origin.id }, createdAt: old(), finishedAt: old() }))).returning(); + await db.insert(heartbeatRunEvents).values(runs.map(run => ({ companyId, agentId, runId: run.id, + seq: 1, eventType: "lifecycle", message: "Bounded retry exhausted: batch fixture" }))); + const service = attentionRetentionService(db), scope = { companyId, sourceKind: "failed_run" as const }; + statements.length = 0; + const page = await service.load(scope); + expect(page.rows).toHaveLength(100); expect(page.complete).toBe(false); + expect(statements.length).toBeLessThanOrEqual(6); + const hiddenRun = runs[98], supersededRun = runs[99]; + const hidden = page.rows!.find(entry => entry.sourceId === hiddenRun.id)!; + const superseded = page.rows!.find(entry => entry.sourceId === supersededRun.id)!; + await db.update(issues).set({ hiddenAt: new Date() }).where(eq(issues.id, origins[98].id)); + await db.insert(heartbeatRuns).values({ companyId, agentId, status: "succeeded", + contextSnapshot: { issueId: origins[99].id }, createdAt: new Date(old().getTime() + 60_000), finishedAt: old() }); + expect(await service.check(hidden)).toBeNull(); expect(await service.check(superseded)).toBeNull(); + expect(await state("failed_run", hiddenRun.id)).toBeNull(); expect(await state("failed_run", supersededRun.id)).toBeNull(); + expect(await db.select().from(decisionArchiveNotificationOutbox)).toHaveLength(0); + expect((await service.load(scope)).complete).toBe(true); + const refreshed = await service.load(scope); + expect(refreshed.rows).toHaveLength(98); + expect(refreshed.rows!.some(entry => [hiddenRun.id, supersededRun.id].includes(entry.sourceId))).toBe(false); + }); + + it("batches 100 blocker graphs with exact terminal timestamps and fresh terminal visibility", async () => { + const roots = await db.insert(issues).values(Array.from({ length: 100 }, () => ({ companyId, + title: randomUUID(), status: "blocked", updatedAt: old(), blockedTransitionAt: old(), + unblockDescriptor: { owner: "board", action: "Pick an owner" } }))).returning(); + const terminals = await db.insert(issues).values(Array.from({ length: 100 }, () => ({ companyId, + title: randomUUID(), status: "backlog", assigneeAgentId: agentId, updatedAt: old() }))).returning(); + await db.insert(issueRelations).values(roots.map((root, index) => ({ companyId, + issueId: terminals[index].id, relatedIssueId: root.id, type: "blocks" }))); + await db.execute(sql`update issues set updated_at = updated_at + interval '0.000321 seconds' where id = ${terminals[99].id}::uuid`); + const service = attentionRetentionService(db), scope = { companyId, sourceKind: "blocker_attention" as const }; + statements.length = 0; + const page = await service.load(scope); + const queries = [...statements]; + expect(page.rows).toHaveLength(200); expect(page.complete).toBe(false); + const retentionReads = queries.filter(query => query.includes('from "decision_retention"')); + expect(retentionReads).toHaveLength(200); + expect(queries.length - retentionReads.length).toBeLessThanOrEqual(30); + const terminal = page.rows!.find(entry => entry.rootId === roots[99].id && entry.variant === "terminal")!; + expect(terminal.sourceId).toBe(terminals[99].id); + const [expected] = await db.execute(sql`select updated_at::text as at, + ceil(extract(epoch from updated_at) * 1000 + ${delay}) as due from issues where id = ${terminals[99].id}::uuid`); + expect(terminal.activityAt).toBe(expected.at); expect(terminal.dueAt).toBe(Number(expected.due)); + await db.update(issues).set({ hiddenAt: new Date() }).where(eq(issues.id, terminals[99].id)); + expect(await service.check(terminal)).toBeNull(); + expect(await state("blocker_attention", terminals[99].id)).toBeNull(); + expect(await db.select().from(decisionArchiveNotificationOutbox)).toHaveLength(0); + expect((await service.load(scope)).complete).toBe(true); + const refreshed = await service.load(scope); + expect(refreshed.rows!.some(entry => entry.sourceId === terminals[99].id)).toBe(false); + }); + + it.each(["depth", "nodes"] as const)("preserves shallow terminal selection when an unrelated graph hits its %s limit", async limit => { + const root = await insertIssue("blocked", { updatedAt: new Date() }); + const middle = await insertIssue("todo", { updatedAt: new Date() }); + const terminal = await insertIssue("backlog", { assigneeAgentId: agentId }); + const heavy = await insertIssue("blocked", { updatedAt: new Date() }); + const count = limit === "depth" ? BLOCKER_ATTENTION_MAX_DEPTH + 1 : BLOCKER_ATTENTION_MAX_NODES; + const heavyNodes = await db.insert(issues).values(Array.from({ length: count }, () => ({ companyId, + title: randomUUID(), status: "todo", updatedAt: new Date() }))).returning(); + await db.insert(issueRelations).values([ + { companyId, issueId: middle.id, relatedIssueId: root.id, type: "blocks" }, + { companyId, issueId: terminal.id, relatedIssueId: middle.id, type: "blocks" }, + ...heavyNodes.map((node, index) => ({ companyId, issueId: node.id, + relatedIssueId: limit === "nodes" || index === 0 ? heavy.id : heavyNodes[index - 1].id, type: "blocks" })), + ]); + const owner = issueService(db); + const expected = (await owner.listBlockerAttention(companyId, [root])).get(root.id)!; + const limited = (await owner.listBlockerAttention(companyId, [heavy])).get(heavy.id)!; + expect(expected.terminalBlockerIssueId).toBe(terminal.id); + expect(limited.terminalBlockerIssueId).toBe(limited.directBlockerIssueId); + for (const input of [[root, heavy], [heavy, root]]) { + const batch = await owner.listBlockerAttentionBatch(companyId, input); + if (!batch.deferredRootIds.has(root.id)) expect(batch.attention.get(root.id)).toEqual(expected); + if (!batch.deferredRootIds.has(heavy.id)) expect(batch.attention.get(heavy.id)).toEqual(limited); + expect(batch.nodeCount).toBeLessThanOrEqual(BLOCKER_ATTENTION_MAX_NODES); + if (limit === "depth") expect(batch.deferredRootIds.has(root.id)).toBe(false); + } + const page = await attentionRetentionService(db).load({ companyId, sourceKind: "blocker_attention" }); + const entry = page.rows!.find(item => item.rootId === root.id && item.variant === "terminal")!; + expect(entry.dueAt).toBeLessThanOrEqual(Date.now()); + if (limit === "depth") expect(entry.sourceId).toBe(terminal.id); + await attentionRetentionService(db).check(entry); + expect((await state("blocker_attention", terminal.id))?.archivedAt).toBeInstanceOf(Date); + expect(await db.select().from(decisionArchiveNotificationOutbox).where(eq(decisionArchiveNotificationOutbox.sourceId, terminal.id))).toHaveLength(1); + }); + + it("keeps root-specific reachability and cycle budgets when another root explores farther", async () => { + const root = await insertIssue("blocked"); + const chain = await db.insert(issues).values(Array.from({ length: BLOCKER_ATTENTION_MAX_DEPTH + 1 }, (_, index) => ({ + companyId, title: randomUUID(), status: index === BLOCKER_ATTENTION_MAX_DEPTH ? "in_progress" + : index === BLOCKER_ATTENTION_MAX_DEPTH - 2 ? "blocked" : "todo", updatedAt: old() }))).returning(); + const overlapping = chain[BLOCKER_ATTENTION_MAX_DEPTH - 2]; + const cycle = await insertIssue("blocked"); + const leaf = await insertIssue("backlog", { assigneeAgentId: agentId }); + await db.insert(issueRelations).values([ + ...chain.map((node, index) => ({ companyId, issueId: node.id, relatedIssueId: index ? chain[index - 1].id : root.id, type: "blocks" })), + { companyId, issueId: leaf.id, relatedIssueId: cycle.id, type: "blocks" }, + { companyId, issueId: cycle.id, relatedIssueId: leaf.id, type: "blocks" }, + ]); + const owner = issueService(db), roots = [root, overlapping, cycle]; + const singles = new Map(); + for (const item of roots) singles.set(item.id, (await owner.listBlockerAttention(companyId, [item])).get(item.id)); + expect(singles.get(root.id).blockingTreeLive).toBe(false); + expect(singles.get(overlapping.id).blockingTreeLive).toBe(true); + for (const input of [roots, [...roots].reverse()]) { + const batch = await owner.listBlockerAttention(companyId, input); + for (const item of roots) expect(batch.get(item.id)).toEqual(singles.get(item.id)); + } + }); + + it("keeps deferred roots visible in the ordinary attention feed above the root cap", async () => { + await db.execute(sql`insert into issues (id, company_id, title, status, updated_at) + select gen_random_uuid(), ${companyId}::uuid, 'overflow-root-' || n, 'blocked', ${new Date().toISOString()}::timestamptz + from generate_series(1, ${BLOCKER_ATTENTION_MAX_NODES + 1}) n`); + const roots = await db.select().from(issues).where(eq(issues.companyId, companyId)); + const batch = await issueService(db).listBlockerAttentionBatch(companyId, roots); + expect(batch.deferredRootIds.size).toBeGreaterThan(0); + expect(batch.nodeCount).toBeLessThanOrEqual(BLOCKER_ATTENTION_MAX_NODES); + for (const rootId of batch.deferredRootIds) expect(batch.attention.get(rootId)).toMatchObject({ + state: "needs_attention", reason: "attention_required", terminalBlockerIssueId: rootId, + }); + const listed = await issueService(db).list(companyId, { status: "blocked", includeBlockedBy: true }); + expect(listed).toHaveLength(roots.length); + for (const root of listed) expect(root.blockerAttention).toMatchObject({ state: "needs_attention", terminalBlockerIssueId: root.id }); + const feed = await attentionService(db).list(companyId, { all: true, allowUnscopedAll: true }); + const visible = new Set(feed.items.filter(item => item.sourceKind === "blocker_attention").map(item => item.subject.id)); + expect(visible).toEqual(new Set(roots.map(root => root.id))); + }); + + it("bounds every graph query when independent root trees contain over 65535 unfinished nodes", async () => { + await db.execute(sql`insert into issues (id, company_id, title, status, updated_at) + select gen_random_uuid(), ${companyId}::uuid, 'parameter-root-' || n, 'blocked', ${old().toISOString()}::timestamptz + from generate_series(1, 70) n`); + await db.execute(sql`insert into issues (id, company_id, parent_id, title, status, updated_at) + select gen_random_uuid(), ${companyId}::uuid, root.id, 'parameter-branch-' || n, 'todo', ${old().toISOString()}::timestamptz + from issues root cross join generate_series(1, 10) n + where root.company_id = ${companyId}::uuid and root.status = 'blocked'`); + await db.execute(sql`insert into issues (id, company_id, parent_id, title, status, assignee_agent_id, updated_at) + select gen_random_uuid(), ${companyId}::uuid, branch.id, 'parameter-leaf-' || n, 'backlog', ${agentId}::uuid, ${old().toISOString()}::timestamptz + from issues branch cross join generate_series(1, 99) n + where branch.company_id = ${companyId}::uuid and branch.title like 'parameter-branch-%'`); + const completed = await insertIssue("blocked"); + const shallow = await insertIssue("blocked", { updatedAt: new Date() }); + const middle = await insertIssue("todo", { updatedAt: new Date() }); + const overdue = await insertIssue("backlog", { assigneeAgentId: agentId }); + await db.insert(issueRelations).values([ + { companyId, issueId: middle.id, relatedIssueId: shallow.id, type: "blocks" }, + { companyId, issueId: overdue.id, relatedIssueId: middle.id, type: "blocks" }, + ]); + const roots = await db.select().from(issues).where(and(eq(issues.companyId, companyId), eq(issues.status, "blocked"))); + parameterCounts.length = 0; + const result = await issueService(db).listBlockerAttentionBatch(companyId, roots).catch(error => { + throw new Error(`Large graph failed: ${String(error.cause?.message ?? error.message).slice(0, 300)}`); + }); + expect(result.attention.size).toBe(72); + expect(result.deferredRootIds.has(completed.id)).toBe(false); + expect(result.attention.get(completed.id)?.terminalBlockerIssueId).toBe(completed.id); + expect([...result.attention.values()].every(item => item.state === "needs_attention" && item.terminalBlockerIssueId)).toBe(true); + expect(Math.max(...parameterCounts)).toBeLessThanOrEqual(1100); + expect(parameterCounts.length).toBeLessThanOrEqual(100); + expect(result.nodeCount).toBeLessThanOrEqual(BLOCKER_ATTENTION_MAX_NODES); + expect(result.edgeCount).toBeLessThanOrEqual(BLOCKER_ATTENTION_MAX_NODES * 2); + expect(result.fetchedRows).toBeLessThanOrEqual(BLOCKER_ATTENTION_MAX_NODES * 6); + expect(result.deferredRootIds.has(shallow.id)).toBe(true); + const service = attentionRetentionService(db); + const page = await service.load({ companyId, sourceKind: "blocker_attention" }); + const immediate = page.rows!.find(entry => entry.rootId === shallow.id && entry.variant === "terminal")!; + expect(immediate.dueAt).toBeLessThanOrEqual(Date.now()); + await service.check(immediate); + expect((await state("blocker_attention", overdue.id))?.archivedAt).toBeInstanceOf(Date); + }, 120_000); + + it("preserves observed live work when a single large root exhausts shared fetch capacity", async () => { + const root = await insertIssue("blocked"); + const live = await insertIssue("in_progress", { assigneeAgentId: agentId }); + await db.insert(issueRelations).values({ companyId, issueId: live.id, relatedIssueId: root.id, type: "blocks" }); + await db.execute(sql`insert into issues (id, company_id, parent_id, title, status, assignee_agent_id, updated_at) + select gen_random_uuid(), ${companyId}::uuid, ${root.id}::uuid, 'bounded-leaf-' || n, 'backlog', ${agentId}::uuid, ${old().toISOString()}::timestamptz + from generate_series(1, 3000) n`); + const batch = await issueService(db).listBlockerAttentionBatch(companyId, [root]); + expect(batch.deferredRootIds.has(root.id)).toBe(true); + expect(batch.attention.get(root.id)?.blockingTreeLive).toBe(true); + const service = attentionRetentionService(db), page = await service.load({ companyId, sourceKind: "blocker_attention" }); + const entry = page.rows!.find(item => item.rootId === root.id && item.variant === "terminal")!; + expect(await service.check(entry)).toBeNull(); + expect(await state("blocker_attention", root.id)).toBeNull(); + expect(await db.select().from(decisionArchiveNotificationOutbox)).toHaveLength(0); + }); + + it.each(["in_progress", "run", "confirmation"])("preserves observed child live evidence (%s) after explicit blockers fill the node budget", async (kind) => { + const root = await insertIssue("blocked"); + await db.execute(sql`with blockers as ( + insert into issues (id, company_id, title, status, assignee_agent_id, updated_at) + select gen_random_uuid(), ${companyId}::uuid, 'explicit-leaf-' || n, 'backlog', ${agentId}::uuid, ${old().toISOString()}::timestamptz + from generate_series(1, 1999) n returning id) + insert into issue_relations (company_id, issue_id, related_issue_id, type) + select ${companyId}::uuid, id, ${root.id}::uuid, 'blocks' from blockers`); + const child = await insertIssue(kind === "in_progress" ? "in_progress" : "todo", { parentId: root.id, assigneeAgentId: agentId }); + if (kind === "run") { + const [run] = await db.insert(heartbeatRuns).values({ companyId, agentId, status: "running" }).returning(); + await db.update(issues).set({ executionRunId: run.id }).where(eq(issues.id, child.id)); + } else if (kind === "confirmation") { + await db.insert(issueThreadInteractions).values({ companyId, issueId: child.id, kind: "request_confirmation", status: "pending", payload: { prompt: "Approve?" } }); + } + parameterCounts.length = 0; + const batch = await issueService(db).listBlockerAttentionBatch(companyId, [root]); + expect(batch.nodeCount).toBeLessThanOrEqual(BLOCKER_ATTENTION_MAX_NODES); + expect(batch.edgeCount).toBeLessThanOrEqual(BLOCKER_ATTENTION_MAX_NODES * 2); + expect(batch.fetchedRows).toBeLessThanOrEqual(BLOCKER_ATTENTION_MAX_NODES * 6); + expect(parameterCounts.length).toBeLessThanOrEqual(100); + expect(Math.max(...parameterCounts)).toBeLessThanOrEqual(1100); + expect(batch.deferredRootIds.has(root.id)).toBe(true); + expect(batch.attention.get(root.id)?.blockingTreeLive).toBe(true); + const service = attentionRetentionService(db), page = await service.load({ companyId, sourceKind: "blocker_attention" }); + const entry = page.rows!.find(item => item.rootId === root.id && item.variant === "terminal")!; + expect(await service.check(entry)).toBeNull(); + expect(await db.select().from(decisionRetention)).toHaveLength(0); + expect(await db.select().from(decisionArchiveNotificationOutbox)).toHaveLength(0); + }); + + it.each(["running", "run", "confirmation", "completed"] as const)("filters completed blocker history before bounded %s attention and archive checks", async kind => { + const root = await insertIssue("blocked"), workspaceId = randomUUID(); + await db.execute(sql`insert into execution_workspaces (id, company_id, name, mode, strategy_type) + values (${workspaceId}::uuid, ${companyId}::uuid, 'Finalized shared history', 'shared', 'project_primary')`); + await db.execute(sql`insert into workspace_operations (company_id, execution_workspace_id, phase, status, started_at) + values (${companyId}::uuid, ${workspaceId}::uuid, 'workspace_finalize', 'succeeded', ${old().toISOString()}::timestamptz)`); + await db.execute(sql`with history as ( + insert into issues (id, company_id, title, status, updated_at, execution_workspace_id) + select ('00000000-0000-4000-8000-' || lpad(to_hex(n), 12, '0'))::uuid, + ${companyId}::uuid, 'completed-blocker-' || n, 'done', ${old().toISOString()}::timestamptz, ${workspaceId}::uuid + from generate_series(1, 2100) n returning id) + insert into issue_relations (company_id, issue_id, related_issue_id, type) + select ${companyId}::uuid, id, ${root.id}::uuid, 'blocks' from history`); + if (kind !== "completed") { + const live = await insertIssue(kind === "running" ? "in_progress" : "todo", { id: "ffffffff-ffff-4fff-8fff-ffffffffffff", assigneeAgentId: agentId }); + await db.insert(issueRelations).values({ companyId, issueId: live.id, relatedIssueId: root.id, type: "blocks" }); + if (kind === "run") { + const [run] = await db.insert(heartbeatRuns).values({ companyId, agentId, status: "running" }).returning(); + await db.update(issues).set({ executionRunId: run.id }).where(eq(issues.id, live.id)); + } else if (kind === "confirmation") await db.insert(issueThreadInteractions).values({ companyId, issueId: live.id, + kind: "request_confirmation", status: "pending", payload: { prompt: "Still waiting" } }); + } + statements.length = 0; parameterCounts.length = 0; + captureBlockerPlan = kind === "completed"; + const batch = await issueService(db).listBlockerAttentionBatch(companyId, [root]); + captureBlockerPlan = false; + if (kind === "completed") { + expect(blockerPlanQuery).toBeTruthy(); + const explain = await db.$client.unsafe("EXPLAIN (ANALYZE, BUFFERS, FORMAT JSON) " + blockerPlanQuery!.query, + blockerPlanQuery!.parameters as never[]); + type Plan = { "Relation Name"?: string; "Actual Rows": number; "Actual Loops": number; "Rows Removed by Filter"?: number; Plans?: Plan[] }; + const rootPlan = (explain[0]!["QUERY PLAN"] as Array<{ Plan: Plan }>)[0]!.Plan; + const visit = (node: Plan): void => { + if (node["Relation Name"] === "issues" && node["Actual Loops"] > 1) { + expect(node["Actual Rows"]).toBeLessThanOrEqual(1); + expect(node["Rows Removed by Filter"] ?? 0).toBe(0); + } + node.Plans?.forEach(visit); + }; + visit(rootPlan); + process.stdout.write(JSON.stringify({ completedBlockerHistoryPlan: explain }) + "\n"); + } + expect(batch.nodeCount).toBeLessThanOrEqual(BLOCKER_ATTENTION_MAX_NODES); + expect(batch.edgeCount).toBeLessThanOrEqual(BLOCKER_ATTENTION_MAX_NODES * 2); + const service = attentionRetentionService(db), page = await service.load({ companyId, sourceKind: "blocker_attention" }); + const entry = page.rows!.find(row => row.rootId === root.id && row.variant === "terminal"); + if (kind !== "completed") { + // Exercise the authoritative fresh check even if discovery correctly omits the live root. + await service.check(entry ?? { id: `blocker_attention:${root.id}:terminal`, companyId, sourceKind: "blocker_attention", + sourceId: root.id, rootId: root.id, issueId: root.id, variant: "terminal", activityAt: old().toISOString(), deadlineAt: Date.now(), dueAt: Date.now() }); + expect(await state("blocker_attention", root.id)).toBeNull(); + expect(await db.select().from(decisionArchiveNotificationOutbox)).toHaveLength(0); + } else { + expect(entry).toBeTruthy(); await service.check(entry!); + expect((await state("blocker_attention", root.id))?.archivedAt).toBeInstanceOf(Date); + expect(await db.select().from(decisionArchiveNotificationOutbox)).toHaveLength(1); + } + expect(batch.fetchedRows).toBeLessThanOrEqual(4); + expect(batch.deferredRootIds.size).toBe(0); + expect(Boolean(batch.attention.get(root.id)?.blockingTreeLive)).toBe(kind !== "completed"); + expect(parameterCounts.length).toBeLessThanOrEqual(100); + }); + + it("does not archive an unknown truncated blocker tree during its fresh check", async () => { + const root = await insertIssue("blocked"); + await db.execute(sql`with blockers as ( + insert into issues (company_id, title, status, updated_at) + select ${companyId}::uuid, 'unresolved-' || n, 'backlog', ${old().toISOString()}::timestamptz + from generate_series(1, 2100) n returning id) + insert into issue_relations (company_id, issue_id, related_issue_id, type) + select ${companyId}::uuid, id, ${root.id}::uuid, 'blocks' from blockers`); + const service = attentionRetentionService(db), page = await service.load({ companyId, sourceKind: "blocker_attention" }); + const entry = page.rows!.find(row => row.rootId === root.id && row.variant === "terminal")!; + expect(entry).toBeTruthy(); expect(await service.check(entry)).toBeNull(); + expect(await db.select().from(decisionRetention)).toHaveLength(0); + expect(await db.select().from(decisionArchiveNotificationOutbox)).toHaveLength(0); + await start(); await quiet(); + }); + + function simulateRecorderLogStorage() { + // The recorder lifecycle and SQL are real; only its log-storage boundary is simulated. + vi.spyOn(workspaceOperationLogs, "getWorkspaceOperationLogStore").mockReturnValue({ + begin: async () => ({ store: "local_file", logRef: "retention-finalize-proof" }), + append: async () => {}, finalize: async () => ({ bytes: 0, compressed: false }), read: async () => ({ content: "" }), + }); + } + + it.each(["issue", "workspace"] as const)("publishes retention intent from ordinary no-run %s finalization", async scope => { + simulateRecorderLogStorage(); + const source = await insertIssue(), workspaceId = randomUUID(); + await db.execute(sql`insert into execution_workspaces (id, company_id, name, mode, strategy_type) + values (${workspaceId}::uuid, ${companyId}::uuid, 'Finalize producer', 'shared', 'project_primary')`); + await start(); await quiet(); + const committedHints: string[] = []; + const unsubscribe = subscribeDatabaseWork(db, DELIVERY_QUEUES.attentionRetention, (event, hints) => { + if (event === "settled") committedHints.push(...hints ?? []); + }); + try { + const result = await workspaceOperationService(db).createRecorder({ companyId, + executionWorkspaceId: scope === "workspace" ? workspaceId : undefined, + issueId: scope === "issue" ? source.id : undefined, + }).recordOperation({ phase: "workspace_finalize", run: async () => ({ status: "succeeded" }) }); + expect(result.status).toBe("succeeded"); + expect(committedHints.length).toBeGreaterThan(0); + expect(committedHints.map(hint => JSON.parse(hint).companyId).every(id => id === companyId)).toBe(true); + await quiet(); + if (scope === "workspace") { + committedHints.length = 0; + await workspaceOperationService(db).createRecorder({ companyId }) + .recordOperation({ phase: "workspace_finalize", run: async () => ({ status: "succeeded" }) }); + expect(committedHints).toEqual([]); await quiet(); + } + } finally { unsubscribe(); } + }); + + describe("no-run finalization with an incomplete blocker graph", () => { + let rootIds: string[], workspaceId: string, subjectId: string; + let loadedScopes: Array["load"]>[0]>; + beforeEach(async () => { + const began = performance.now(); + simulateRecorderLogStorage(); + const roots = await Promise.all([insertIssue("blocked"), insertIssue("blocked")]); + rootIds = roots.map(root => root.id); workspaceId = randomUUID(); + await db.execute(sql`insert into execution_workspaces (id, company_id, name, mode, strategy_type) + values (${workspaceId}::uuid, ${companyId}::uuid, 'Pending shared export', 'shared', 'project_primary')`); + await workspaceOperationService(db).createRecorder({ companyId, executionWorkspaceId: workspaceId }) + .recordOperation({ phase: "workspace_provision", run: async () => ({ status: "succeeded" }) }); + await db.execute(sql`with history as ( + insert into issues (company_id, title, status, updated_at, execution_workspace_id) + select ${companyId}::uuid, 'waiting-for-export-' || n, 'done', ${old().toISOString()}::timestamptz, ${workspaceId}::uuid + from generate_series(1, 2100) n returning id) + insert into issue_relations (company_id, issue_id, related_issue_id, type) + select ${companyId}::uuid, history.id, root.id, 'blocks' from history + cross join (values (${rootIds[0]}::uuid), (${rootIds[1]}::uuid)) root(id)`); + const [subject] = await db.select({ id: issues.id }).from(issues) + .where(and(eq(issues.companyId, companyId), eq(issues.status, "done"))).limit(1); + subjectId = subject!.id; + const review = await insertIssue("in_review", { assigneeUserId: "board" }); + await decisionRetentionService(db).setKeep({ ...mutation("review", review.id), keep: true }); + const service = attentionRetentionService(db), load = service.load.bind(service); loadedScopes = []; + vi.spyOn(service, "load").mockImplementation((scope, signal) => { loadedScopes.push(scope); return load(scope, signal); }); + process.stdout.write(JSON.stringify({ incompleteFinalizePhase: "seeded", setupMs: performance.now() - began }) + "\n"); + await start(service); await quiet(); + for (const rootId of rootIds) expect(await state("blocker_attention", rootId)).toBeNull(); + process.stdout.write(JSON.stringify({ incompleteFinalizeFixtureRows: 2100, incompleteRoots: 2, setupMs: performance.now() - began }) + "\n"); + }, 30_000); + it("archives only after the ordinary workspace-only finalize commits, then settles quiet", async () => { + const finish = (database: Db) => workspaceOperationService(database).createRecorder({ companyId, executionWorkspaceId: workspaceId, issueId: subjectId }) + .recordOperation({ phase: "workspace_finalize", run: async () => ({ status: "succeeded" }) }); + await expect(db.transaction(async tx => { + await finish(tx as unknown as Db); throw new Error("rollback finalization"); + })).rejects.toThrow("rollback finalization"); + await settle(); for (const rootId of rootIds) expect(await state("blocker_attention", rootId)).toBeNull(); + loadedScopes.length = 0; + expect((await finish(db)).status).toBe("succeeded"); + for (const rootId of rootIds) await archived("blocker_attention", rootId); + await quiet(); + expect(loadedScopes.length).toBeGreaterThan(0); + expect(loadedScopes.every(scope => scope?.companyId === companyId && scope.sourceKind === "blocker_attention")).toBe(true); + expect(await db.select().from(decisionArchiveNotificationOutbox)).toHaveLength(2); + }); + }); + + it("disarms finite deferred-root checks when large retained graphs have no archive obligation", async () => { + await db.execute(sql`insert into issues (id, company_id, title, status, updated_at) + select gen_random_uuid(), ${companyId}::uuid, 'held-root-' || n, 'blocked', ${old().toISOString()}::timestamptz + from generate_series(1, 4) n`); + await db.execute(sql`insert into issues (id, company_id, parent_id, title, status, assignee_agent_id, updated_at) + select gen_random_uuid(), ${companyId}::uuid, root.id, 'held-leaf-' || n, 'backlog', ${agentId}::uuid, ${old().toISOString()}::timestamptz + from issues root cross join generate_series(1, 600) n + where root.company_id = ${companyId}::uuid and root.status = 'blocked'`); + await db.execute(sql`insert into decision_retention (company_id, source_kind, source_id, source_activity_at, keep) + select company_id, 'blocker_attention', id::text, updated_at, true from issues + where company_id = ${companyId}::uuid and status = 'backlog'`); + const service = attentionRetentionService(db), check = vi.spyOn(service, "check"); + await start(service); await quiet(); + expect(check).toHaveBeenCalledTimes(4); + expect(await db.select().from(decisionArchiveNotificationOutbox)).toHaveLength(0); + }); + + it("bounds high-fanin ordinary issue writes to one coalescible hint", async () => { + const service = attentionRetentionService(db), original = service.load, load = vi.spyOn(service, "load"); + await start(service); await settle(); load.mockClear(); + const leaf = await insertIssue("in_review", { updatedAt: new Date(), assigneeUserId: "board" }); + await db.execute(sql`with parents as ( + insert into issues (id, company_id, title, status) + select gen_random_uuid(), ${companyId}::uuid, 'fan-in-' || n, 'todo' + from generate_series(1, 2100) n returning id) + insert into issue_relations (company_id, issue_id, related_issue_id, type) + select ${companyId}::uuid, ${leaf.id}::uuid, id, 'blocks' from parents`); + const hints: string[] = []; + const unsubscribe = subscribeDatabaseWork(db, DELIVERY_QUEUES.attentionRetention, (event, values) => { + if (event === "settled") hints.push(...values ?? []); + }); + let release!: () => void, entered = false; + const gate = new Promise(resolve => { release = resolve; }); + load.mockImplementationOnce(async (...args) => { const page = await original(...args); entered = true; await gate; return page; }); + statements.length = 0; + try { + await issueService(db).update(leaf.id, { title: "High fan-in ordinary edit 0" }); + await vi.waitFor(() => expect(entered).toBe(true), { timeout: 15_000 }); + for (let n = 1; n < 8; n++) await issueService(db).update(leaf.id, { title: `High fan-in ordinary edit ${n}` }); + } finally { release(); unsubscribe(); } + expect(statements.filter(query => query.includes("with recursive affected"))).toHaveLength(8); + expect(hints).toHaveLength(8); + expect(new Set(hints)).toEqual(new Set([JSON.stringify({ companyId })])); + await settle(); + expect(load.mock.calls.length).toBeLessThanOrEqual(3); + expect(load.mock.calls.every(([scope]) => scope?.companyId === companyId && !scope.issueId && !scope.issueIds)).toBe(true); + let query!: Parameters[0], returned = 0; + await db.transaction(async tx => { + const execute = tx.execute.bind(tx); + vi.spyOn(tx, "execute").mockImplementation(async statement => { + const result = await execute(statement); + if (new PgDialect().sqlToQuery(statement as SQL).sql.includes("with recursive affected")) { + query = statement; returned = result.length; + } + return result; + }); + await notifyIssueAttentionRetentionWork(tx, { companyId, issueId: leaf.id }); + }); + expect(returned).toBe(101); + const plans = await db.execute(sql`explain (analyze, format json) ${query as SQL}`); + const plan = (plans[0]["QUERY PLAN"] as any)[0].Plan; + expect(plan["Node Type"]).toBe("Limit"); + expect(plan["Actual Rows"]).toBe(101); + const nodes: any[] = [], collect = (node: any) => { nodes.push(node); for (const child of node.Plans ?? []) collect(child); }; + collect(plan); + const recursive = nodes.filter(node => node["Node Type"] === "Recursive Union"); + const relations = nodes.filter(node => node["Relation Name"] === "issue_relations"); + expect(recursive).toHaveLength(1); + expect(recursive[0]["Actual Rows"]).toBe(101); + expect(relations.length).toBeGreaterThan(0); + expect(relations.every(node => node["Actual Rows"] * node["Actual Loops"] <= 101)).toBe(true); + process.stdout.write(JSON.stringify({ reverseFanInPlan: plan }) + "\n"); + }); + + it("retains far ancestor deadlines when a cyclic chain exceeds the precise hint budget", async () => { + const service = attentionRetentionService(db), load = vi.spyOn(service, "load"); + await start(service); await settle(); load.mockClear(); + await db.execute(sql`insert into issues (id, company_id, title, status, updated_at, assignee_user_id) + select gen_random_uuid(), ${companyId}::uuid, 'chain-' || n, + case when n = 150 then 'in_review' else 'todo' end, + case when n = 150 then ${new Date(Date.now() - delay + 15_000).toISOString()}::timestamptz else now() end, + case when n = 150 then 'board' else null end + from generate_series(0, 150) n`); + await db.execute(sql`update issues child set parent_id = parent.id from issues parent + where child.company_id = ${companyId}::uuid and parent.company_id = child.company_id + and parent.title = 'chain-' || ((substring(child.title from 7)::integer + 1) % 151)`); + const [leaf, far] = await Promise.all([db.select().from(issues).where(and(eq(issues.companyId, companyId), eq(issues.title, "chain-0"))).then(rows => rows[0]), + db.select().from(issues).where(and(eq(issues.companyId, companyId), eq(issues.title, "chain-150"))).then(rows => rows[0])]); + await issueService(db).update(leaf.id, { title: "Ordinary deep chain edit" }); + await settle(); + expect(load.mock.calls.some(([scope]) => scope?.companyId === companyId && !scope.issueId && !scope.issueIds)).toBe(true); + expect(await state("review", far.id)).toBeNull(); + expect(coordinator!.nextWakeAt()).not.toBeNull(); + await archived("review", far.id); + await quiet(); + }); + + it("canonicalizes a complete cyclic affected set into one bounded batch hint", async () => { + const a = await insertIssue(), b = await insertIssue(), c = await insertIssue(); + await db.update(issues).set({ parentId: b.id }).where(eq(issues.id, a.id)); + await db.update(issues).set({ parentId: c.id }).where(eq(issues.id, b.id)); + await db.update(issues).set({ parentId: a.id }).where(eq(issues.id, c.id)); + const hints: string[] = []; + const unsubscribe = subscribeDatabaseWork(db, DELIVERY_QUEUES.attentionRetention, (event, values) => { + if (event === "settled") hints.push(...values ?? []); + }); + try { + await db.transaction(tx => notifyIssueAttentionRetentionWork(tx, { companyId, issueId: a.id })); + await db.transaction(tx => notifyIssueAttentionRetentionWork(tx, { companyId, issueId: c.id })); + const wide = { companyId, issueIds: [b.id, a.id, c.id, a.id], ignored: {} as unknown }; + wide.ignored = wide; + await db.transaction(tx => notifyAttentionRetentionWork(tx, wide)); + } finally { unsubscribe(); } + expect(hints).toHaveLength(3); + expect(new Set(hints)).toEqual(new Set([JSON.stringify({ companyId, issueIds: [a.id, b.id, c.id].sort() })])); + }); + + it("canonicalizes wide circular notification variables to the supported identity fields", async () => { + const sourceId = randomUUID(), issueId = randomUUID(), hints: string[] = []; + const wide = { companyId, sourceKind: "review" as const, sourceId, issueId, ignored: {} as unknown }; + wide.ignored = wide; + const unsubscribe = subscribeDatabaseWork(db, DELIVERY_QUEUES.attentionRetention, (event, values) => { + if (event === "settled") hints.push(...values ?? []); + }); + try { await db.transaction(tx => notifyAttentionRetentionWork(tx, wide)); } finally { unsubscribe(); } + expect(hints).toEqual([JSON.stringify({ companyId, sourceKind: "review", sourceId, issueId })]); + }); + + it("keeps held, burst and sustained ordinary issue writes scoped while an unrelated deadline fires", async () => { + const row = await issueService(db).create(companyId, { title: "Active review", status: "in_review", assigneeUserId: "board", createdByAgentId: agentId }); + const urgent = await insertIssue("in_review", { assigneeUserId: "board", updatedAt: new Date(Date.now() - delay + 10_000) }); + const otherCompany = randomUUID(); + await db.insert(companies).values({ id: otherCompany, name: "Unrelated", issuePrefix: "BURST" }); + const foreign = await db.insert(issues).values(Array.from({ length: 125 }, () => ({ companyId: otherCompany, + title: randomUUID(), status: "in_review", assigneeUserId: "board", updatedAt: old() }))).returning(); + await db.insert(decisionRetention).values(foreign.map(issue => ({ companyId: otherCompany, sourceKind: "review", sourceId: issue.id, sourceActivityAt: issue.updatedAt, keep: true }))); + const service = attentionRetentionService(db), original = service.load, load = vi.spyOn(service, "load"); + await start(service); await settle(); load.mockClear(); + let release!: () => void, entered = false; + const gate = new Promise(resolve => { release = resolve; }); + load.mockImplementationOnce(async (...args) => { const page = await original(...args); entered = true; await gate; return page; }); + statements.length = 0; + await issueService(db).update(row.id, { title: "Held mutation" }); + await vi.waitFor(() => expect(entered).toBe(true), { timeout: 15_000 }); + await Promise.all(Array.from({ length: 5 }, (_, index) => issueService(db).update(row.id, { title: `Burst ${index}` }))); + release(); await settle(); + for (let index = 0; index < 8; index++) { + await issueService(db).update(row.id, { title: `Steady ${index}` }); await vi.advanceTimersByTimeAsync(1000); await settle(); + } + await archived("review", urgent.id); + const routine = load.mock.calls.map(([scope]) => scope).filter(scope => !scope?.sourceId); + expect(routine.length).toBeGreaterThan(0); + expect(routine.every(scope => scope?.companyId === companyId && scope.issueId === row.id)).toBe(true); + const scans = statements.filter(query => query.includes("with source_candidates")); + expect(scans.length).toBeGreaterThan(0); + expect(scans.every(query => !query.includes("budget_incidents") && !query.includes("agent_error_alert"))).toBe(true); + expect(await state("review", row.id)).toBeNull(); + }); + + it("wakes all linked issue sources when an ordinary visibility mutation reveals their origin", async () => { + const origin = await insertIssue("in_review", { hiddenAt: old(), assigneeUserId: "board", responsibleUserId: "board" }); + const [run] = await db.insert(heartbeatRuns).values({ companyId, agentId, status: "failed", contextSnapshot: { issueId: origin.id }, createdAt: old(), finishedAt: old() }).returning(); + await db.insert(heartbeatRunEvents).values({ companyId, agentId, runId: run.id, seq: 1, eventType: "lifecycle", message: "Bounded retry exhausted: visibility" }); + const [decision] = await db.insert(decisions).values({ companyId, originAgentId: agentId, originIssueId: origin.id, originRunId: run.id, + title: "Decision", body: "Choose", options: [], signedSpec: "fixture", targetSnapshots: {}, expiresAt: new Date(Date.now() + 86_400_000), updatedAt: old() }).returning(); + const [recovery] = await db.insert(issueRecoveryActions).values({ companyId, sourceIssueId: origin.id, kind: "execution", ownerType: "board", cause: "fixture", fingerprint: "fixture", nextAction: "Choose", updatedAt: old() }).returning(); + const [interaction] = await db.insert(issueThreadInteractions).values({ companyId, issueId: origin.id, kind: "ask_user_questions", payload: { questions: [] }, updatedAt: old() }).returning(); + await start(); await quiet(); + await issueService(db).update(origin.id, { hiddenAt: null }); + for (const [kind, id] of [["decision", decision.id], ["recovery_action", recovery.id], ["failed_run", run.id], ["issue_thread_interaction", interaction.id]]) await archived(kind, id); + await settle(); expect(await state("review", origin.id)).toBeNull(); expect(coordinator!.nextWakeAt()).not.toBeNull(); + }); + + it("invalidates old and new parents, terminal visibility and repointed blocker relations", async () => { + const oldParent = await insertIssue("blocked", { updatedAt: new Date(), responsibleUserId: "board" }); + const newParent = await insertIssue("blocked", { updatedAt: new Date(), responsibleUserId: "board" }); + const terminal = await insertIssue("backlog", { parentId: oldParent.id, hiddenAt: old(), assigneeAgentId: agentId, responsibleUserId: "board" }); + const service = attentionRetentionService(db), load = vi.spyOn(service, "load"); await start(service); await settle(); load.mockClear(); + await issueService(db).update(terminal.id, { parentId: newParent.id }); await settle(); + const ids = load.mock.calls.flatMap(([scope]) => scope?.issueIds ?? (scope?.issueId ? [scope.issueId] : [])); + expect(ids).toEqual(expect.arrayContaining([oldParent.id, newParent.id, terminal.id])); + load.mockClear(); + await issueService(db).update(terminal.id, { hiddenAt: null }); await settle(); + expect(await state("blocker_attention", terminal.id)).toBeNull(); + const visible = await service.load({ companyId, issueId: newParent.id }); + expect(visible.rows!.some(entry => entry.sourceId === terminal.id && entry.dueAt > Date.now())).toBe(true); + expect(load.mock.calls.some(([scope]) => scope?.issueId === newParent.id || scope?.issueIds?.includes(newParent.id))).toBe(true); + const next = await insertIssue("backlog", { assigneeAgentId: agentId }); + await issueService(db).update(oldParent.id, { blockedByIssueIds: [next.id] }); await archived("blocker_attention", next.id); + }); + + it("cancels and reactivates a blocker deadline through ordinary liveness incident creation and hiding", async () => { + const root = await insertIssue("blocked"); + const terminal = await insertIssue("backlog", { assigneeAgentId: agentId, updatedAt: new Date(Date.now() - delay + 10_000) }); + await db.insert(issueRelations).values({ companyId, issueId: terminal.id, relatedIssueId: root.id, type: "blocks" }); + await start(); await settle(); expect(coordinator!.nextWakeAt()).not.toBeNull(); + const incident = await issueService(db).create(companyId, { title: "Resolve liveness", status: "todo", assigneeUserId: "board", + createdByUserId: "board", originKind: RECOVERY_ORIGIN_KINDS.issueGraphLivenessEscalation, + originId: buildIssueGraphLivenessIncidentKey({ companyId, issueId: root.id, state: "needs_attention", blockerIssueId: terminal.id }) }); + await quiet(); expect(await state("blocker_attention", terminal.id)).toBeNull(); + await issueService(db).update(incident.id, { hiddenAt: new Date() }); + await archived("blocker_attention", terminal.id); await quiet(); + }); + + it("resolves opaque run issue references through trusted company issue rows", async () => { + const issue = await insertIssue(); + const [valid, opaque] = await db.insert(heartbeatRuns).values([ + { companyId, agentId, status: "succeeded", contextSnapshot: { issueId: issue.id } }, + { companyId, agentId, status: "succeeded", contextSnapshot: { issueId: "none" } }, + ]).returning(); + const hints: string[] = []; + const unsubscribe = subscribeDatabaseWork(db, DELIVERY_QUEUES.attentionRetention, (event, values) => { + if (event === "settled") hints.push(...values ?? []); + }); + try { + await db.transaction(tx => notifyIssueAttentionRetentionWork(tx, { companyId, runId: valid.id })); + await db.transaction(tx => notifyIssueAttentionRetentionWork(tx, { companyId, runId: opaque.id })); + } finally { unsubscribe(); } + expect(hints).toEqual([JSON.stringify({ companyId, issueId: issue.id }), JSON.stringify({ companyId })]); + }); + + it("accepts ordinary liveness incident creation and completion with a none leaf sentinel", async () => { + const source = await insertIssue("blocked", { responsibleUserId: "board" }); + const service = issueService(db); + const incident = await service.create(companyId, { title: "Invalid participant", status: "todo", assigneeUserId: "board", + createdByUserId: "board", originKind: RECOVERY_ORIGIN_KINDS.issueGraphLivenessEscalation, + originId: `harness_liveness:${companyId}:${source.id}:invalid_review_participant:none` }); + await service.update(source.id, { blockedByIssueIds: [incident.id] }); + await service.update(incident.id, { status: "done" }); + expect((await service.getById(incident.id))?.status).toBe("done"); + expect((await service.getRelationSummaries(source.id)).blockedBy).toEqual([]); + }); + + it.each(["hidden", "done"] as const)("reactivates an older review when moved stranded recovery is %s", async transition => { + const review = await insertIssue("in_review", { assigneeAgentId: agentId, responsibleUserId: "board" }); + const parent = await insertIssue("todo", { responsibleUserId: "board" }); + const recovery = await insertIssue("todo", { parentId: review.id, originKind: RECOVERY_ORIGIN_KINDS.strandedIssueRecovery, + originId: review.id, assigneeUserId: "board", responsibleUserId: "board" }); + await issueService(db).update(recovery.id, { parentId: parent.id }); + expect((await issueService(db).listReviewAttention(companyId, [review])).get(review.id)?.state).not.toBe("stalled"); + await start(); await quiet(); + await issueService(db).update(recovery.id, transition === "hidden" ? { hiddenAt: new Date() } : { status: "done" }); + await settle(); + expect((await state("review", review.id))?.archivedAt).toBeInstanceOf(Date); + expect(await db.select().from(decisionArchiveNotificationOutbox).where(eq(decisionArchiveNotificationOutbox.sourceId, review.id))).toHaveLength(1); + }); + + it("scopes ordinary execution-control writes to their company", async () => { + const otherCompany = randomUUID(); + await db.insert(companies).values({ id: otherCompany, name: "Other", issuePrefix: "OTHER" }); + await db.insert(issues).values(Array.from({ length: 125 }, () => ({ companyId: otherCompany, + title: randomUUID(), status: "in_review", assigneeUserId: "board", updatedAt: old() }))); + const [run] = await db.insert(heartbeatRuns).values({ companyId, agentId, status: "running" }).returning(); + const service = attentionRetentionService(db), load = vi.spyOn(service, "load"); + await start(service); await quiet(); load.mockClear(); + for (let count = 0; count < 3; count++) { + await writeExecutionControl(db, DELIVERY_QUEUES.executionFinalization, + tx => tx.update(heartbeatRuns).set({ updatedAt: new Date() }).where(eq(heartbeatRuns.id, run.id)), { companyId }); + await quiet(); + } + expect(load).toHaveBeenCalled(); + expect(load.mock.calls.every(([scope]) => scope?.companyId === companyId)).toBe(true); + }); + + it("registers and cancels budget deadlines through ordinary policy and incident producers", async () => { + const service = budgetService(db); + await service.upsertPolicy(companyId, { scopeType: "agent", scopeId: agentId, amount: 100, + warnPercent: 85, hardStopEnabled: false, notifyEnabled: true }, "board"); + const retention = attentionRetentionService(db), load = vi.spyOn(retention, "load"); + await start(retention); await quiet(); load.mockClear(); + const [event] = await db.insert(costEvents).values({ companyId, agentId, provider: "openai", biller: "openai", + billingType: "metered_api", model: "fixture", inputTokens: 100, outputTokens: 20, + costCents: 90, occurredAt: new Date() }).returning(); + await service.evaluateCostEvent(event); + await settle(); expect(coordinator!.nextWakeAt()).not.toBeNull(); + const [incident] = await db.select().from(budgetIncidents).where(eq(budgetIncidents.companyId, companyId)); + expect(incident).toMatchObject({ status: "open", thresholdType: "soft" }); + expect(await state("budget_alert", incident.id)).toBeNull(); + await service.upsertPolicy(companyId, { scopeType: "agent", scopeId: agentId, amount: 200 }, "board"); + await quiet(); + expect((await db.select().from(budgetIncidents).where(eq(budgetIncidents.id, incident.id)))[0].status).toBe("resolved"); + expect(load.mock.calls.every(([scope]) => scope?.companyId === companyId)).toBe(true); + }); + + it("matches the existing feed across all eight remaining source kinds and exclusion variants", async () => { + const origin = await insertIssue(); + const review = await insertIssue("in_review", { assigneeUserId: "board" }); + const blocker = await insertIssue("blocked", { unblockDescriptor: { owner: "board", action: "Choose a dependency" }, blockedTransitionAt: old() }); + await insertIssue("in_review", { hiddenAt: old(), assigneeUserId: "board" }); + const hidden = await insertIssue("todo", { hiddenAt: old() }); + const [run] = await db.insert(heartbeatRuns).values({ companyId, agentId, status: "failed", contextSnapshot: { issueId: origin.id }, createdAt: old(), updatedAt: old(), finishedAt: old() }).returning(); + await db.insert(heartbeatRunEvents).values({ companyId, agentId, runId: run.id, seq: 1, eventType: "lifecycle", message: "Bounded retry exhausted: fixture" }); + await db.insert(decisions).values({ companyId, originAgentId: agentId, originIssueId: origin.id, originRunId: run.id, + title: "Decision", body: "Choose", options: [], signedSpec: "fixture", targetSnapshots: {}, expiresAt: new Date(Date.now() + 86_400_000), updatedAt: old() }); + await db.insert(issueThreadInteractions).values([ + { companyId, issueId: origin.id, kind: "ask_user_questions", payload: { questions: [] }, updatedAt: old() }, + { companyId, issueId: origin.id, kind: "ask_user_questions", payload: { questions: [] }, addresseeAgentId: agentId, updatedAt: old() }, + { companyId, issueId: origin.id, kind: "ask_user_questions", payload: { questions: [] }, addresseeUserId: "board", updatedAt: old() }, + { companyId, issueId: hidden.id, kind: "ask_user_questions", payload: { questions: [] }, updatedAt: old() }, + ]); + await db.insert(issueRecoveryActions).values({ companyId, sourceIssueId: origin.id, kind: "execution", ownerType: "board", cause: "fixture", fingerprint: "fixture", nextAction: "Choose", updatedAt: old() }); + const [policy] = await db.insert(budgetPolicies).values({ companyId, scopeType: "company", scopeId: companyId, windowKind: "calendar_month_utc", amount: 100 }).returning(); + await db.insert(budgetIncidents).values({ companyId, policyId: policy.id, scopeType: "company", scopeId: companyId, + metric: "billed_cents", windowKind: "calendar_month_utc", windowStart: old(), windowEnd: new Date(), thresholdType: "hard", amountLimit: 100, amountObserved: 100, updatedAt: old() }); + await db.insert(agents).values({ companyId, name: "Error", role: "engineer", status: "error", updatedAt: old() }); + const feed = await attentionService(db).list(companyId, { all: true, allowUnscopedAll: true, includeDismissed: true, excludeSources: ["approval", "join_request"] }); + const expected = [...new Set(feed.items.map(item => `${item.sourceKind}:${item.subject.id}`))].sort(); + const actual = [...new Set((await loadAll()).map(item => `${item.sourceKind}:${item.sourceId}`))].sort(); + expect(new Set(feed.items.map(item => item.sourceKind)).size).toBe(8); expect(actual).toEqual(expected); + expect(actual).toContain(`review:${review.id}`); expect(actual).toContain(`blocker_attention:${blocker.id}`); + await start(); for (const item of feed.items) await archived(item.sourceKind, item.subject.id); await quiet(); + }); + + it("keeps unseen sources, cancels their deadline, and reactivates when unkept or revived", async () => { + const row = await insertIssue("in_review", { assigneeUserId: "board" }); + await decisionRetentionService(db).setKeep({ ...mutation("review", row.id), keep: true }); + await start(); await quiet(); expect((await state("review", row.id))?.archivedAt).toBeNull(); + await decisionRetentionService(db).setKeep({ ...mutation("review", row.id), keep: false }); + await archived("review", row.id); await quiet(); + await decisionRetentionService(db).revive(mutation("review", row.id)); await archived("review", row.id); await quiet(); + }); + + it("moves deadlines on source activity and cancels when the source leaves attention", async () => { + const row = await insertIssue("in_review", { assigneeUserId: "board", updatedAt: new Date() }); + await start(); await settle(); const first = coordinator!.nextWakeAt()!; + await change(tx => tx.update(issues).set({ updatedAt: new Date(Date.now() - delay + 600_000) }).where(eq(issues.id, row.id))); + await settle(); expect(coordinator!.nextWakeAt()).toBeLessThan(first); + await issueService(db).update(row.id, { status: "todo" }); await quiet(); expect(await state("review", row.id)).toBeNull(); + }); + + it("preserves outer transaction ownership and first writes during drain", async () => { + await start(); await quiet(); draining = true; + let id: string | undefined; + await db.transaction(async outer => { + await outer.transaction(async nested => { + await notifyAttentionRetentionWork(nested, { companyId }); + id = (await insertIssue("in_review", { assigneeUserId: "board" }, nested as unknown as Db)).id; + }); + await vi.advanceTimersByTimeAsync(60_000); expect(await state("review", id!)).toBeNull(); + }); + await vi.advanceTimersByTimeAsync(120_000); expect(await state("review", id!)).toBeNull(); + draining = false; await vi.advanceTimersByTimeAsync(30_000); await archived("review", id!); await quiet(); + await expect(db.transaction(async outer => { + await outer.transaction(async nested => { await notifyAttentionRetentionWork(nested, { companyId }); await insertIssue("in_review", { assigneeUserId: "board" }, nested as unknown as Db); }); + throw new Error("rollback"); + })).rejects.toThrow("rollback"); + await quiet(); expect(await db.select().from(decisionRetention)).toHaveLength(1); + }); + + it("isolates companies, catches up at activation, and bounds retained-history pages", async () => { + const other = randomUUID(); await db.insert(companies).values({ id: other, name: "Paused", issuePrefix: "RP", status: "paused" }); + const row = await insertIssue("in_review", { companyId: other, assigneeUserId: "board" }); + await db.insert(issues).values(Array.from({ length: 1000 }, () => ({ companyId, title: randomUUID(), status: "done", updatedAt: old() }))); + const service = attentionRetentionService(db); const page = await service.load({ companyId }); + expect(page).toEqual({ rows: [], complete: true }); + await start(service); await quiet(); expect(await db.select().from(decisionRetention)).toHaveLength(0); + await companyService(db).update(other, { status: "active" }); + await vi.waitFor(async () => { await vi.advanceTimersByTimeAsync(1000); expect(await db.select().from(decisionRetention)).toHaveLength(1); }); + expect((await db.select().from(decisionRetention))[0]).toMatchObject({ companyId: other, sourceId: row.id }); await quiet(); + }); + it("preserves terminal blocker source identity and its own activity deadline", async () => { + const root = await insertIssue("blocked", { unblockDescriptor: { owner: "board", action: "Pick an owner" }, blockedTransitionAt: old() }); + const terminal = await insertIssue("backlog", { assigneeAgentId: agentId, updatedAt: new Date(old().getTime() + 60_000) }); + await db.insert(issueRelations).values({ companyId, issueId: terminal.id, relatedIssueId: root.id, type: "blocks" }); + const feed = await attentionService(db).list(companyId, { all: true, allowUnscopedAll: true }); + const entries = await loadAll(); + const expected = feed.items.filter(item => item.sourceKind === "blocker_attention") + .map(item => `${item.subject.id}:${new Date(item.activityAt).toISOString()}`).sort(); + expect(entries.filter(item => item.sourceKind === "blocker_attention") + .map(item => `${item.sourceId}:${new Date(item.activityAt).toISOString()}`).sort()).toEqual(expected); + expect(expected).toContain(`${terminal.id}:${terminal.updatedAt.toISOString()}`); + await start(); await archived("blocker_attention", terminal.id); await archived("blocker_attention", root.id); await quiet(); + expect((await state("blocker_attention", terminal.id))?.sourceActivityAt).toEqual(terminal.updatedAt); + expect((await state("blocker_attention", root.id))?.sourceActivityAt).toEqual(root.blockedTransitionAt); + }); + + it("matches newest confirmations, independent gates, org invokability, and conversation visibility", async () => { + const origin = await insertIssue(); + const conversation = await insertIssue("in_review", { conversationAgentId: agentId, conversationUserId: "board", conversationState: "waiting", assigneeAgentId: agentId }); + const hidden = await insertIssue("todo", { hiddenAt: old() }); + const [paused, manager, unhealthy] = await db.insert(agents).values([ + { companyId, name: "Paused", role: "engineer", status: "paused" }, + { companyId, name: "Terminated", role: "manager", status: "terminated" }, + { companyId, name: "Unhealthy", role: "engineer", status: "active" }, + ]).returning(); + await db.update(agents).set({ reportsTo: manager.id }).where(eq(agents.id, unhealthy.id)); + const common = { companyId, issueId: origin.id, kind: "request_confirmation", payload: { prompt: "Approve?" }, createdAt: old(), updatedAt: old() }; + const [older, newest, independent] = await db.insert(issueThreadInteractions).values([ + common, { ...common, createdAt: new Date(old().getTime() + 1000) }, + { ...common, payload: { prompt: "Independent", qualityReview: { artifactKey: "copy" } } }, + { ...common, createdAt: new Date(), addresseeUserId: "board" }, + { ...common, addresseeAgentId: agentId }, + { ...common, issueId: hidden.id }, + { ...common, issueId: conversation.id, kind: "ask_user_questions", payload: { questions: [] } }, + { ...common, issueId: conversation.id, addresseeAgentId: paused.id }, + { ...common, issueId: conversation.id, addresseeAgentId: unhealthy.id, kind: "ask_user_questions", payload: { questions: [] } }, + ] as typeof issueThreadInteractions.$inferInsert[]).returning(); + const feed = await attentionService(db).list(companyId, { all: true, allowUnscopedAll: true }); + const expected = [...new Set(feed.items.map(item => `${item.sourceKind}:${item.subject.id}`))].sort(); + expect([...new Set((await loadAll()).map(item => `${item.sourceKind}:${item.sourceId}`))].sort()).toEqual(expected); + expect(expected).not.toContain(`issue_thread_interaction:${older.id}`); + expect(expected).toContain(`issue_thread_interaction:${newest.id}`); expect(expected).toContain(`issue_thread_interaction:${independent.id}`); + await start(); await archived("issue_thread_interaction", newest.id); await archived("issue_thread_interaction", independent.id); await quiet(); + await change(tx => tx.update(issueThreadInteractions).set({ status: "cancelled" }).where(eq(issueThreadInteractions.id, newest.id))); + await archived("issue_thread_interaction", older.id); await quiet(); + }); + + it("excludes maintained reviews until a pending approval creates human attention", async () => { + const row = await insertIssue("in_review", { assigneeAgentId: agentId }); + const [interaction] = await db.insert(issueThreadInteractions).values({ companyId, issueId: row.id, + kind: "request_confirmation", addresseeAgentId: agentId, payload: { prompt: "Agent verdict" } }).returning(); + expect((await loadAll()).some(entry => entry.sourceKind === "review")).toBe(false); + const [approval] = await db.insert(approvals).values({ companyId, type: "hire_agent", payload: {}, status: "pending" }).returning(); + await db.insert(issueApprovals).values({ companyId, issueId: row.id, approvalId: approval.id }); + const feed = await attentionService(db).list(companyId, { all: true, allowUnscopedAll: true }); + expect(feed.items.some(item => item.sourceKind === "review" && item.subject.id === row.id)).toBe(true); + await start(); await archived("review", row.id); await quiet(); + expect(await state("issue_thread_interaction", interaction.id)).toBeNull(); + }); + + it("reactivates an exhausted failed run through the ordinary durable event producer", async () => { + const origin = await insertIssue(); + const [run, newer] = await db.insert(heartbeatRuns).values([ + { companyId, agentId, status: "failed", contextSnapshot: { issueId: origin.id }, createdAt: old(), finishedAt: old() }, + { companyId, agentId, status: "succeeded", contextSnapshot: { issueId: origin.id }, createdAt: new Date(old().getTime() + 1000), finishedAt: old() }, + ]).returning(); + await start(); await quiet(); + await appendHeartbeatRunEvent(db, { companyId, agentId, runId: run.id, eventType: "lifecycle", message: "Bounded retry exhausted after 3 attempts" }); + await quiet(); expect(await state("failed_run", run.id)).toBeNull(); + await change(tx => tx.delete(heartbeatRuns).where(eq(heartbeatRuns.id, newer.id))); + await archived("failed_run", run.id); await quiet(); + }); + + it("cancels an agent error deadline through ordinary clearError", async () => { + await db.update(agents).set({ status: "error", updatedAt: new Date() }).where(eq(agents.id, agentId)); + await start(); await settle(); expect(coordinator!.nextWakeAt()).not.toBeNull(); + await createAgentLifecycle(db).clearError(agentId); await quiet(); + expect(await state("agent_error_alert", agentId)).toBeNull(); + }); + + it("drains a paged backlog with source-scoped self notifications and leaves completed history quiet", async () => { + await db.insert(issues).values(Array.from({ length: 225 }, () => ({ companyId, title: randomUUID(), status: "in_review", assigneeUserId: "board", updatedAt: old() }))); + const service = attentionRetentionService(db); + const load = vi.spyOn(service, "load"); + await start(service); + await vi.waitFor(async () => { await vi.advanceTimersByTimeAsync(1000); expect(await db.select().from(decisionRetention)).toHaveLength(225); }, { timeout: 30_000 }); + await quiet(); + const scopes = load.mock.calls.map(([scope]) => scope); + expect(scopes.filter(scope => !scope)).toHaveLength(3); + expect(scopes.filter(scope => scope && !scope.sourceId)).toHaveLength(0); + expect(scopes.filter(scope => scope?.sourceId).length).toBeLessThanOrEqual(225); + expect(new Set((await db.select().from(decisionRetention)).map(row => row.sourceId)).size).toBe(225); + }); + + it("retries a failed atomic archive across drain and preserves notifications during a scan", async () => { + const row = await insertIssue("in_review", { assigneeUserId: "board" }); + await db.execute(sql`create function attention_retention_fail() returns trigger language plpgsql as $$ begin raise exception 'retention injected'; end $$`); + await db.execute(sql`create trigger attention_retention_fail before insert on decision_retention for each row execute function attention_retention_fail()`); + await start(); expect(errors).toHaveLength(1); errors.length = 0; + expect(await state("review", row.id)).toBeNull(); expect(await db.select().from(decisionArchiveNotificationOutbox)).toHaveLength(0); + draining = true; await db.execute(sql`drop trigger attention_retention_fail on decision_retention`); + statements.length = 0; await vi.advanceTimersByTimeAsync(120_000); + expect(statements.length).toBeLessThanOrEqual(1); + expect(statements.every(query => query.includes("pg_xact_status"))).toBe(true); + statements.length = 0; await vi.advanceTimersByTimeAsync(300_000); expect(statements).toEqual([]); + draining = false; await vi.advanceTimersByTimeAsync(30_000); await archived("review", row.id); await quiet(); + await coordinator!.stop(); coordinator = undefined; + let release!: () => void; let scanned!: () => void; + const gate = new Promise(resolve => { release = resolve; }); + const entered = new Promise(resolve => { scanned = resolve; }); + const service = attentionRetentionService(db), original = service.load; + vi.spyOn(service, "load").mockImplementationOnce(async (...args) => { const result = await original(...args); scanned(); await gate; return result; }); + const starting = start(service); await entered; + const next = await change(async tx => insertIssue("in_review", { assigneeUserId: "board" }, tx)); + release(); await starting; await archived("review", next.id); await quiet(); + }); + + it("registers and cancels a deadline through ordinary interaction creation and withdrawal", async () => { + const origin = await insertIssue(); await start(); await quiet(); + const service = issueThreadInteractionService(db); + const interaction = await service.create(origin, { kind: "request_confirmation", payload: { version: 1, prompt: "Approve the result?" } }, { userId: "board" }); + await settle(); expect(coordinator!.nextWakeAt()).not.toBeNull(); expect(await state("issue_thread_interaction", interaction.id)).toBeNull(); + await service.withdrawInteraction(origin, interaction.id, { reason: "No longer needed" }, { userId: "board" }); await quiet(); + }); + +}); diff --git a/server/src/__tests__/attention-service.test.ts b/server/src/__tests__/attention-service.test.ts index a024ed421d7..5af78c2cb8b 100644 --- a/server/src/__tests__/attention-service.test.ts +++ b/server/src/__tests__/attention-service.test.ts @@ -62,7 +62,7 @@ describeEmbeddedPostgres("attention service", () => { beforeAll(async () => { if (!externalUrl) tempDb = await startEmbeddedPostgresTestDatabase("paperclip-attention-service-"); - db = createDb(externalUrl ?? tempDb!.connectionString); + db = createDb(externalUrl ?? tempDb!.connectionString, { maxConnections: 6 }); }, 30_000); afterEach(async () => { diff --git a/server/src/__tests__/blocker-workspace-history-performance.test.ts b/server/src/__tests__/blocker-workspace-history-performance.test.ts new file mode 100644 index 00000000000..c40a4b46f1e --- /dev/null +++ b/server/src/__tests__/blocker-workspace-history-performance.test.ts @@ -0,0 +1,103 @@ +import { randomUUID } from "node:crypto"; +import { sql } from "drizzle-orm"; +import { afterAll, afterEach, beforeAll, describe, expect, it } from "vitest"; +import { applyPendingMigrations, companies, createDb, issues } from "@paperclipai/db"; +import { issueService } from "../services/issues.js"; +import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js"; +import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; + +const external = resolveExternalTestDatabaseUrl("PAPERCLIP_BLOCKER_HISTORY_TEST_DATABASE_URL"); +const support = external ? { supported: true } : await getEmbeddedPostgresTestSupport(); +type Plan = { "Node Type": string; "Relation Name"?: string; "Actual Rows": number; "Actual Loops": number; + "Rows Removed by Filter"?: number; "Rows Removed by Index Recheck"?: number; Plans?: Plan[] }; + +describe.skipIf(!support.supported)("blocker workspace retained history cost", () => { + let temporary: Awaited> | undefined; + let db: ReturnType; + let capture = false; + const queries: Array<{ query: string; parameters: unknown[] }> = []; + beforeAll(async () => { + if (external) await applyPendingMigrations(external); + else temporary = await startEmbeddedPostgresTestDatabase("blocker-history-"); + db = createDb(external ?? temporary!.connectionString, { maxConnections: 6, idleTimeoutSeconds: 0, maxLifetimeSeconds: 0 }); + db.$client.options.onnotice = () => {}; + db.$client.options.debug = (_connection, query, parameters) => { + if (capture && query.includes("workspace_operations")) queries.push({ query, parameters: [...parameters] }); + }; + }, 30_000); + afterEach(async () => { capture = false; queries.length = 0; await db.execute(sql`truncate companies cascade`); }); + afterAll(async () => { await db?.$client.end({ timeout: 1 }); await temporary?.cleanup(); }); + + it.each(["finalized", "pending"] as const)("bounds operation-history work for 2100 blockers sharing 5000 %s operations", async kind => { + const companyId = randomUUID(), workspaceId = randomUUID(), unrelatedWorkspaceId = randomUUID(); + await db.insert(companies).values({ id: companyId, name: "Shared history", issuePrefix: `SH${companyId.slice(0, 6)}` }); + const [root] = await db.insert(issues).values({ companyId, title: "History dependent", status: "blocked" }).returning(); + await db.execute(sql`insert into execution_workspaces (id, company_id, name, mode, strategy_type) + values (${workspaceId}::uuid, ${companyId}::uuid, 'Shared operation history', 'shared', 'project_primary'), + (${unrelatedWorkspaceId}::uuid, ${companyId}::uuid, 'Unrelated history', 'shared', 'project_primary')`); + await db.execute(sql`insert into workspace_operations (company_id, execution_workspace_id, phase, status, started_at) + select ${companyId}::uuid, ${unrelatedWorkspaceId}::uuid, 'worktree_prepare', 'succeeded', + '2025-01-01T00:00:00Z'::timestamptz + n * interval '1 second' from generate_series(1, 50000) n`); + await db.execute(sql`insert into workspace_operations (company_id, execution_workspace_id, phase, status, started_at) + select ${companyId}::uuid, ${workspaceId}::uuid, + case when n = 5000 and ${kind} = 'finalized' then 'workspace_finalize' else 'worktree_prepare' end, + 'succeeded', '2026-01-01T00:00:00Z'::timestamptz + n * interval '1 second' from generate_series(1, 5000) n`); + await db.execute(sql`with history as ( + insert into issues (company_id, title, status, execution_workspace_id) + select ${companyId}::uuid, 'done blocker ' || n, 'done', ${workspaceId}::uuid + from generate_series(1, 2100) n returning id) + insert into issue_relations (company_id, issue_id, related_issue_id, type) + select ${companyId}::uuid, id, ${root.id}::uuid, 'blocks' from history`); + await db.execute(sql`analyze workspace_operations`); + await db.execute(sql`analyze issues`); + await db.execute(sql`analyze issue_relations`); + capture = true; + const batch = await issueService(db).listBlockerAttentionBatch(companyId, [root]); + capture = false; + expect(queries.length).toBeGreaterThan(0); + expect(Math.max(...queries.map(statement => statement.parameters.length))).toBeLessThanOrEqual(1100); + let operationRowsExamined = 0; + const measurements: unknown[] = []; + for (const statement of queries) { + const result = await db.$client.unsafe("EXPLAIN (ANALYZE, BUFFERS, FORMAT JSON) " + statement.query, + statement.parameters as never[]); + const explain = (result[0]!["QUERY PLAN"] as Array<{ Plan: Plan }>)[0]!; + const visit = (node: Plan) => { + if (node["Relation Name"] === "workspace_operations") operationRowsExamined += node["Actual Loops"] + * (node["Actual Rows"] + (node["Rows Removed by Filter"] ?? 0) + (node["Rows Removed by Index Recheck"] ?? 0)); + node.Plans?.forEach(visit); + }; + visit(explain.Plan); + measurements.push({ ...statement, explain }); + } + process.stdout.write(JSON.stringify({ sharedWorkspaceHistory: { kind, blockers: 2100, retainedOperations: 5000, unrelatedOperations: 50000, + operationRowsExamined, queryCount: queries.length, measurements } }) + "\n"); + // The finite graph/pair pages may each read the 5,000 candidate rows once. + // Neither 2,100 blockers nor 50,000 unrelated operations may multiply that work. + expect(operationRowsExamined).toBeLessThanOrEqual(55_000); + expect(batch.deferredRootIds.has(root.id)).toBe(kind === "pending"); + if (kind === "finalized") expect(batch.nodeCount).toBe(1); + }, 120_000); + it("preserves attributed precedence and the strict later-finalize boundary", async () => { + const companyId = randomUUID(), workspaceId = randomUUID(); + await db.insert(companies).values({ id: companyId, name: "Finalize policy", issuePrefix: `FP${companyId.slice(0, 6)}` }); + await db.execute(sql`insert into execution_workspaces (id, company_id, name, mode, strategy_type) + values (${workspaceId}::uuid, ${companyId}::uuid, 'Policy workspace', 'shared', 'project_primary')`); + const roots = await db.insert(issues).values(["older", "equal", "fallback"].map(title => ({ companyId, title, status: "blocked" }))).returning(); + const blockers = await db.insert(issues).values(roots.map(root => ({ companyId, title: root.title + " blocker", status: "done", executionWorkspaceId: workspaceId }))).returning(); + for (let index = 0; index < roots.length; index += 1) { + await db.execute(sql`insert into issue_relations (company_id, issue_id, related_issue_id, type) + values (${companyId}::uuid, ${blockers[index]!.id}::uuid, ${roots[index]!.id}::uuid, 'blocks')`); + } + await db.execute(sql`insert into workspace_operations (company_id, execution_workspace_id, issue_id, phase, status, started_at) values + (${companyId}::uuid, ${workspaceId}::uuid, ${blockers[0]!.id}::uuid, 'worktree_prepare', 'succeeded', '2026-01-01T00:00:01Z'), + (${companyId}::uuid, ${workspaceId}::uuid, ${blockers[1]!.id}::uuid, 'worktree_prepare', 'succeeded', '2026-01-01T00:00:02Z'), + (${companyId}::uuid, ${workspaceId}::uuid, null, 'workspace_finalize', 'succeeded', '2026-01-01T00:00:02Z'), + (${companyId}::uuid, ${workspaceId}::uuid, null, 'worktree_prepare', 'succeeded', '2026-01-01T00:00:03Z')`); + const service = issueService(db); + expect((await service.listBlockerAttentionBatch(companyId, [roots[0]!])).nodeCount).toBe(1); + expect((await service.listBlockerAttentionBatch(companyId, [roots[1]!])).nodeCount).toBe(2); + expect((await service.listBlockerAttentionBatch(companyId, [roots[2]!])).nodeCount).toBe(2); + }); + +}); diff --git a/server/src/__tests__/company-portability.test.ts b/server/src/__tests__/company-portability.test.ts index f6dbbf552b6..9b7cfb3a2a9 100644 --- a/server/src/__tests__/company-portability.test.ts +++ b/server/src/__tests__/company-portability.test.ts @@ -4185,6 +4185,7 @@ describe("company portability", () => { }), // eslint-disable-next-line @typescript-eslint/no-explicit-any } as any; + db.transaction = async (write: (tx: typeof db) => Promise) => write(db); return { db, insertedRelationValues, monitorUpdates }; } diff --git a/server/src/__tests__/decision-effect-recovery-work.test.ts b/server/src/__tests__/decision-effect-recovery-work.test.ts index 6c1a17d2b39..8a7d4e300c8 100644 --- a/server/src/__tests__/decision-effect-recovery-work.test.ts +++ b/server/src/__tests__/decision-effect-recovery-work.test.ts @@ -73,7 +73,7 @@ const support = externalUrl ? { supported: true } : await getEmbeddedPostgresTes const claim = (id: string, updatedAt = new Date(), owner = db) => writeDecisionChange(owner, async tx => tx.update(decisions).set({ status: "decided", executionStatus: "running", chosenOptionId: "yes", decidedByUserId: userId, updatedAt, inputValues: {}, metadata: { continuationPending: true }, - }).where(eq(decisions.id, id))); + }).where(eq(decisions.id, id)), { companyId }); async function start() { coordinator = createDeliveryWorkCoordinator({ owner: db, canRun: () => !draining, canReconcile: () => true, onError: error => errors.push(error) }); @@ -467,7 +467,7 @@ const support = externalUrl ? { supported: true } : await getEmbeddedPostgresTes } else { await writeDecisionChange(db, async tx => tx.update(decisions).set({ status: "decided", executionStatus: "running", chosenOptionId: "archive", decidedByUserId: userId, updatedAt: new Date(Date.now() - 120_000), - metadata: { ...row.metadata, continuationPending: true } }).where(eq(decisions.id, row.id))); + metadata: { ...row.metadata, continuationPending: true } }).where(eq(decisions.id, row.id)), { companyId }); } await start(); await settled(row.id, committed ? "failed" : "succeeded"); await quiet(); const [retained] = await db.select().from(decisionRetention); diff --git a/server/src/__tests__/decision-retention-materialization-security.test.ts b/server/src/__tests__/decision-retention-materialization-security.test.ts new file mode 100644 index 00000000000..3c734455c59 --- /dev/null +++ b/server/src/__tests__/decision-retention-materialization-security.test.ts @@ -0,0 +1,187 @@ +import { randomUUID } from "node:crypto"; +import { and, eq, inArray, sql } from "drizzle-orm"; +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it } from "vitest"; +import { agents, authUsers, companies, companyMemberships, createDb, decisionRetention, issueRecoveryActions, issueRelations, issues } from "@paperclipai/db"; +import type { AttentionSourceKind } from "@paperclipai/shared"; +import { attentionService } from "../services/attention.js"; +import { decisionRetentionService } from "../services/decision-retention.js"; +import { issueRecoveryActionService } from "../services/issue-recovery-actions.js"; +import { issueService } from "../services/issues.js"; +import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; +import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js"; + +const external = resolveExternalTestDatabaseUrl("PAPERCLIP_RETENTION_MATERIALIZATION_TEST_DATABASE_URL"); +const support = external ? { supported: true } : await getEmbeddedPostgresTestSupport(); +(support.supported ? describe : describe.skip)("retention first-materialization security", () => { + let temporary: Awaited> | undefined; + let db: ReturnType; + let companyId: string, agentId: string, userId: string; + const users: string[] = []; + beforeAll(async () => { + if (!external) temporary = await startEmbeddedPostgresTestDatabase("retention-security-"); + db = createDb(external ?? temporary!.connectionString, { maxConnections: 6, idleTimeoutSeconds: 0, maxLifetimeSeconds: 0 }); + db.$client.options.onnotice = () => {}; + }, 30_000); + beforeEach(async () => { + companyId = randomUUID(); agentId = randomUUID(); userId = randomUUID(); users.push(userId); + await db.insert(authUsers).values({ id: userId, name: "Security operator", email: `${userId}@retention.test`, createdAt: new Date(), updatedAt: new Date() }); + await db.insert(companies).values({ id: companyId, name: "Materialization", issuePrefix: `S${companyId.slice(0, 6)}`, defaultResponsibleUserId: userId }); + await db.insert(companyMemberships).values({ companyId, principalType: "user", principalId: userId, membershipRole: "owner", status: "active" }); + await db.insert(agents).values({ id: agentId, companyId, name: "Source agent", status: "idle", adapterType: "process" }); + }); + afterEach(async () => { await db.execute(sql`truncate companies cascade`); }); + afterAll(async () => { if (db) { await db.delete(authUsers).where(inArray(authUsers.id, users)); await db.$client.end(); } await temporary?.cleanup(); }); + const mutation = (sourceKind: AttentionSourceKind, sourceId: string) => ({ companyId, sourceKind, sourceId, + authActor: { type: "board" as const, source: "local_implicit" as const, userId, companyIds: [companyId], isInstanceAdmin: true }, + actor: { actorType: "user" as const, actorId: userId, userId, agentId: null, runId: null, agentApiKeyId: null, responsibleUserId: userId } }); + async function issue(values: Partial = {}) { + return (await db.insert(issues).values({ companyId, title: "Security source", status: "todo", createdByAgentId: agentId, ...values }).returning())[0]!; + } + async function visible(sourceKind: AttentionSourceKind, sourceId: string) { + const feed = await attentionService(db).list(companyId, { all: true, allowUnscopedAll: true, userId, excludeSources: ["approval", "join_request"] }); + return feed.items.some(item => item.sourceKind === sourceKind && item.subject.id === sourceId); + } + + it("cannot pre-archive an unseen agent recovery before ordinary retry exhaustion asks the board for help", async () => { + const row = await issue(); + const owner = issueRecoveryActionService(db); + const input = { companyId, sourceIssueId: row.id, kind: "stranded_assigned_issue" as const, + ownerType: "agent" as const, ownerAgentId: agentId, cause: "fixture", fingerprint: "same-recovery", + nextAction: "Retry the existing work", maxAttempts: 2 }; + const action = await owner.upsertSourceScoped(input); + expect(action).toMatchObject({ ownerType: "agent", status: "active" }); + let rejected: unknown; + try { await decisionRetentionService(db).archive(mutation("recovery_action", action.id)); } catch (error) { rejected = error; } + const escalated = await owner.upsertSourceScoped(input); + expect(escalated).toMatchObject({ id: action.id, ownerType: "board", status: "escalated" }); + expect(await visible("recovery_action", action.id)).toBe(true); + expect(rejected).toMatchObject({ status: 404 }); + expect((await decisionRetentionService(db).getState(companyId, "recovery_action", action.id))?.archivedAt).toBeNull(); + }); + + it.each(["system", "resolved", "hidden", "harness", "conversation", "other_company"] as const)( + "rejects first retention state for a currently ineligible recovery source: %s", async scenario => { + const foreignCompany = randomUUID(); + if (scenario === "other_company") await db.insert(companies).values({ id: foreignCompany, name: "Foreign source", issuePrefix: `F${foreignCompany.slice(0, 6)}` }); + const row = await issue({ companyId: scenario === "other_company" ? foreignCompany : companyId, + hiddenAt: scenario === "hidden" ? new Date() : null, harnessKind: scenario === "harness" ? "fixture" : null, + ...(scenario === "conversation" ? { conversationAgentId: agentId, conversationUserId: userId, assigneeAgentId: agentId, conversationState: "active", status: "in_progress" } : {}) }); + const [action] = await db.insert(issueRecoveryActions).values({ companyId: row.companyId, sourceIssueId: row.id, + kind: "stranded_assigned_issue", cause: "fixture", fingerprint: scenario, nextAction: "Recover", + ownerType: scenario === "system" ? "system" : "board", status: scenario === "resolved" ? "resolved" : "active" }).returning(); + await expect(decisionRetentionService(db).archive(mutation("recovery_action", action.id))).rejects.toMatchObject({ status: 404 }); + expect(await db.select().from(decisionRetention).where(eq(decisionRetention.sourceId, action.id))).toEqual([]); + }); + + it("preserves existing recovery manual controls and legacy agent-owned TTL archival", async () => { + const row = await issue(), owner = issueRecoveryActionService(db), retention = decisionRetentionService(db); + const input = { companyId, sourceIssueId: row.id, kind: "stranded_assigned_issue" as const, + cause: "fixture", fingerprint: "known-recovery", nextAction: "Recover" }; + const action = await owner.upsertSourceScoped({ ...input, ownerType: "board" }); + expect(await visible("recovery_action", action.id)).toBe(true); + await owner.upsertSourceScoped({ ...input, ownerType: "agent", ownerAgentId: agentId }); + expect((await retention.archive(mutation("recovery_action", action.id))).archivedAt).toBeInstanceOf(Date); + await retention.revive(mutation("recovery_action", action.id)); + await retention.setKeep({ ...mutation("recovery_action", action.id), keep: true }); + await retention.setKeep({ ...mutation("recovery_action", action.id), keep: false }); + // An already materialized legacy row retains its existing TTL contract. + const activityAt = new Date(Date.now() - 100 * 86_400_000).toISOString(); + await db.update(issueRecoveryActions).set({ updatedAt: new Date(activityAt) }).where(eq(issueRecoveryActions.id, action.id)); + const states = await retention.syncItems(companyId, [{ sourceKind: "recovery_action", subject: { id: action.id }, activityAt }]); + const state = [...states.values()][0]; + expect(await retention.autoArchive({ companyId, items: [{ sourceKind: "recovery_action", subject: { id: action.id }, activityAt, + keep: state.keep, archivedAt: null, retentionVersion: state.version }] })).toBe(1); + expect(await retention.getState(companyId, "recovery_action", action.id)).toMatchObject({ archivedReason: "idle_ttl" }); + }); + + it.each(["review", "blocker_attention"] as const)("cannot pre-archive a future %s before an ordinary transition makes it visible", async kind => { + const row = await issue({ assigneeUserId: userId }); + let rejected: unknown; + try { await decisionRetentionService(db).archive(mutation(kind, row.id)); } catch (error) { rejected = error; } + await issueService(db).update(row.id, kind === "review" + ? { status: "in_review", actorUserId: userId } + : { status: "blocked", unblockDescriptor: { owner: "board", action: "Choose the next owner" }, actorUserId: userId }); + expect(await visible(kind, row.id)).toBe(true); + expect(rejected).toMatchObject({ status: 404 }); + expect((await decisionRetentionService(db).getState(companyId, kind, row.id))?.archivedAt).toBeNull(); + }); + + it.each(["review", "blocker_attention", "productivity_review"] as const)("does not materialize Keep for an ineligible %s", async kind => { + const row = await issue(); + await expect(decisionRetentionService(db).setKeep({ ...mutation(kind, row.id), keep: true })).rejects.toMatchObject({ status: 404 }); + expect(await decisionRetentionService(db).getState(companyId, kind, row.id)).toBeNull(); + }); + + it("allows Keep on an unseen eligible review and preserves existing manual actions after it leaves review", async () => { + const row = await issue({ status: "in_review", assigneeUserId: userId }); + const service = decisionRetentionService(db), input = mutation("review", row.id); + expect(await service.getState(companyId, "review", row.id)).toBeNull(); + expect((await service.setKeep({ ...input, keep: true })).keep).toBe(true); + await issueService(db).update(row.id, { status: "todo", actorUserId: userId }); + expect((await service.archive(input)).archivedReason).toBe("manual"); + expect((await service.setKeep({ ...input, keep: false })).keep).toBe(false); + expect((await service.revive(input)).archivedAt).toBeNull(); + }); + + it("allows unseen current board-owned blocker attention", async () => { + const row = await issue({ status: "blocked", unblockDescriptor: { owner: "board", action: "Choose next owner" }, blockedTransitionAt: new Date() }); + expect((await decisionRetentionService(db).setKeep({ ...mutation("blocker_attention", row.id), keep: true })).keep).toBe(true); + }); + + it("requires current agent error status for a new alert but preserves existing row actions", async () => { + const service = decisionRetentionService(db), input = mutation("agent_error_alert", agentId); + await expect(service.archive(input)).rejects.toMatchObject({ status: 404 }); + expect(await service.getState(companyId, "agent_error_alert", agentId)).toBeNull(); + await db.update(agents).set({ status: "error" }).where(eq(agents.id, agentId)); + expect((await service.setKeep({ ...input, keep: true })).keep).toBe(true); + await db.update(agents).set({ status: "idle" }).where(eq(agents.id, agentId)); + expect((await service.archive(input)).archivedAt).toBeInstanceOf(Date); + expect((await service.revive(input)).archivedAt).toBeNull(); + }); + + it("preserves historical productivity and terminal-leaf rows without inventing a new source", async () => { + const row = await issue(); const service = decisionRetentionService(db); + await service.syncItems(companyId, ["productivity_review", "blocker_attention"].map(sourceKind => ({ sourceKind: sourceKind as AttentionSourceKind, subject: { id: row.id }, activityAt: row.updatedAt.toISOString() }))); + for (const kind of ["productivity_review", "blocker_attention"] as const) { + const input = mutation(kind, row.id); + expect((await service.archive(input)).archivedAt).toBeInstanceOf(Date); + expect((await service.revive(input)).archivedAt).toBeNull(); + expect((await service.setKeep({ ...input, keep: true })).keep).toBe(true); + } + }); + + it("requires ordinary feed materialization for an unseen TODO terminal leaf, then preserves its manual controls", async () => { + const root = await issue({ status: "blocked" }); + const leaf = await issue({ status: "todo" }); + await db.insert(issueRelations).values({ companyId, issueId: leaf.id, relatedIssueId: root.id, type: "blocks" }); + const service = decisionRetentionService(db), input = mutation("blocker_attention", leaf.id); + await expect(service.setKeep({ ...input, keep: true })).rejects.toMatchObject({ status: 404 }); + expect(await service.getState(companyId, "blocker_attention", leaf.id)).toBeNull(); + const attention = await issueService(db).listBlockerAttention(companyId, [root]); + expect(attention.get(root.id)).toMatchObject({ terminalBlockerIssueId: leaf.id, state: "needs_attention", blockingTreeLive: false }); + expect(await visible("blocker_attention", leaf.id)).toBe(true); + expect((await service.archive(input)).archivedAt).toBeInstanceOf(Date); + expect((await service.revive(input)).archivedAt).toBeNull(); + expect((await service.setKeep({ ...input, keep: true })).keep).toBe(true); + }); + + it.each(["review", "blocker_attention"] as const)("does not pre-materialize hidden %s", async kind => { + const row = await issue({ hiddenAt: new Date(), status: kind === "review" ? "in_review" : "blocked", assigneeUserId: userId, + unblockDescriptor: { owner: "board", action: "Choose next owner" }, blockedTransitionAt: new Date() }); + await expect(decisionRetentionService(db).archive(mutation(kind, row.id))).rejects.toMatchObject({ status: 404 }); + expect(await decisionRetentionService(db).getState(companyId, kind, row.id)).toBeNull(); + }); + + it.each(["review", "agent_error_alert"] as const)("keeps company and agent authorization isolated for %s", async kind => { + const foreign = randomUUID(); + await db.insert(companies).values({ id: foreign, name: "Foreign", issuePrefix: "FOREIGN" }); + const sourceId = kind === "review" ? (await issue({ companyId: foreign, status: "in_review", assigneeUserId: userId })).id + : (await db.insert(agents).values({ companyId: foreign, name: "Foreign error", status: "error" }).returning())[0]!.id; + const service = decisionRetentionService(db); + await expect(service.archive(mutation(kind, sourceId))).rejects.toMatchObject({ status: 404 }); + const input = { ...mutation(kind, sourceId), companyId: foreign, authActor: { type: "agent" as const, agentId, companyId }, + actor: { actorType: "agent" as const, actorId: agentId, userId: null, agentId, runId: null, agentApiKeyId: null, responsibleUserId: null } }; + await expect(service.archive(input)).rejects.toMatchObject({ status: 404 }); + expect(await db.select().from(decisionRetention).where(and(eq(decisionRetention.companyId, foreign), eq(decisionRetention.sourceId, sourceId)))).toHaveLength(0); + }); +}); diff --git a/server/src/__tests__/decisions-service.test.ts b/server/src/__tests__/decisions-service.test.ts index 7fc2cc50c0d..999bde8d60e 100644 --- a/server/src/__tests__/decisions-service.test.ts +++ b/server/src/__tests__/decisions-service.test.ts @@ -793,7 +793,7 @@ describePg("decisionService", () => { expect((await service().get(id))?.metadata.continuationPending).toBe(false); } const markReady = (id: string, source = db) => writeDecisionChange(source, async tx => tx.update(decisions) - .set({ status: "cancelled", metadata: { continuationPending: true } }).where(eq(decisions.id, id))); + .set({ status: "cancelled", metadata: { continuationPending: true } }).where(eq(decisions.id, id)), { companyId }); it("coalesces sequential tenant writes without dropping pending recovery", async () => { const row = await createCommentDecision(); diff --git a/server/src/__tests__/git-restore-point-ledger.test.ts b/server/src/__tests__/git-restore-point-ledger.test.ts index 8b2e894dc44..0f30dea022b 100644 --- a/server/src/__tests__/git-restore-point-ledger.test.ts +++ b/server/src/__tests__/git-restore-point-ledger.test.ts @@ -73,7 +73,7 @@ describeEmbeddedPostgres("git restore point retention", () => { } = {}) { const companyId = input.companyId ?? randomUUID(); if (!input.companyId) { - await db.insert(companies).values({ id: companyId, name: "Restore", issuePrefix: `R${companyId.slice(0, 6)}` }); + await db.insert(companies).values({ id: companyId, name: "Restore", issuePrefix: `R${companyId.replaceAll("-", "")}` }); } const issueStatus = input.issueStatus ?? "done"; const closedAt = new Date(now.getTime() - (input.closedDaysAgo ?? 10) * DAY_MS); diff --git a/server/src/__tests__/issue-blocker-attention.test.ts b/server/src/__tests__/issue-blocker-attention.test.ts index 1e6c00631e7..4f1a0e71573 100644 --- a/server/src/__tests__/issue-blocker-attention.test.ts +++ b/server/src/__tests__/issue-blocker-attention.test.ts @@ -8,6 +8,7 @@ import { approvals, companies, createDb, + applyPendingMigrations, heartbeatRuns, issueApprovals, issueRelations, @@ -21,7 +22,10 @@ import { import { issueService } from "../services/issues.js"; import { buildIssueGraphLivenessIncidentKey } from "../services/recovery/origins.js"; -const embeddedPostgresSupport = await getEmbeddedPostgresTestSupport(); +import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js"; + +const externalUrl = resolveExternalTestDatabaseUrl("PAPERCLIP_BLOCKER_ATTENTION_TEST_DATABASE_URL"); +const embeddedPostgresSupport = externalUrl ? { supported: true } : await getEmbeddedPostgresTestSupport(); const describeEmbeddedPostgres = embeddedPostgresSupport.supported ? describe : describe.skip; if (!embeddedPostgresSupport.supported) { @@ -36,8 +40,9 @@ describeEmbeddedPostgres("issue blocker attention", () => { let tempDb: Awaited> | null = null; beforeAll(async () => { - tempDb = await startEmbeddedPostgresTestDatabase("paperclip-issue-blocker-attention-"); - db = createDb(tempDb.connectionString); + if (externalUrl) await applyPendingMigrations(externalUrl); + else tempDb = await startEmbeddedPostgresTestDatabase("paperclip-issue-blocker-attention-"); + db = createDb(externalUrl ?? tempDb!.connectionString, { maxConnections: 6 }); svc = issueService(db); }, 20_000); @@ -55,6 +60,7 @@ describeEmbeddedPostgres("issue blocker attention", () => { }); afterAll(async () => { + await db?.$client.end({ timeout: 1 }); await tempDb?.cleanup(); }); diff --git a/server/src/__tests__/issues-service.test.ts b/server/src/__tests__/issues-service.test.ts index fb13ff43e82..0d28858b95c 100644 --- a/server/src/__tests__/issues-service.test.ts +++ b/server/src/__tests__/issues-service.test.ts @@ -4151,7 +4151,7 @@ const dependencyTestDatabaseUrl = resolveExternalTestDatabaseUrl("PAPERCLIP_ISSU beforeAll(async () => { if (!dependencyTestDatabaseUrl) tempDb = await startEmbeddedPostgresTestDatabase("paperclip-issues-blockers-"); - db = createDb(dependencyTestDatabaseUrl ?? tempDb!.connectionString); + db = createDb(dependencyTestDatabaseUrl ?? tempDb!.connectionString, { maxConnections: 6 }); svc = issueService(db); await ensureIssueRelationsTable(db); }, 20_000); diff --git a/server/src/__tests__/server-startup-feedback-export.test.ts b/server/src/__tests__/server-startup-feedback-export.test.ts index 19634e87940..65efb679fc1 100644 --- a/server/src/__tests__/server-startup-feedback-export.test.ts +++ b/server/src/__tests__/server-startup-feedback-export.test.ts @@ -1,3 +1,4 @@ +import { attentionRetentionService } from "../services/attention-retention.js"; import { statusCardEvaluationService, registerStatusCardEvaluationWork } from "../services/status-card-evaluation.js"; import { nativeHarnessBackupExpirationService } from "../services/native-runtime/native-harness-backup-expiration.js"; import { skillVersionRetentionService } from "../services/skill-version-retention.js"; @@ -381,6 +382,11 @@ vi.mock("../realtime/live-events-ws.js", () => ({ setupLiveEventsWebSocketServer: vi.fn(), })); +vi.mock("../services/attention-retention.js", async importOriginal => ({ + ...await importOriginal(), + attentionRetentionService: vi.fn(() => ({ load: vi.fn(async () => ({ rows: [], complete: true })), check: vi.fn() })), +})); + vi.mock("../services/join-request-retention.js", async importOriginal => ({ ...await importOriginal(), joinRequestRetentionService: vi.fn(() => ({ sweep: vi.fn(async () => ({ archived: 0 })), next: vi.fn(async () => null) })), @@ -1022,7 +1028,7 @@ describe("startServer feedback export wiring", () => { } }); - it.each([false, true])("registers archive delivery separately from retention sweeps (scheduler %s)", async enabled => { + it.each([false, true])("registers source retention separately from archive delivery without tick scans (scheduler %s)", async enabled => { loadConfigMock.mockReturnValue(buildTestConfig({ heartbeatSchedulerEnabled: enabled, heartbeatSchedulerIntervalMs: 30000 })); let tick: (() => void) | undefined; const timer = vi.spyOn(globalThis, "setInterval").mockImplementation(((callback: () => void, interval: number) => { @@ -1034,12 +1040,17 @@ describe("startServer feedback export wiring", () => { const registration = deliveryWorkRegisterMock.mock.calls.find(([queue]) => queue === DELIVERY_QUEUES.decisionArchiveDelivery); if (!enabled) { expect(registration).toBeUndefined(); + expect(attentionRetentionService).not.toHaveBeenCalled(); expect(decisionRetentionService).not.toHaveBeenCalled(); expect(tick).toBeDefined(); tick!(); await settleStartupHousekeeping(); expect(decisionRetentionService).not.toHaveBeenCalled(); return; } expect(registration).toBeDefined(); + expect(deliveryWorkRegisterMock.mock.calls.some(([queue]) => queue === DELIVERY_QUEUES.attentionRetention)).toBe(true); + expect(attentionRetentionService).toHaveBeenCalledOnce(); + const source = vi.mocked(attentionRetentionService).mock.results[0]!.value; + expect(source.load).toHaveBeenCalledOnce(); const retention = vi.mocked(decisionRetentionService).mock.results[0]!.value; expect(retention.deliverNotifications).toHaveBeenCalledOnce(); expect(retention.nextArchiveNotificationAt).toHaveBeenCalledOnce(); @@ -1047,9 +1058,10 @@ describe("startServer feedback export wiring", () => { const companyReads = () => vi.mocked(db.select).mock.results.filter(({ value }) => vi.mocked(value.from).mock.calls.some(([table]) => table === companies)).length; const before = companyReads(); - expect(before).toBeGreaterThan(0); expect(tick).toBeDefined(); + expect(before).toBe(0); expect(tick).toBeDefined(); tick!(); await settleStartupHousekeeping(); - expect(companyReads()).toBeGreaterThan(before); + expect(companyReads()).toBe(before); + expect(source.load).toHaveBeenCalledOnce(); expect(retention.deliverNotifications).toHaveBeenCalledOnce(); await registration![1].run(new AbortController().signal); expect(retention.deliverNotifications).toHaveBeenCalledTimes(2); diff --git a/server/src/__tests__/work-products.test.ts b/server/src/__tests__/work-products.test.ts index 476e1156184..cef395409fd 100644 --- a/server/src/__tests__/work-products.test.ts +++ b/server/src/__tests__/work-products.test.ts @@ -1,3 +1,5 @@ +import { PgDialect } from "drizzle-orm/pg-core"; +import type { SQL } from "drizzle-orm"; import { describe, expect, it, vi } from "vitest"; import { enrichWorkProductMetadataWithDiff, @@ -33,6 +35,18 @@ function createWorkProductRow(overrides: Partial> = {}) }; } +function retentionReads(insideTransaction: () => boolean) { + return vi.fn(async (query: SQL) => { + const statement = new PgDialect().sqlToQuery(query); + expect(insideTransaction()).toBe(true); + expect(statement.params).toContain("company-1"); + expect(statement.params).toContain("issue-1"); + if (statement.sql.includes("origin_kind")) return [{ id: "issue-1", originKind: null, originId: null }]; + if (statement.sql.includes("with recursive affected")) return [{ issueId: "issue-1" }]; + throw new Error(`Unexpected work-product fixture query: ${statement.sql}`); + }); +} + describe("workProductService", () => { it("extracts runner totals and enriches work-product metadata", () => { const summary = workProductDiffSummaryFromEventPayload({ @@ -158,11 +172,16 @@ describe("workProductService", () => { const insertValues = vi.fn(() => ({ returning: insertReturning })); const txInsert = vi.fn(() => ({ values: insertValues })); + let transactionOpen = false; const tx = { + execute: retentionReads(() => transactionOpen), update: txUpdate, insert: txInsert, }; - const transaction = vi.fn(async (callback: (input: typeof tx) => Promise) => await callback(tx)); + const transaction = vi.fn(async (callback: (input: typeof tx) => Promise) => { + transactionOpen = true; + try { return await callback(tx); } finally { transactionOpen = false; } + }); const svc = workProductService({ transaction } as any); const result = await svc.createForIssue("issue-1", "company-1", { @@ -194,11 +213,16 @@ describe("workProductService", () => { const updateSet = vi.fn(() => ({ where: updateWhere })); const txUpdate = vi.fn(() => ({ set: updateSet })); + let transactionOpen = false; const tx = { + execute: retentionReads(() => transactionOpen), select: txSelect, update: txUpdate, }; - const transaction = vi.fn(async (callback: (input: typeof tx) => Promise) => await callback(tx)); + const transaction = vi.fn(async (callback: (input: typeof tx) => Promise) => { + transactionOpen = true; + try { return await callback(tx); } finally { transactionOpen = false; } + }); const svc = workProductService({ transaction } as any); const result = await svc.update("work-product-1", { diff --git a/server/src/index.ts b/server/src/index.ts index 069551303a8..d7122c1fcaa 100644 --- a/server/src/index.ts +++ b/server/src/index.ts @@ -1,3 +1,4 @@ +import { attentionRetentionService, registerAttentionRetentionWork } from "./services/attention-retention.js"; import { registerProviderTraceRetentionWork } from "./services/provider-trace-retention-work.js"; import { registerRunEvidenceMigrationWork } from "./services/run-evidence-migration-work.js"; import { registerSkillVersionRetentionWork } from "./services/skill-version-retention-work.js"; @@ -126,7 +127,6 @@ import { cloudActorHeaderSourceFromHeaders, resolveCloudTenantActor } from "./mi import { feedbackService, applyManagedEnvironments, - attentionService, backfillPrincipalAccessCompatibility, backfillLegacyToolOAuthTokens, bootstrapExecutionPolicyFromEnv, @@ -1887,23 +1887,6 @@ async function startServerWithDatabaseTeardown( if (config.heartbeatSchedulerEnabled) await heartbeat.registerStoppedInstructionCollectionWork(deliveryWork, () => !isIdleTaskDrainActive() && !heartbeatSchedulerStopped && !isWarmStandby()).ready; - const runRetentionSweep = async () => { - const activeCompanies = await db.select({ id: companies.id }).from(companies).where(eq(companies.status, "active")); - let archived = 0; - for (const company of activeCompanies) { - // Cursor pagination rebuilds the whole feed for every page; one - // unscoped all-items build keeps this sweep at a single feed build - // per company per tick. - const page = await attentionService(db as any).list(company.id, { - includeDismissed: true, - excludeSources: ["approval", "join_request"], - all: true, - allowUnscopedAll: true, - }); - archived += await retentionExecutor.autoArchive({ companyId: company.id, items: page.items }); - } - return { archived }; - }; // Startup housekeeping: provider and connection sweeps that every timer tick // repeats and that neither serving requests nor run recovery depends on. // Readiness does not wait for it. It is tracked as scheduler work, so idle @@ -1948,7 +1931,8 @@ async function startServerWithDatabaseTeardown( () => !isIdleTaskDrainActive() && !isWarmStandby() && !heartbeatSchedulerStopped).ready); await step("status_card_evaluation", () => registerStatusCardEvaluationWork(deliveryWork, statusCardEvaluation, () => !isIdleTaskDrainActive() && !isWarmStandby() && !heartbeatSchedulerStopped).ready); - await step("retention_sweep", runRetentionSweep); + await step("attention_retention", () => registerAttentionRetentionWork(deliveryWork, attentionRetentionService(db as any), + () => !isIdleTaskDrainActive() && !isWarmStandby() && !heartbeatSchedulerStopped).ready); }; const createHeartbeatSchedulerTick = (scheduled: ReturnType) => () => { @@ -1958,9 +1942,6 @@ async function startServerWithDatabaseTeardown( trackHeartbeatSchedulerWork((async () => { if (isIdleTaskDrainActive() || heartbeatSchedulerStopped) return; scheduleRoutineStatusReconcile(); - trackHeartbeatSchedulerWork(runRetentionSweep().catch((err: unknown) => { - logger.error({ err }, "decision retention sweep failed"); - })); const sweptRuntimeStatuses = heartbeat.sweepExpiredRuntimeStatuses(); if (sweptRuntimeStatuses > 0) { logger.info( diff --git a/server/src/modules/agent-lifecycle/__tests__/lifecycle.test.ts b/server/src/modules/agent-lifecycle/__tests__/lifecycle.test.ts index 246cb9f707a..404e76d6b30 100644 --- a/server/src/modules/agent-lifecycle/__tests__/lifecycle.test.ts +++ b/server/src/modules/agent-lifecycle/__tests__/lifecycle.test.ts @@ -40,7 +40,7 @@ const support = external ? { supported: true } : await getEmbeddedPostgresTestSu let companyId: string; const workers: ReturnType[] = []; beforeAll(async () => { - if (external) { await applyPendingMigrations(external); db = createDb(external); } + if (external) { await applyPendingMigrations(external); db = createDb(external, { maxConnections: 6 }); } else { database = await startEmbeddedPostgresTestDatabase("paperclip-agent-lifecycle-"); db = createDb(database.connectionString); } }, 90_000); afterAll(async () => { diff --git a/server/src/services/accounting-integrity.ts b/server/src/services/accounting-integrity.ts index e3f5ef1f649..02b6043ab2d 100644 --- a/server/src/services/accounting-integrity.ts +++ b/server/src/services/accounting-integrity.ts @@ -1,3 +1,4 @@ +import { notifyAttentionRetentionWork } from "./retention-work-notifications.js"; import { and, eq, sql } from "drizzle-orm"; import { agents, agentRuntimeState, companies, heartbeatRuns, type Db } from "@paperclipai/db"; import { normalizeCents, usdToCents, type AccountingFinding, type AccountingHealth, type AccountingInspection } from "@paperclipai/shared"; @@ -75,6 +76,8 @@ export function accountingIntegrityService(db: Db, hooks: BudgetServiceHooks = { if (!reason.trim()) throw conflict("A repair reason is required"); const before = await inspect(tx, companyId); if (before.fingerprint !== fingerprint) throw conflict("Accounting changed since inspection; inspect again before repairing"); + if (before.findings.some(f => f.repairable && f.kind === "agent_projection")) + await notifyAttentionRetentionWork(tx, { companyId, sourceKind: "agent_error_alert" }); for (const finding of before.findings.filter(f => f.repairable)) { if (finding.kind === "company_projection") await tx.update(companies).set({ spentMonthlyCents: sql`${finding.expected.cents}::numeric`, updatedAt: new Date() }).where(eq(companies.id, companyId)); if (finding.kind === "agent_projection") await tx.update(agents).set({ spentMonthlyCents: sql`${finding.expected.cents}::numeric`, updatedAt: new Date() }).where(and(eq(agents.id, finding.entityId), eq(agents.companyId, companyId))); diff --git a/server/src/services/agent-configuration.ts b/server/src/services/agent-configuration.ts index 2d30da27cb7..745dd9ff1ab 100644 --- a/server/src/services/agent-configuration.ts +++ b/server/src/services/agent-configuration.ts @@ -1,3 +1,4 @@ +import { notifyAttentionRetentionWork } from "./retention-work-notifications.js"; import { notifyDeferredExecutionEligibility } from "./deferred-execution-notifications.js"; import { notifyRunDispatch } from "./run-dispatch-notifications.js"; import { notifySkillVersionPinChange } from "./skill-version-retention-notifications.js"; @@ -18,6 +19,7 @@ import { clearPrimaryAgent } from "./primary-agent.js"; import { budgetServiceInTransaction } from "./budgets.js"; export async function prepareAgentConfiguration(db: Db, existing: AgentConfigurationRecord, data: AgentConfigurationPatch, options?: UpdateAgentOptions): Promise { + await notifyAttentionRetentionWork(db, { companyId: existing.companyId }); const id = existing.id; const { ensureManager, assertNoCycle, assertCompanyShortnameAvailable, assertBuiltInAgentMetadataMutationAllowed } = agentRecordQueries(db); if (existing.status === "pending_approval" && !options?.allowPendingApprovalConfigUpdate) { diff --git a/server/src/services/agent-deletion.ts b/server/src/services/agent-deletion.ts index 5907d222e1d..8c3defcbd86 100644 --- a/server/src/services/agent-deletion.ts +++ b/server/src/services/agent-deletion.ts @@ -1,3 +1,4 @@ +import { notifyAttentionRetentionWork } from "./retention-work-notifications.js"; import { notifyStatusCardIssueChange } from "./status-card-notifications.js"; import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js"; import { notifyExternalConversationSettlement } from "./external-conversation-settlement-notifications.js"; @@ -35,6 +36,7 @@ export async function deleteAgentDependencies(tx: Db, companyId: string, id: str if (classifierHold) throw conflict("Wait for active prompt injection checks or resolve their unknown charges in Costs before deleting this agent", { code: "agent_prompt_injection_accounting_pending", }); + await notifyAttentionRetentionWork(tx, { companyId }); await notifyExternalConversationSettlement(tx, { companyId }); await issueThreadInteractionService(tx).cancelPendingForDeletedAddressee(companyId, id); await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors); diff --git a/server/src/services/agent-hiring.ts b/server/src/services/agent-hiring.ts index a150937a212..0c238907bbc 100644 --- a/server/src/services/agent-hiring.ts +++ b/server/src/services/agent-hiring.ts @@ -1,3 +1,4 @@ +import { notifyAttentionRetentionWork } from "./retention-work-notifications.js"; import { notifySkillVersionRetention, notifySkillVersionPinChange } from "./skill-version-retention-notifications.js"; import { agentAppearanceSchema, randomAgentAppearance } from "@paperclipai/shared"; import { normalizePaperclipRunnerAdapterConfig } from "@paperclipai/adapter-utils/server-utils"; @@ -16,6 +17,7 @@ import { initializePrimaryAgent } from "./primary-agent.js"; import { resolveAgentRunnerTaskDriverForCompany, resolveNewAgentRunnerForCompany } from "./agent-runner-selection.js"; export async function prepareAgentHire(db: Db, companyId: string, input: CreateAgentData, options?: CreateAgentOptions): Promise { + await notifyAttentionRetentionWork(db, { companyId }); const { runner, ...data } = input; const selection = options?.runnerResolved ? { adapterType: data.adapterType ?? "process", adapterConfig: isPlainRecord(data.adapterConfig) ? data.adapterConfig : {} } diff --git a/server/src/services/agent-lifecycle.ts b/server/src/services/agent-lifecycle.ts index 8bb46882256..b7d42462ba0 100644 --- a/server/src/services/agent-lifecycle.ts +++ b/server/src/services/agent-lifecycle.ts @@ -1,3 +1,4 @@ +import { notifyAttentionRetentionWork } from "./retention-work-notifications.js"; import { notifyDeferredExecutionEligibility } from "./deferred-execution-notifications.js"; import { notifyRunDispatch } from "./run-dispatch-notifications.js"; import { notifyNativeReviewResponse } from "./native-review-response-notifications.js"; @@ -40,6 +41,7 @@ export function createAgentLifecycleEffects(_hooks: BudgetServiceHooks = {}): li }, notifyWork: tx => notifyDeliveryWork(tx, DELIVERY_QUEUES.agentLifecycle), async beforeStatusChange(tx, companyId, agentId, before, after) { + await notifyAttentionRetentionWork(tx, { companyId }); if (before !== after) { await notifyRunDispatch(tx, { companyId }); await notifyDeferredExecutionEligibility(tx, { companyId }); diff --git a/server/src/services/agent-project-access.ts b/server/src/services/agent-project-access.ts index 4c6ee81659a..feb183924c1 100644 --- a/server/src/services/agent-project-access.ts +++ b/server/src/services/agent-project-access.ts @@ -1,3 +1,4 @@ +import { notifyAttentionRetentionWork } from "./retention-work-notifications.js"; import { and, eq, inArray } from "drizzle-orm"; import { agents, projects, type Db } from "@paperclipai/db"; import { trustAuthorizationPolicySchema, type AgentProjectAccess } from "@paperclipai/shared"; @@ -55,6 +56,7 @@ export async function addAgentProjectAccess(db: Db, actor: AuthorizationActor, c const added = requested.filter(id => !previous.includes(id)); const next = [...previous, ...added]; if (added.length) { + await notifyAttentionRetentionWork(tx, { companyId, sourceKind: "agent_error_alert" }); await tx.update(agents).set({ permissions: { ...agent.permissions, authorizationPolicy: { ...policy, trustBoundary: { ...boundary, mode: "low_trust_review", companyId, projectIds: next }, } }, updatedAt: new Date() }).where(and(eq(agents.id, agentId), eq(agents.companyId, companyId))); diff --git a/server/src/services/agents.ts b/server/src/services/agents.ts index c9dd824f495..8a5f7551ee2 100644 --- a/server/src/services/agents.ts +++ b/server/src/services/agents.ts @@ -1,3 +1,4 @@ +import { notifyAttentionRetentionWork } from "./retention-work-notifications.js"; import { type BudgetServiceHooks } from "./budgets.js"; import { and, desc, eq, inArray } from "drizzle-orm"; @@ -52,7 +53,9 @@ export function agentService(db: Db, budgetHooks: BudgetServiceHooks = {}) { }); } - const updated = await db + const updated = await db.transaction(async tx => { + await notifyAttentionRetentionWork(tx, { companyId: existing.companyId }); + return tx .update(agents) .set({ permissions: normalizeAgentPermissions({ ...existing.permissions, ...permissions }), @@ -61,6 +64,7 @@ export function agentService(db: Db, budgetHooks: BudgetServiceHooks = {}) { .where(eq(agents.id, id)) .returning() .then((rows) => rows[0] ?? null); + }); return updated ? getById(updated.id) : null; }, diff --git a/server/src/services/attention-retention.ts b/server/src/services/attention-retention.ts new file mode 100644 index 00000000000..ddb74d682e5 --- /dev/null +++ b/server/src/services/attention-retention.ts @@ -0,0 +1,324 @@ +import { and, eq, getTableColumns, inArray, isNull, sql, type SQL } from "drizzle-orm"; +import { agents, decisionRetention, decisions, heartbeatRuns, + issueApprovals, issueRecoveryActions, issueThreadInteractions, issues, approvals, type Db } from "@paperclipai/db"; +import { evaluateAgentInvokability } from "./agent-invokability.js"; +import { HUMAN_RECOVERY_OWNER_TYPES, ATTENTION_BUDGET_WARNING_PERCENT, ATTENTION_AGENT_ERROR_STATUS, collapsePendingConfirmationsToNewest, listFailedRunsWithNewerRun, readRunIssueId, requiresReviewAttention, terminalBlockerRequiresAttention } from "./attention-source-eligibility.js"; +import { decisionRetentionService, DEFAULT_DECISION_ARCHIVE_DAYS } from "./decision-retention.js"; +import type { createDeliveryWorkCoordinator } from "./delivery-work-coordinator.js"; +import { DELIVERY_QUEUES } from "./delivery-work-notifications.js"; +import { attentionIssueCondition } from "./issue-visibility.js"; +import { RETRY_EXHAUSTION_MESSAGE_PREFIX } from "./heartbeat-run-events.js"; +import { issueService } from "./issues.js"; +import { parseIssueExecutionState } from "./issue-execution-policy.js"; +import { isProspectiveBlockedTransition } from "./routable-blocked.js"; +import { registerScopedDeadlineWork } from "./scoped-deadline-work.js"; +import type { AttentionRetentionScope } from "./retention-work-notifications.js"; + +const archiveDelay = DEFAULT_DECISION_ARCHIVE_DAYS * 86_400_000; +const kinds = ["decision", "issue_thread_interaction", "recovery_action", "blocker_attention", "review", + "failed_run", "budget_alert", "agent_error_alert"] as const; +type Kind = typeof kinds[number]; +type Scope = AttentionRetentionScope; +type Candidate = { sourceKind: Kind; sourceId: string; companyId: string; activityAt: string; deadlineAt: number; issueId: string | null }; +type Entry = Candidate & { id: string; dueAt: number; rootId: string; variant?: "owner" | "terminal" }; +type Cursor = { sourceKind: string; sourceId: string }; + +type PageIssue = typeof issues.$inferSelect & { activityAt: string; deadlineAt: number; + blockedActivityAt: string | null; blockedDeadlineAt: number | null }; +type PageEvidence = { + decisions: Map; + recoveries: Map; + runs: Map; + agents: Set; + visibleIssues: Map; + newerRuns: Set; + deferredRoots: Set; + blockers: Awaited["listBlockerAttention"]>>; +}; + +const sourceTables: Record = { + decision: sql`decisions`, issue_thread_interaction: sql`issue_thread_interactions`, + recovery_action: sql`issue_recovery_actions`, blocker_attention: sql`issues`, review: sql`issues`, + failed_run: sql`heartbeat_runs`, budget_alert: sql`budget_incidents`, agent_error_alert: sql`agents`, +}; + +const sources: Record = { + decision: sql`select 'decision' as source_kind, d.id::text as source_id, d.company_id, d.updated_at as activity_at, d.origin_issue_id::text as issue_id + from decisions d where d.status = 'open'`, + issue_thread_interaction: sql`select 'issue_thread_interaction', i.id::text, i.company_id, i.updated_at, i.issue_id::text + from issue_thread_interactions i where i.status = 'pending' and i.addressee_user_id is null`, + recovery_action: sql`select 'recovery_action', r.id::text, r.company_id, r.updated_at, r.source_issue_id::text + from issue_recovery_actions r where r.status in ('active', 'escalated') and r.owner_type in (${sql.join(HUMAN_RECOVERY_OWNER_TYPES.map(owner => sql`${owner}`), sql`, `)})`, + blocker_attention: sql`select 'blocker_attention', i.id::text, i.company_id, i.updated_at, i.id::text + from issues i where i.status = 'blocked' and i.hidden_at is null and i.harness_kind is null`, + review: sql`select 'review', i.id::text, i.company_id, i.updated_at, i.id::text + from issues i where i.status = 'in_review' and i.hidden_at is null and i.harness_kind is null`, + failed_run: sql`select 'failed_run', r.id::text, r.company_id, coalesce(r.finished_at, r.updated_at, r.created_at), coalesce(r.context_snapshot->>'issueId', r.context_snapshot->>'taskId') + from heartbeat_runs r where r.status in ('failed', 'timed_out') and exists ( + select 1 from heartbeat_run_events e where e.run_id = r.id and e.company_id = r.company_id + and e.event_type = 'lifecycle' and e.message like ${`${RETRY_EXHAUSTION_MESSAGE_PREFIX}%`})`, + budget_alert: sql`select 'budget_alert', b.id::text, b.company_id, b.updated_at, null::text from budget_incidents b + where b.status = 'open' and (b.threshold_type = 'hard' or (b.amount_limit > 0 + and round(b.amount_observed::numeric / b.amount_limit * 10000) / 100 >= ${ATTENTION_BUDGET_WARNING_PERCENT}))`, + agent_error_alert: sql`select 'agent_error_alert', a.id::text, a.company_id, a.updated_at, null::text from agents a where a.status = ${ATTENTION_AGENT_ERROR_STATUS}`, +}; + +export function attentionRetentionService(db: Db) { + const cursors = new Map(); + async function candidates(database: Db, scope: Scope = {}, cursor?: Cursor, sourceId?: string): Promise { + const issueIds = scope.issueIds ?? (scope.issueId ? [scope.issueId] : []); + const selected = kinds.filter(kind => (!scope.sourceKind || scope.sourceKind === kind) + && (!issueIds.length || !["budget_alert", "agent_error_alert"].includes(kind))); + if (!selected.length) return []; + const rows = await database.execute(sql`with source_candidates(source_kind, source_id, company_id, activity_at, issue_id) as (${sql.join(selected.map(kind => sources[kind]), sql` union all `)}) + select s.source_kind as "sourceKind", s.source_id as "sourceId", s.company_id as "companyId", + s.issue_id as "issueId", s.activity_at::text as "activityAt", ceil(extract(epoch from s.activity_at) * 1000 + ${archiveDelay}) as "deadlineAt" + from source_candidates s inner join companies c on c.id = s.company_id + left join decision_retention r on r.company_id = s.company_id and r.source_kind = s.source_kind and r.source_id = s.source_id + where c.status = 'active' + ${issueIds.length ? sql`and s.issue_id in (${sql.join(issueIds.map(id => sql`${id}`), sql`, `)})` : sql``} + and (${scope.companyId ?? null}::uuid is null or s.company_id = ${scope.companyId ?? null}::uuid) + and (${sourceId ?? scope.sourceId ?? null}::text is null or s.source_id = ${sourceId ?? scope.sourceId ?? null}) + and (s.source_kind = 'blocker_attention' or ((r.id is null or r.keep = false) and r.archived_at is null)) + ${cursor ? sql`and (s.source_kind, s.source_id) > (${cursor.sourceKind}, ${cursor.sourceId})` : sql``} + order by s.source_kind, s.source_id limit 100`) as unknown as Candidate[]; + return rows.map(row => ({ ...row, deadlineAt: Number(row.deadlineAt) })); + } + + async function visibleIssue(database: Db, companyId: string, issueId: string, conversations = false) { + const [row] = await database.select().from(issues).where(and(eq(issues.companyId, companyId), eq(issues.id, issueId), + attentionIssueCondition(conversations))); + return row; + } + + async function interactionEligibility(database: Db, candidates: Candidate[]) { + const result = new Map(); + const byCompany = new Map(); + for (const candidate of candidates) { + if (candidate.sourceKind !== "issue_thread_interaction") continue; + const ids = byCompany.get(candidate.companyId) ?? []; + ids.push(candidate.sourceId); + byCompany.set(candidate.companyId, ids); + } + for (const [companyId, ids] of byCompany) { + const rows = await database.select().from(issueThreadInteractions).where(and( + eq(issueThreadInteractions.companyId, companyId), inArray(issueThreadInteractions.id, ids), + eq(issueThreadInteractions.status, "pending"), isNull(issueThreadInteractions.addresseeUserId))); + if (!rows.length) continue; + const visible = await database.select({ id: issues.id }).from(issues).where(and(eq(issues.companyId, companyId), + inArray(issues.id, rows.map(row => row.issueId)), attentionIssueCondition(true))); + const visibleIds = new Set(visible.map(row => row.id)); + const roster = await database.select().from(agents).where(eq(agents.companyId, companyId)); + const boardAddressed = (row: typeof issueThreadInteractions.$inferSelect) => !row.addresseeAgentId + || !evaluateAgentInvokability(roster.find(agent => agent.id === row.addresseeAgentId), roster).invokable; + const confirmationIssueIds = rows.filter(row => row.kind === "request_confirmation").map(row => row.issueId); + const siblings = confirmationIssueIds.length ? await database.select().from(issueThreadInteractions).where(and( + eq(issueThreadInteractions.companyId, companyId), inArray(issueThreadInteractions.issueId, confirmationIssueIds), + eq(issueThreadInteractions.status, "pending"), eq(issueThreadInteractions.kind, "request_confirmation"), + isNull(issueThreadInteractions.addresseeUserId))) : []; + const newest = new Set(collapsePendingConfirmationsToNewest(siblings.filter(boardAddressed)).map(row => row.id)); + for (const row of rows) result.set(row.id, visibleIds.has(row.issueId) && boardAddressed(row) + && (row.kind !== "request_confirmation" || newest.has(row.id))); + } + return result; + } + + async function reviewEligibility(database: Db, candidates: Candidate[]) { + const result = new Map(); + const byCompany = new Map(); + for (const candidate of candidates) { + if (candidate.sourceKind !== "review") continue; + const ids = byCompany.get(candidate.companyId) ?? []; + ids.push(candidate.sourceId); + byCompany.set(candidate.companyId, ids); + } + for (const [companyId, ids] of byCompany) { + const rows = await database.select().from(issues).where(and(eq(issues.companyId, companyId), + inArray(issues.id, ids), attentionIssueCondition(false))); + const pendingApprovals = await database.select({ issueId: issueApprovals.issueId }).from(issueApprovals) + .innerJoin(approvals, eq(approvals.id, issueApprovals.approvalId)).where(and( + eq(issueApprovals.companyId, companyId), eq(approvals.companyId, companyId), + inArray(issueApprovals.issueId, ids), eq(approvals.status, "pending"))); + const approved = new Set(pendingApprovals.map(row => row.issueId)); + const stalled = await issueService(database).listReviewAttention(companyId, rows); + for (const row of rows) { + const state = parseIssueExecutionState(row.executionState); + result.set(row.id, requiresReviewAttention({ + hasHumanParticipant: state?.status === "pending" && state.currentParticipant?.type === "user", + hasUserAssignee: Boolean(row.assigneeUserId), hasPendingApproval: approved.has(row.id), + stalled: stalled.get(row.id)?.state === "stalled", + })); + } + } + return result; + } + + async function pageEvidence(database: Db, candidates: Candidate[]) { + const byCompany = new Map(); + for (const candidate of candidates) { + if (!["decision", "recovery_action", "failed_run", "blocker_attention"].includes(candidate.sourceKind)) continue; + const rows = byCompany.get(candidate.companyId) ?? []; + rows.push(candidate); + byCompany.set(candidate.companyId, rows); + } + const result = new Map(); + for (const [companyId, rows] of byCompany) { + const ids = (kind: Kind) => rows.filter(row => row.sourceKind === kind).map(row => row.sourceId); + const decisionIds = ids("decision"), recoveryIds = ids("recovery_action"), runIds = ids("failed_run"), blockerIds = ids("blocker_attention"); + const decisionRows = decisionIds.length ? await database.select({ id: decisions.id, originIssueId: decisions.originIssueId }) + .from(decisions).where(and(eq(decisions.companyId, companyId), inArray(decisions.id, decisionIds), eq(decisions.status, "open"))) : []; + const recoveryRows = recoveryIds.length ? await database.select().from(issueRecoveryActions).where(and( + eq(issueRecoveryActions.companyId, companyId), inArray(issueRecoveryActions.id, recoveryIds))) : []; + const runRows = runIds.length ? await database.select().from(heartbeatRuns).where(and( + eq(heartbeatRuns.companyId, companyId), inArray(heartbeatRuns.id, runIds))) : []; + const runAgents = runRows.length ? await database.select({ id: agents.id }).from(agents).where(and( + eq(agents.companyId, companyId), inArray(agents.id, [...new Set(runRows.map(row => row.agentId))]), + sql`${agents.status} <> 'terminated'`)) : []; + const readIssues = async (issueIds: string[]) => issueIds.length ? database.select({ ...getTableColumns(issues), + activityAt: sql`${issues.updatedAt}::text`, + deadlineAt: sql`ceil(extract(epoch from ${issues.updatedAt}) * 1000 + ${archiveDelay})`.mapWith(Number), + blockedActivityAt: sql`${issues.blockedTransitionAt}::text`, + blockedDeadlineAt: sql`ceil(extract(epoch from ${issues.blockedTransitionAt}) * 1000 + ${archiveDelay})`.mapWith(Number), + }).from(issues).where(and(eq(issues.companyId, companyId), inArray(issues.id, [...new Set(issueIds)]), + attentionIssueCondition(false))) : []; + const issueIds = [...blockerIds, ...decisionRows.map(row => row.originIssueId), ...recoveryRows.map(row => row.sourceIssueId), + ...runRows.map(row => readRunIssueId(row.contextSnapshot))].filter((id): id is string => Boolean(id)); + const visibleRows = await readIssues(issueIds); + const visible = new Map(visibleRows.map(row => [row.id, row])); + const blockerRoots = visibleRows.filter(row => blockerIds.includes(row.id)); + const blockerBatch = blockerRoots.length ? await issueService(database).listBlockerAttentionBatch(companyId, blockerRoots) + : { attention: new Map(), deferredRootIds: new Set() }; + const blockers: PageEvidence["blockers"] = blockerBatch.attention; + const terminalIds = [...blockers.values()].filter(terminalBlockerRequiresAttention) + .map(attention => attention.terminalBlockerIssueId).filter((id): id is string => id != null && !visible.has(id)); + for (const row of await readIssues(terminalIds)) visible.set(row.id, row); + result.set(companyId, { + decisions: new Map(decisionRows.map(row => [row.id, row])), recoveries: new Map(recoveryRows.map(row => [row.id, row])), + runs: new Map(runRows.map(row => [row.id, row])), agents: new Set(runAgents.map(row => row.id)), visibleIssues: visible, + newerRuns: await listFailedRunsWithNewerRun(database, companyId, + runRows.map(run => ({ ...run, issueId: readRunIssueId(run.contextSnapshot) }))), blockers, deferredRoots: blockerBatch.deferredRootIds, + }); + } + return result; + } + + async function eligible(database: Db, candidate: Candidate, variant?: "owner" | "terminal", lock = false, reviewAttention?: Map, interactionAttention?: Map, evidence?: PageEvidence): Promise { + if (candidate.sourceKind === "blocker_attention" && variant === "terminal" && evidence?.deferredRoots.has(candidate.sourceId)) { + return { ...candidate, rootId: candidate.sourceId, variant, + id: `${candidate.sourceKind}:${candidate.sourceId}:terminal`, dueAt: Date.now() }; + } + let sourceId = candidate.sourceId; + let activityAt = candidate.activityAt; + let dueAt = candidate.deadlineAt; + if (candidate.sourceKind === "decision") { + const row = evidence ? evidence.decisions.get(sourceId) : (await database.select({ originIssueId: decisions.originIssueId }).from(decisions) + .where(and(eq(decisions.companyId, candidate.companyId), eq(decisions.id, sourceId), eq(decisions.status, "open"))))[0]; + if (!row || row.originIssueId && !(evidence ? evidence.visibleIssues.has(row.originIssueId) + : await visibleIssue(database, candidate.companyId, row.originIssueId))) return null; + } else if (candidate.sourceKind === "issue_thread_interaction") { + const attention = interactionAttention ?? await interactionEligibility(database, [candidate]); + if (!attention.get(sourceId)) return null; + } else if (candidate.sourceKind === "recovery_action") { + const row = evidence ? evidence.recoveries.get(sourceId) : (await database.select().from(issueRecoveryActions).where(and(eq(issueRecoveryActions.companyId, candidate.companyId), eq(issueRecoveryActions.id, sourceId))))[0]; + if (!row || !(evidence ? evidence.visibleIssues.has(row.sourceIssueId) + : await visibleIssue(database, candidate.companyId, row.sourceIssueId))) return null; + } else if (candidate.sourceKind === "review") { + const attention = reviewAttention ?? await reviewEligibility(database, [candidate]); + if (!attention.get(sourceId)) return null; + } else if (candidate.sourceKind === "blocker_attention") { + const row = evidence ? evidence.visibleIssues.get(sourceId) : await visibleIssue(database, candidate.companyId, sourceId); + if (!row) return null; + if (variant === "owner") { + if (row.unblockDescriptor?.owner !== "board" || !isProspectiveBlockedTransition(row)) return null; + const activity = evidence ? { at: evidence.visibleIssues.get(sourceId)!.blockedActivityAt!, due: evidence.visibleIssues.get(sourceId)!.blockedDeadlineAt! } : (await database.execute(sql`select blocked_transition_at::text as at, + ceil(extract(epoch from blocked_transition_at) * 1000 + ${archiveDelay}) as due from issues + where company_id = ${candidate.companyId}::uuid and id = ${row.id}::uuid`) as unknown as Array<{ at: string; due: string }>)[0]; + activityAt = activity.at; dueAt = Number(activity.due); + } else { + const fresh = evidence ? undefined : await issueService(database).listBlockerAttentionBatch(candidate.companyId, [row]); + // A bounded graph cannot establish absence of live work beyond its frontier. + // Keep unknown roots visible; a later source change can revalidate them. + if (fresh?.deferredRootIds.has(row.id)) return null; + const attention = evidence ? evidence.blockers.get(row.id) : fresh!.attention.get(row.id); + if (!terminalBlockerRequiresAttention(attention)) return null; + sourceId = attention.terminalBlockerIssueId ?? row.id; + if (lock && sourceId !== candidate.sourceId) await database.execute(sql`select id from issues + where company_id = ${candidate.companyId}::uuid and id = ${sourceId}::uuid for update`); + const terminal = evidence ? evidence.visibleIssues.get(sourceId) : await visibleIssue(database, candidate.companyId, sourceId); + if (!terminal) return null; + const activity = evidence ? { at: evidence.visibleIssues.get(sourceId)!.activityAt, due: evidence.visibleIssues.get(sourceId)!.deadlineAt } : (await database.execute(sql`select updated_at::text as at, + ceil(extract(epoch from updated_at) * 1000 + ${archiveDelay}) as due from issues + where company_id = ${candidate.companyId}::uuid and id = ${sourceId}::uuid`) as unknown as Array<{ at: string; due: string }>)[0]; + activityAt = activity.at; dueAt = Number(activity.due); + } + const [retention] = await database.select().from(decisionRetention).where(and(eq(decisionRetention.companyId, candidate.companyId), + eq(decisionRetention.sourceKind, candidate.sourceKind), eq(decisionRetention.sourceId, sourceId))); + if (retention?.keep || retention?.archivedAt) return null; + } else if (candidate.sourceKind === "failed_run") { + const run = evidence ? evidence.runs.get(sourceId) : (await database.select().from(heartbeatRuns).where(and(eq(heartbeatRuns.companyId, candidate.companyId), eq(heartbeatRuns.id, sourceId))))[0]; + if (!run) return null; + const agent = evidence ? evidence.agents.has(run.agentId) : (await database.select({ id: agents.id }).from(agents).where(and(eq(agents.companyId, candidate.companyId), + eq(agents.id, run.agentId), sql`${agents.status} <> 'terminated'`)))[0]; + const issueId = readRunIssueId(run.contextSnapshot); + if (!agent || issueId && !(evidence ? evidence.visibleIssues.has(issueId) + : await visibleIssue(database, candidate.companyId, issueId))) return null; + if ((evidence ? evidence.newerRuns : await listFailedRunsWithNewerRun(database, candidate.companyId, [{ ...run, issueId }])).has(run.id)) return null; + } + return { ...candidate, sourceId, activityAt, rootId: candidate.sourceId, variant, + id: `${candidate.sourceKind}:${candidate.sourceId}:${variant ?? "source"}`, dueAt }; + } + + async function load(scope?: Scope, signal?: AbortSignal) { + const key = JSON.stringify(scope ?? {}); + const rows = await candidates(db, scope, cursors.get(key)); + const entries: Entry[] = []; + const reviews = await reviewEligibility(db, rows); + const interactions = await interactionEligibility(db, rows); + const evidence = await pageEvidence(db, rows); + for (const row of rows) { + if (signal?.aborted) return { rows: null, complete: true }; + for (const variant of row.sourceKind === "blocker_attention" ? ["owner", "terminal"] as const : [undefined]) { + const entry = await eligible(db, row, variant, false, reviews, interactions, evidence.get(row.companyId)); + if (entry) entries.push(entry); + } + } + if (rows.length === 100) { + const last = rows[rows.length - 1]!; + cursors.set(key, { sourceKind: last.sourceKind, sourceId: last.sourceId }); + } else cursors.delete(key); + return { rows: entries, complete: rows.length < 100 }; + } + + async function check(entry: Entry) { + return db.transaction(async transaction => { + const database = transaction as unknown as Db; + await database.execute(sql`select id from ${sourceTables[entry.sourceKind]} + where company_id = ${entry.companyId}::uuid and id = ${entry.rootId}::uuid for update`); + const [candidate] = await candidates(database, { companyId: entry.companyId, sourceKind: entry.sourceKind }, undefined, entry.rootId); + if (!candidate) return null; + const current = await eligible(database, candidate, entry.variant, true); + if (!current || current.dueAt > Date.now()) return current; + const retention = decisionRetentionService(database); + const item = { sourceKind: current.sourceKind, subject: { id: current.sourceId }, activityAt: current.activityAt }; + const state = (await retention.syncItems(current.companyId, [item])).get(`${current.sourceKind}:${current.sourceId}`)!; + await retention.autoArchive({ companyId: current.companyId, items: [{ ...item, + keep: state.keep, archivedAt: state.archivedAt?.toISOString() ?? null, retentionVersion: state.version }] }); + return null; + }); + } + return { load, check }; +} + +export function registerAttentionRetentionWork(coordinator: ReturnType, + service: ReturnType, canRun: () => boolean) { + return registerScopedDeadlineWork(coordinator, { + queue: DELIVERY_QUEUES.attentionRetention, + decodeScope: hint => hint === "*" ? undefined : JSON.parse(hint) as Scope, + matches: (entry, scope) => (!scope?.companyId || scope.companyId === entry.companyId) + && (!scope?.sourceKind || scope.sourceKind === entry.sourceKind) + && (!scope?.sourceId || scope.sourceId === entry.sourceId) + && (!scope?.issueId || scope.issueId === entry.issueId) + && (!scope?.issueIds?.length || entry.issueId !== null && scope.issueIds.includes(entry.issueId)), + load: service.load, check: service.check, + }, canRun); +} diff --git a/server/src/services/attention-source-eligibility.ts b/server/src/services/attention-source-eligibility.ts new file mode 100644 index 00000000000..971ca5f1ee5 --- /dev/null +++ b/server/src/services/attention-source-eligibility.ts @@ -0,0 +1,118 @@ +import { and, eq, gt, inArray, sql, type SQL } from "drizzle-orm"; +import { heartbeatRuns, issueRecoveryActions, type Db } from "@paperclipai/db"; + +export const HUMAN_RECOVERY_OWNER_TYPES = ["user", "board"] as const; +const OPEN_RECOVERY_ATTENTION_STATUSES = ["active", "escalated"] as const; + +export function recoveryActionAttentionCondition() { + return and(inArray(issueRecoveryActions.status, [...OPEN_RECOVERY_ATTENTION_STATUSES]), + inArray(issueRecoveryActions.ownerType, [...HUMAN_RECOVERY_OWNER_TYPES]))!; +} +export const ATTENTION_BUDGET_WARNING_PERCENT = 85; +export const ATTENTION_AGENT_ERROR_STATUS = "error"; + +export function collapsePendingConfirmationsToNewest(rows: T[]) { + const independent = (row: T) => Boolean(row.payload && typeof row.payload === "object" && "qualityReview" in row.payload && row.payload.qualityReview); + const newestByGroup = new Map(); + for (const row of rows) { + if (row.kind !== "request_confirmation" || independent(row)) continue; + const groupKey = `${row.issueId}:${row.kind}`; + const newest = newestByGroup.get(groupKey); + if ( + !newest + || row.createdAt.getTime() > newest.createdAt.getTime() + || (row.createdAt.getTime() === newest.createdAt.getTime() && row.id > newest.id) + ) { + newestByGroup.set(groupKey, row); + } + } + + return rows.filter((row) => ( + row.kind !== "request_confirmation" + || independent(row) + || newestByGroup.get(`${row.issueId}:${row.kind}`)?.id === row.id + )); +} + +export function readRunIssueId(contextSnapshot: Record | null) { + const issueId = contextSnapshot?.issueId ?? contextSnapshot?.taskId; + return typeof issueId === "string" && issueId.length > 0 ? issueId : null; +} + +type FailedRunProbe = { id: string; agentId: string; issueId: string | null; createdAt: Date }; + +const FAILED_RUN_PROBE_BATCH_SIZE = 1000; + +export async function listFailedRunsWithNewerRun(db: Db, companyId: string, failedRuns: FailedRunProbe[]) { + const withNewerRun = new Set(); + const keyed = failedRuns.filter(run => run.issueId !== null); + for (let start = 0; start < keyed.length; start += FAILED_RUN_PROBE_BATCH_SIZE) { + const batch = keyed.slice(start, start + FAILED_RUN_PROBE_BATCH_SIZE); + for (const id of await listFailedRunBatchWithNewerRun(db, companyId, batch)) withNewerRun.add(id); + } + const unkeyed = failedRuns.filter(run => run.issueId === null); + if (unkeyed.length === 0) return withNewerRun; + const agentIds = [...new Set(unkeyed.map(run => run.agentId))]; + const oldestCreatedAt = unkeyed.reduce((oldest, run) => run.createdAt < oldest ? run.createdAt : oldest, + unkeyed[0]!.createdAt); + const latestByAgent = new Map(); + for (let start = 0; start < agentIds.length; start += FAILED_RUN_PROBE_BATCH_SIZE) { + const rows = await db.select({ + agentId: heartbeatRuns.agentId, + createdAt: sql`max(${heartbeatRuns.createdAt})`.mapWith(heartbeatRuns.createdAt), + }).from(heartbeatRuns).where(and( + eq(heartbeatRuns.companyId, companyId), + inArray(heartbeatRuns.agentId, agentIds.slice(start, start + FAILED_RUN_PROBE_BATCH_SIZE)), + gt(heartbeatRuns.createdAt, oldestCreatedAt), + sql`coalesce(${heartbeatRuns.contextSnapshot} ->> 'issueId', ${heartbeatRuns.contextSnapshot} ->> 'taskId', '') = ''`, + )).groupBy(heartbeatRuns.agentId); + for (const row of rows) latestByAgent.set(row.agentId, row.createdAt); + } + for (const run of unkeyed) { + if ((latestByAgent.get(run.agentId)?.getTime() ?? 0) > run.createdAt.getTime()) withNewerRun.add(run.id); + } + return withNewerRun; +} + +async function listFailedRunBatchWithNewerRun(db: Db, companyId: string, failedRuns: FailedRunProbe[]) { + if (failedRuns.length === 0) return []; + const failed = sql.join( + failedRuns.map((run) => sql`(${run.id}::uuid, ${run.agentId}::uuid, ${run.issueId}::text, ${new Date(run.createdAt.getTime() + 1).toISOString()}::timestamptz)`), + sql`, `, + ); + const runIssueId = sql`(${heartbeatRuns.contextSnapshot} ->> 'issueId')`; + const runTaskId = sql`(${heartbeatRuns.contextSnapshot} ->> 'taskId')`; + const newerRunExists = (match: SQL) => sql`exists ( + select 1 from ${heartbeatRuns} + where ${heartbeatRuns.companyId} = ${companyId} + and ${heartbeatRuns.agentId} = failed.agent_id + and ${heartbeatRuns.createdAt} >= failed.newer_from + and ${match} + )`; + const rows = await db.execute(sql` + select failed.id + from (values ${failed}) as failed (id, agent_id, issue_id, newer_from) + where ${newerRunExists(sql`${runIssueId} = failed.issue_id`)} + or ${newerRunExists(sql`${runTaskId} = failed.issue_id and ${runIssueId} is null`)} + `) as unknown as Array<{ id: string }>; + return rows.map((row) => row.id); +} + + +export function requiresReviewAttention(input: { + hasHumanParticipant: boolean; hasUserAssignee: boolean; hasPendingApproval: boolean; stalled: boolean; +}) { + return input.hasHumanParticipant || input.hasUserAssignee || input.hasPendingApproval || input.stalled; +} + +export function terminalBlockerRequiresAttention( + attention: T | null | undefined, +): attention is T & { state: "stalled" | "needs_attention" } { + return Boolean(attention && !attention.blockingTreeLive && ["stalled", "needs_attention"].includes(attention.state ?? "")); +} diff --git a/server/src/services/attention.ts b/server/src/services/attention.ts index 9f583e774da..bbbf505f480 100644 --- a/server/src/services/attention.ts +++ b/server/src/services/attention.ts @@ -1,5 +1,6 @@ +import { recoveryActionAttentionCondition, ATTENTION_BUDGET_WARNING_PERCENT, ATTENTION_AGENT_ERROR_STATUS, collapsePendingConfirmationsToNewest, readRunIssueId, listFailedRunsWithNewerRun, requiresReviewAttention, terminalBlockerRequiresAttention } from "./attention-source-eligibility.js"; import { approvalExecutionIssueCondition, firstApprovalIssueId } from "./approval-visibility.js"; -import { and, asc, desc, eq, gt, inArray, isNotNull, isNull, notInArray, or, sql, type SQL } from "drizzle-orm"; +import { and, asc, desc, eq, inArray, isNull, notInArray, or, sql, type SQL } from "drizzle-orm"; import type { Db } from "@paperclipai/db"; import { agents, @@ -58,7 +59,7 @@ import { BLOCKER_ATTENTION_MAX_NODES, issueService, } from "./issues.js"; -import { executionIssueCondition, visibleIssueCondition } from "./issue-visibility.js"; +import { attentionIssueCondition, executionIssueCondition } from "./issue-visibility.js"; import { parseIssueExecutionState } from "./issue-execution-policy.js"; import { isProspectiveBlockedTransition } from "./routable-blocked.js"; import { issueReadSqlCondition, type AuthorizationActor } from "./authorization.js"; @@ -106,8 +107,6 @@ const SOURCE_RANK: Record = { }; const PENDING_INTERACTION_STATUSES = ["pending"] as const; -const OPEN_RECOVERY_STATUSES = ["active", "escalated"] as const; -const HUMAN_RECOVERY_OWNER_TYPES = ["user", "board"] as const; const DETAIL_EXCERPT_LENGTH = 160; const DETAIL_IMAGE_LIMIT = 3; const OPEN_DECISION_DEFAULT_LIMIT = 500; @@ -766,37 +765,6 @@ export function interactionResolverAudience( }; } -function collapsePendingConfirmationsToNewest(rows: T[]) { - // Quality gates bind independent outputs; a newer visual approval must not - // hide a still-pending copy approval (or affect its badge count). - const independent = (row: T) => Boolean(row.payload && typeof row.payload === "object" && "qualityReview" in row.payload && row.payload.qualityReview); - const newestByGroup = new Map(); - for (const row of rows) { - if (row.kind !== "request_confirmation" || independent(row)) continue; - const groupKey = `${row.issueId}:${row.kind}`; - const newest = newestByGroup.get(groupKey); - if ( - !newest - || row.createdAt.getTime() > newest.createdAt.getTime() - || (row.createdAt.getTime() === newest.createdAt.getTime() && row.id > newest.id) - ) { - newestByGroup.set(groupKey, row); - } - } - - return rows.filter((row) => ( - row.kind !== "request_confirmation" - || independent(row) - || newestByGroup.get(`${row.issueId}:${row.kind}`)?.id === row.id - )); -} - function budgetObservedPercent(amountObserved: number, amountLimit: number) { return amountLimit > 0 ? Math.round((amountObserved / amountLimit) * 10_000) / 100 : 0; } @@ -866,9 +834,7 @@ async function issueSummaryMap( .where(and( eq(issues.companyId, companyId), inArray(issues.id, ids), - includeConversations - ? or(executionIssueCondition(), and(visibleIssueCondition(), isNotNull(issues.conversationAgentId))) - : executionIssueCondition(), + attentionIssueCondition(includeConversations), readCondition, )); return new Map(rows.map((row) => [row.id, { @@ -1108,70 +1074,6 @@ function resolveBlockingIssue( return null; } -function readRunIssueId(contextSnapshot: Record | null) { - const issueId = contextSnapshot?.issueId ?? contextSnapshot?.taskId; - return typeof issueId === "string" && issueId.length > 0 ? issueId : null; -} - -type FailedRunProbe = { id: string; agentId: string; issueId: string | null; createdAt: Date }; - -const FAILED_RUN_PROBE_BATCH_SIZE = 1000; - -async function listFailedRunsWithNewerRun(db: Db, companyId: string, failedRuns: FailedRunProbe[]) { - const withNewerRun = new Set(); - const keyed = failedRuns.filter(run => run.issueId !== null); - for (let start = 0; start < keyed.length; start += FAILED_RUN_PROBE_BATCH_SIZE) { - const batch = keyed.slice(start, start + FAILED_RUN_PROBE_BATCH_SIZE); - for (const id of await listFailedRunBatchWithNewerRun(db, companyId, batch)) withNewerRun.add(id); - } - const unkeyed = failedRuns.filter(run => run.issueId === null); - if (unkeyed.length === 0) return withNewerRun; - const agentIds = [...new Set(unkeyed.map(run => run.agentId))]; - const oldestCreatedAt = unkeyed.reduce((oldest, run) => run.createdAt < oldest ? run.createdAt : oldest, - unkeyed[0]!.createdAt); - const latestByAgent = new Map(); - for (let start = 0; start < agentIds.length; start += FAILED_RUN_PROBE_BATCH_SIZE) { - const rows = await db.select({ - agentId: heartbeatRuns.agentId, - createdAt: sql`max(${heartbeatRuns.createdAt})`.mapWith(heartbeatRuns.createdAt), - }).from(heartbeatRuns).where(and( - eq(heartbeatRuns.companyId, companyId), - inArray(heartbeatRuns.agentId, agentIds.slice(start, start + FAILED_RUN_PROBE_BATCH_SIZE)), - gt(heartbeatRuns.createdAt, oldestCreatedAt), - sql`coalesce(${heartbeatRuns.contextSnapshot} ->> 'issueId', ${heartbeatRuns.contextSnapshot} ->> 'taskId', '') = ''`, - )).groupBy(heartbeatRuns.agentId); - for (const row of rows) latestByAgent.set(row.agentId, row.createdAt); - } - for (const run of unkeyed) { - if ((latestByAgent.get(run.agentId)?.getTime() ?? 0) > run.createdAt.getTime()) withNewerRun.add(run.id); - } - return withNewerRun; -} - -async function listFailedRunBatchWithNewerRun(db: Db, companyId: string, failedRuns: FailedRunProbe[]) { - if (failedRuns.length === 0) return []; - const failed = sql.join( - failedRuns.map((run) => sql`(${run.id}::uuid, ${run.agentId}::uuid, ${run.issueId}::text, ${new Date(run.createdAt.getTime() + 1).toISOString()}::timestamptz)`), - sql`, `, - ); - const runIssueId = sql`(${heartbeatRuns.contextSnapshot} ->> 'issueId')`; - const runTaskId = sql`(${heartbeatRuns.contextSnapshot} ->> 'taskId')`; - const newerRunExists = (match: SQL) => sql`exists ( - select 1 from ${heartbeatRuns} - where ${heartbeatRuns.companyId} = ${companyId} - and ${heartbeatRuns.agentId} = failed.agent_id - and ${heartbeatRuns.createdAt} >= failed.newer_from - and ${match} - )`; - const rows = await db.execute(sql` - select failed.id - from (values ${failed}) as failed (id, agent_id, issue_id, newer_from) - where ${newerRunExists(sql`${runIssueId} = failed.issue_id`)} - or ${newerRunExists(sql`${runTaskId} = failed.issue_id and ${runIssueId} is null`)} - `) as unknown as Array<{ id: string }>; - return rows.map((row) => row.id); -} - export function attentionService(db: Db, serviceOptions: AttentionServiceOptions = {}) { const openDecisionLimit = Math.min( Math.max(Math.trunc(serviceOptions.openDecisionLimit ?? OPEN_DECISION_DEFAULT_LIMIT), 1), @@ -1510,8 +1412,7 @@ export function attentionService(db: Db, serviceOptions: AttentionServiceOptions .from(issueRecoveryActions) .where(and( eq(issueRecoveryActions.companyId, companyId), - inArray(issueRecoveryActions.status, [...OPEN_RECOVERY_STATUSES]), - inArray(issueRecoveryActions.ownerType, [...HUMAN_RECOVERY_OWNER_TYPES]), + recoveryActionAttentionCondition(), )) .orderBy(desc(issueRecoveryActions.updatedAt), desc(issueRecoveryActions.id)); const [recoveryIssueMap, recoveryImageMap] = await Promise.all([ @@ -1638,8 +1539,7 @@ export function attentionService(db: Db, serviceOptions: AttentionServiceOptions })); } const blockerAttention = issue.blockerAttention; - if (blockerAttention?.state !== "stalled" && blockerAttention?.state !== "needs_attention") continue; - if (blockerAttention.blockingTreeLive) continue; + if (!terminalBlockerRequiresAttention(blockerAttention)) continue; const issueSummary = blockedIssueSummaries.get(issue.id) ?? null; const terminalIssueId = blockerAttention.terminalBlockerIssueId ?? issue.id; const terminalSummary = terminalBlockerSummaries.get(terminalIssueId) @@ -1741,7 +1641,8 @@ export function attentionService(db: Db, serviceOptions: AttentionServiceOptions const pendingApprovalId = pendingApprovalByIssueId.get(review.id) ?? null; const reviewAttention = reviewAttentionByIssueId.get(review.id); const stalled = reviewAttention?.state === "stalled"; - if (!hasHumanParticipant && !review.assigneeUserId && !pendingApprovalId && !stalled) continue; + if (!requiresReviewAttention({ hasHumanParticipant, hasUserAssignee: Boolean(review.assigneeUserId), + hasPendingApproval: Boolean(pendingApprovalId), stalled })) continue; const issue = reviewIssueMap.get(review.id); if (!issue) continue; const dedupKey = `review:${review.id}`; @@ -1860,7 +1761,7 @@ export function attentionService(db: Db, serviceOptions: AttentionServiceOptions const budgetOverview = await budgetService(db).overview(companyId); for (const incident of budgetOverview.activeIncidents) { const observedPercent = budgetObservedPercent(incident.amountObserved, incident.amountLimit); - if (incident.thresholdType !== "hard" && observedPercent < 85) continue; + if (incident.thresholdType !== "hard" && observedPercent < ATTENTION_BUDGET_WARNING_PERCENT) continue; const dedupKey = `budget:${incident.policyId}:${toIso(incident.windowStart)}:${incident.thresholdType}`; add(createItem({ companyId, @@ -1923,7 +1824,7 @@ export function attentionService(db: Db, serviceOptions: AttentionServiceOptions updatedAt: agents.updatedAt, }) .from(agents) - .where(and(eq(agents.companyId, companyId), eq(agents.status, "error"))) + .where(and(eq(agents.companyId, companyId), eq(agents.status, ATTENTION_AGENT_ERROR_STATUS))) .orderBy(desc(agents.updatedAt), desc(agents.id)); for (const agent of erroredAgents) { diff --git a/server/src/services/budgets.ts b/server/src/services/budgets.ts index b3b3989a049..1b31bcf6994 100644 --- a/server/src/services/budgets.ts +++ b/server/src/services/budgets.ts @@ -2,7 +2,7 @@ import { notifyStatusCardIssueChange } from "./status-card-notifications.js"; import { notifyDeferredExecutionEligibility } from "./deferred-execution-notifications.js"; import { notifyRunDispatch } from "./run-dispatch-notifications.js"; import { notifyApprovalWork } from "./approval-work-notifications.js"; -import { notifyCompanyRetentionWork } from "./retention-work-notifications.js"; +import { notifyAttentionRetentionWork, notifyCompanyRetentionWork } from "./retention-work-notifications.js"; import { addCents, compareCents, normalizeCents } from "@paperclipai/shared"; import { and, desc, eq, inArray, sql, or, isNull } from "drizzle-orm"; import type { Db } from "@paperclipai/db"; @@ -497,6 +497,7 @@ export function budgetServiceInTransaction(db: Db, publications: ActivityPublica windowEnd: end, }); + await notifyAttentionRetentionWork(db, { companyId: policy.companyId, sourceKind: "budget_alert" }); if (thresholdType === "hard") await notifyApprovalWork(db, policy.companyId); const approval = thresholdType === "hard" ? await db @@ -535,7 +536,8 @@ export function budgetServiceInTransaction(db: Db, publications: ActivityPublica return incident ? { incident, created: true } : null; } - async function resolveOpenSoftIncidents(policyId: string) { + async function resolveOpenSoftIncidents(policyId: string, companyId: string) { + await notifyAttentionRetentionWork(db, { companyId, sourceKind: "budget_alert" }); await db .update(budgetIncidents) .set({ @@ -554,10 +556,12 @@ export function budgetServiceInTransaction(db: Db, publications: ActivityPublica async function resolveOpenIncidentsForPolicy( policyId: string, + companyId: string, approvalStatus: "approved" | "rejected" | null, decidedByUserId: string | null, thresholdType?: BudgetThresholdType, ) { + await notifyAttentionRetentionWork(db, { companyId, sourceKind: "budget_alert" }); const condition = and(eq(budgetIncidents.policyId, policyId), eq(budgetIncidents.status, "open"), thresholdType ? eq(budgetIncidents.thresholdType, thresholdType) : undefined); const openRows = await db @@ -646,9 +650,9 @@ export function budgetServiceInTransaction(db: Db, publications: ActivityPublica if (policy.isActive && policy.hardStopEnabled && policy.amount > 0 && observed.recoveringRunCount > 0) recoveringPolicy ??= policy; // A closed UTC window, inactive policy, or budget raise releases only // budget-owned pauses. Manual pauses remain operator decisions. - await resolveOpenIncidentsForPolicy(policy.id, "approved", "budget_service", "hard"); + await resolveOpenIncidentsForPolicy(policy.id, policy.companyId, "approved", "budget_service", "hard"); if (!policy.isActive || !policy.notifyEnabled || compareCents(observed.totalExact, Math.ceil(policy.amount * policy.warnPercent / 100)) < 0) { - await resolveOpenIncidentsForPolicy(policy.id, "approved", "budget_service", "soft"); + await resolveOpenIncidentsForPolicy(policy.id, policy.companyId, "approved", "budget_service", "soft"); } } // Agent status has its own admission gate and error code. Preserve a @@ -705,7 +709,7 @@ export function budgetServiceInTransaction(db: Db, publications: ActivityPublica } if (observedBlocks(policy, observed)) { - await resolveOpenSoftIncidents(policy.id); + await resolveOpenSoftIncidents(policy.id, policy.companyId); const hardIncident = await createIncidentIfNeeded(policy, "hard", observed); await pauseAndCancelScopeForBudget(policy); if (hardIncident?.created) { @@ -875,11 +879,11 @@ export function budgetServiceInTransaction(db: Db, publications: ActivityPublica const observed = await computeObservedSpend(db, row); if (!observedBlocks(row, observed)) { await resumeScopeFromBudget(row); - await resolveOpenIncidentsForPolicy(row.id, "approved", actorUserId ?? "budget_service", "hard"); + await resolveOpenIncidentsForPolicy(row.id, row.companyId, "approved", actorUserId ?? "budget_service", "hard"); if (row.notifyEnabled && compareCents(observed.totalExact, Math.ceil(row.amount * row.warnPercent / 100)) >= 0) { await createIncidentIfNeeded(row, "soft", observed); } else { - await resolveOpenIncidentsForPolicy(row.id, null, null, "soft"); + await resolveOpenIncidentsForPolicy(row.id, row.companyId, null, null, "soft"); } } else { const softThreshold = Math.ceil((row.amount * row.warnPercent) / 100); @@ -887,14 +891,14 @@ export function budgetServiceInTransaction(db: Db, publications: ActivityPublica await createIncidentIfNeeded(row, "soft", observed); } if (observedBlocks(row, observed)) { - await resolveOpenSoftIncidents(row.id); + await resolveOpenSoftIncidents(row.id, row.companyId); await createIncidentIfNeeded(row, "hard", observed); await pauseAndCancelScopeForBudget(row); } } } else { await resumeScopeFromBudget(row); - await resolveOpenIncidentsForPolicy(row.id, "approved", actorUserId ?? "budget_service"); + await resolveOpenIncidentsForPolicy(row.id, row.companyId, "approved", actorUserId ?? "budget_service"); } await recordActivity({ @@ -975,6 +979,7 @@ export function budgetServiceInTransaction(db: Db, publications: ActivityPublica } const now = new Date(); + await notifyAttentionRetentionWork(db, { companyId, sourceKind: "budget_alert" }); await notifyDeliveryWork(db, DELIVERY_QUEUES.budgetEnforcement, companyId); await db .update(budgetPolicies) diff --git a/server/src/services/company-portability.ts b/server/src/services/company-portability.ts index 585b8fcf5f4..0b41482b0fa 100644 --- a/server/src/services/company-portability.ts +++ b/server/src/services/company-portability.ts @@ -1,3 +1,4 @@ +import { notifyAttentionRetentionWork } from "./retention-work-notifications.js"; import { notifyIssueRecovery } from "./issue-recovery-notifications.js"; import { createAgentLifecycle } from "./agent-lifecycle.js"; import { agentAppearanceSchema, agentHarnessType, paperclipRunnerProfileForHarness, agentRunner, type AgentRunnerChoice } from "@paperclipai/shared"; @@ -6467,6 +6468,7 @@ export function companyPortabilityService(db: Db, storage?: StorageService) { if (relationRows.length > 0) { const relationCompanyId = targetCompany.id; await db.transaction(async tx => { + await notifyAttentionRetentionWork(tx, { companyId: relationCompanyId }); await notifyIssueRecovery(tx, { companyId: relationCompanyId }); await tx .insert(issueRelations) diff --git a/server/src/services/decision-expiry.ts b/server/src/services/decision-expiry.ts index ea5236a6ac0..796200128a7 100644 --- a/server/src/services/decision-expiry.ts +++ b/server/src/services/decision-expiry.ts @@ -23,7 +23,7 @@ export function decisionExpiryService(db: Db, deliverContinuation: (id: string) agentId: row.originAgentId, runId: row.originRunId, action: "decision.expired", entityType: "decision", entityId: row.id, details: { expiredReason: reason } }, publications); return row; - }); + }, { companyId: decision.companyId, sourceId: decision.id }); if (!updated) return 0; for (const publication of publications) publishActivity(publication); await deliverContinuation(updated.id); diff --git a/server/src/services/decision-retention.ts b/server/src/services/decision-retention.ts index fb22df2a583..4ff2b094416 100644 --- a/server/src/services/decision-retention.ts +++ b/server/src/services/decision-retention.ts @@ -1,3 +1,8 @@ +import { isProspectiveBlockedTransition } from "./routable-blocked.js"; +import { recoveryActionAttentionCondition, requiresReviewAttention, terminalBlockerRequiresAttention, ATTENTION_AGENT_ERROR_STATUS } from "./attention-source-eligibility.js"; +import { attentionIssueCondition } from "./issue-visibility.js"; +import { issueService } from "./issues.js"; +import { parseIssueExecutionState } from "./issue-execution-policy.js"; import { notifyRetentionWork } from "./retention-work-notifications.js"; import { decisionArchiveDeliveryService, type ArchiveNotificationBatch } from "./decision-archive-delivery.js"; import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js"; @@ -15,7 +20,6 @@ import { issueRecoveryActions, issueThreadInteractions, issues, - joinRequests, } from "@paperclipai/db"; import type { AttentionArchiveManifestEntry, @@ -145,7 +149,7 @@ export function decisionRetentionService( const existing = new Set(rows.map(row => sourceKey(row.sourceKind, row.sourceId))); const missing = unique.filter(item => !existing.has(itemSourceKey(item))); if (missing.length) { - for (const kind of new Set(missing.map(item => item.sourceKind))) await notifyRetentionWork(tx, kind); + for (const item of missing) await notifyRetentionWork(tx, item.sourceKind, { companyId, sourceId: item.subject.id }); await tx.insert(decisionRetention).values(missing.map(item => ({ companyId, sourceKind: item.sourceKind, sourceId: item.subject.id, sourceActivityAt: new Date(item.activityAt), }))).onConflictDoNothing({ target: [decisionRetention.companyId, decisionRetention.sourceKind, decisionRetention.sourceId] }); @@ -155,7 +159,7 @@ export function decisionRetentionService( for (const item of unique) { const key = itemSourceKey(item), row = byKey.get(key), activityAt = new Date(item.activityAt); if (!row || row.sourceActivityAt.getTime() === activityAt.getTime()) continue; - await notifyRetentionWork(tx, item.sourceKind); + await notifyRetentionWork(tx, item.sourceKind, { companyId, sourceId: item.subject.id }); const [updated] = await tx.update(decisionRetention).set({ sourceActivityAt: activityAt, version: sql`${decisionRetention.version} + 1`, updatedAt: new Date(), }).where(and(eq(decisionRetention.id, row.id), ne(decisionRetention.sourceActivityAt, activityAt))).returning(); @@ -169,9 +173,9 @@ export function decisionRetentionService( return sourceKey(item.sourceKind, item.subject.id); } - async function writeRetention(kind: AttentionSourceKind, write: (tx: Db) => Promise) { + async function writeRetention(companyId: string, kind: AttentionSourceKind, sourceId: string, write: (tx: Db) => Promise) { return db.transaction(async tx => { - await notifyRetentionWork(tx, kind); + await notifyRetentionWork(tx, kind, { companyId, sourceId }); return write(tx as unknown as Db); }); } @@ -184,14 +188,87 @@ export function decisionRetentionService( )).then((rows) => rows[0] ?? null); } - async function materializeSourceState(database: Db, input: { companyId: string; sourceKind: AttentionSourceKind; sourceId: string }) { - const table = input.sourceKind === "approval" ? approvals : input.sourceKind === "join_request" ? joinRequests : null; - if (!table) return; - const [source] = await database.select({ updatedAt: table.updatedAt }).from(table) - .where(and(eq(table.companyId, input.companyId), eq(table.id, input.sourceId))).for("update"); + async function materializeSourceState(database: Db, input: { companyId: string; sourceKind: AttentionSourceKind; sourceId: string; authActor: AuthorizationActor; actor: Pick }) { + const tables: Record = { + approval: "approvals", join_request: "join_requests", decision: "decisions", + issue_thread_interaction: "issue_thread_interactions", recovery_action: "issue_recovery_actions", + failed_run: "heartbeat_runs", productivity_review: "issues", blocker_attention: "issues", review: "issues", + budget_alert: "budget_incidents", agent_error_alert: "agents", + }; + const existingState = () => database.select({ id: decisionRetention.id }).from(decisionRetention).where(and( + eq(decisionRetention.companyId, input.companyId), eq(decisionRetention.sourceKind, input.sourceKind), + eq(decisionRetention.sourceId, input.sourceId), + )).then(rows => rows[0]); + if (input.sourceKind === "recovery_action") { + const [source] = await database.select().from(issueRecoveryActions).where(and( + eq(issueRecoveryActions.companyId, input.companyId), eq(issueRecoveryActions.id, input.sourceId), + )).for("update"); + if (!source) throw notFound("Attention source not found"); + if (!await existingState()) { + const [eligible] = await database.select({ id: issueRecoveryActions.id }).from(issueRecoveryActions) + .innerJoin(issues, and(eq(issues.id, issueRecoveryActions.sourceIssueId), eq(issues.companyId, issueRecoveryActions.companyId))) + .where(and(eq(issueRecoveryActions.companyId, input.companyId), eq(issueRecoveryActions.id, source.id), + recoveryActionAttentionCondition(), attentionIssueCondition(false))); + if (!eligible || !await canReadDecisionSource(database, input.authActor, input.companyId, input.sourceKind, input.sourceId)) { + throw notFound("Attention source not found"); + } + } + await decisionRetentionService(database).syncItems(input.companyId, [{ + sourceKind: input.sourceKind, subject: { id: source.id }, activityAt: source.updatedAt.toISOString(), + }]); + return; + } + if (input.sourceKind === "agent_error_alert") { + const [source] = await database.select().from(agents).where(and(eq(agents.companyId, input.companyId), eq(agents.id, input.sourceId))).for("update"); + if (!source || (!await existingState() && source.status !== ATTENTION_AGENT_ERROR_STATUS)) throw notFound("Attention source not found"); + await decisionRetentionService(database).syncItems(input.companyId, [{ + sourceKind: input.sourceKind, subject: { id: input.sourceId }, activityAt: source.updatedAt.toISOString(), + }]); + return; + } + if (["review", "blocker_attention", "productivity_review"].includes(input.sourceKind)) { + const [source] = await database.select().from(issues).where(and(eq(issues.companyId, input.companyId), eq(issues.id, input.sourceId))).for("update"); + if (!source) throw notFound("Attention source not found"); + if (!await existingState()) { + const [visible] = await database.select({ id: issues.id }).from(issues).where(and( + eq(issues.companyId, input.companyId), eq(issues.id, source.id), attentionIssueCondition(false), + )); + let eligible = false; + if (visible && input.sourceKind === "review" && source.status === "in_review") { + const [pendingApproval] = await database.select({ id: approvals.id }).from(issueApprovals) + .innerJoin(approvals, eq(approvals.id, issueApprovals.approvalId)).where(and( + eq(issueApprovals.companyId, input.companyId), eq(approvals.companyId, input.companyId), + eq(issueApprovals.issueId, source.id), eq(approvals.status, "pending"), + )).limit(1); + const state = parseIssueExecutionState(source.executionState); + const attention = await issueService(database).listReviewAttention(input.companyId, [source]); + eligible = requiresReviewAttention({ + hasHumanParticipant: state?.status === "pending" && state.currentParticipant?.type === "user", + hasUserAssignee: Boolean(source.assigneeUserId), hasPendingApproval: Boolean(pendingApproval), + stalled: attention.get(source.id)?.state === "stalled", + }); + } else if (visible && input.sourceKind === "blocker_attention" && source.status === "blocked") { + const owner = source.unblockDescriptor?.owner; + const ownsDescriptor = owner === "board" || (owner && "userId" in owner && owner.userId === input.actor.userId); + const attention = (await issueService(database).listBlockerAttention(input.companyId, [source])).get(source.id); + eligible = Boolean(ownsDescriptor && isProspectiveBlockedTransition(source)) + || (terminalBlockerRequiresAttention(attention) && (attention.terminalBlockerIssueId ?? source.id) === source.id); + } + if (!eligible) throw notFound("Attention source not found"); + } + const activityAt = input.sourceKind === "blocker_attention" && source.unblockDescriptor?.owner === "board" && isProspectiveBlockedTransition(source) + ? source.blockedTransitionAt : source.updatedAt; + await decisionRetentionService(database).syncItems(input.companyId, [{ + sourceKind: input.sourceKind, subject: { id: input.sourceId }, activityAt: activityAt.toISOString(), + }]); + return; + } + const activity = input.sourceKind === "failed_run" ? sql`coalesce(finished_at, updated_at, created_at)` : sql`updated_at`; + const [source] = await database.execute(sql`select ${activity} as "activityAt" from ${sql.identifier(tables[input.sourceKind])} + where company_id = ${input.companyId}::uuid and id = ${input.sourceId}::uuid for update`) as unknown as Array<{ activityAt: Date | string }>; if (!source) throw notFound("Attention source not found"); await decisionRetentionService(database).syncItems(input.companyId, [{ - sourceKind: input.sourceKind, subject: { id: input.sourceId }, activityAt: source.updatedAt.toISOString(), + sourceKind: input.sourceKind, subject: { id: input.sourceId }, activityAt: new Date(source.activityAt).toISOString(), }]); } @@ -206,7 +283,7 @@ export function decisionRetentionService( if (!(await canReadDecisionSource(db, input.authActor, input.companyId, input.sourceKind, input.sourceId))) { throw notFound("Attention source not found"); } - const updated = await writeRetention(input.sourceKind, async tx => { + const updated = await writeRetention(input.companyId, input.sourceKind, input.sourceId, async tx => { await materializeSourceState(tx, input); return tx.update(decisionRetention).set({ keep: input.keep, @@ -247,7 +324,7 @@ export function decisionRetentionService( throw notFound("Attention source not found"); } const now = new Date(); - const updated = await writeRetention(input.sourceKind, async tx => { + const updated = await writeRetention(input.companyId, input.sourceKind, input.sourceId, async tx => { await materializeSourceState(tx, input); return tx.update(decisionRetention).set(input.archived ? { archivedAt: now, @@ -324,7 +401,7 @@ export function decisionRetentionService( } const now = new Date(); for (const entry of input.manifest) { - await notifyRetentionWork(txDb, entry.sourceKind as AttentionSourceKind); + await notifyRetentionWork(txDb, entry.sourceKind as AttentionSourceKind, { companyId: input.companyId, sourceId: entry.sourceId }); const archived = await txDb.update(decisionRetention).set({ archivedAt: now, archivedReason: "accepted_proposal", @@ -368,7 +445,7 @@ export function decisionRetentionService( await db.transaction(async (tx) => { const txDb = tx as unknown as Db; for (const item of candidates) { - await notifyRetentionWork(txDb, item.sourceKind); + await notifyRetentionWork(txDb, item.sourceKind, { companyId: input.companyId, sourceId: item.subject.id }); const updated = await txDb.update(decisionRetention).set({ archivedAt: now, archivedReason: "idle_ttl", diff --git a/server/src/services/decision-work-notifications.ts b/server/src/services/decision-work-notifications.ts index 081edc8fc39..9e2ca840960 100644 --- a/server/src/services/decision-work-notifications.ts +++ b/server/src/services/decision-work-notifications.ts @@ -1,18 +1,23 @@ +import { notifyAttentionRetentionWork } from "./retention-work-notifications.js"; import type { Db } from "@paperclipai/db"; import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js"; -export function notifyDecisionCreated(transaction: object) { +type DecisionWorkScope = { companyId: string; sourceId?: string }; + +export async function notifyDecisionCreated(transaction: object, scope: DecisionWorkScope) { + await notifyAttentionRetentionWork(transaction, { ...scope, sourceKind: "decision" }); return notifyDeliveryWork(transaction, DELIVERY_QUEUES.decisionExpiry); } -export async function notifyDecisionChange(transaction: object) { +export async function notifyDecisionChange(transaction: object, scope: DecisionWorkScope) { + await notifyAttentionRetentionWork(transaction, { ...scope, sourceKind: "decision" }); await notifyDeliveryWork(transaction, DELIVERY_QUEUES.decisionContinuation); await notifyDeliveryWork(transaction, DELIVERY_QUEUES.decisionExpiry); } -export function writeDecisionChange(db: Db, write: (transaction: Db) => Promise): Promise { +export function writeDecisionChange(db: Db, write: (transaction: Db) => Promise, scope: DecisionWorkScope): Promise { return db.transaction(async transaction => { - await notifyDecisionChange(transaction); + await notifyDecisionChange(transaction, scope); await notifyDeliveryWork(transaction, DELIVERY_QUEUES.decisionEffectRecovery); return write(transaction as unknown as Db); }); diff --git a/server/src/services/decisions.ts b/server/src/services/decisions.ts index 282327d433f..8a84743fcef 100644 --- a/server/src/services/decisions.ts +++ b/server/src/services/decisions.ts @@ -239,7 +239,7 @@ export function decisionService(db: Db, options: DecisionServiceOptions) { ...(input.additionalTargetSnapshots ?? {}), }; const id = randomUUID(); - await notifyDecisionCreated(dbOrTx); + await notifyDecisionCreated(dbOrTx, { companyId: input.companyId, sourceId: id }); await notifyDecisionTargetWork(dbOrTx, `decision:${id}`); const [created] = await dbOrTx.insert(decisions).values({ id, companyId: input.companyId, bundleId: input.bundleId ?? null, originAgentId: input.agentId, originIssueId: provenance.issueId, originRunId: input.runId, ruleKey: input.ruleKey ?? null, @@ -581,7 +581,8 @@ export function decisionService(db: Db, options: DecisionServiceOptions) { return attempt; } - async function settleInvalidEffectOutcome(id: string) { + async function settleInvalidEffectOutcome(decision: typeof decisions.$inferSelect) { + const id = decision.id; const publications: ActivityPublication[] = []; await writeDecisionChange(db, async tx => { const [current] = await tx.select().from(decisions).where(eq(decisions.id, id)).for("update"); @@ -601,14 +602,14 @@ export function decisionService(db: Db, options: DecisionServiceOptions) { action: "decision.recovery_failed", entityType: "decision", entityId: id, details: { reason, executionStatus, chosenOptionId: current.chosenOptionId, decidedByUserId: current.decidedByUserId }, }, publications); - }); + }, { companyId: decision.companyId, sourceId: id }); for (const publication of publications) publishActivity(publication); return outcome(id); } async function executeDecisionEffects(decision: typeof decisions.$inferSelect, userActor: AuthorizationActor) { const option = decision.options.find((item) => item.id === decision.chosenOptionId); - if (!option || !decision.decidedByUserId) return settleInvalidEffectOutcome(decision.id); + if (!option || !decision.decidedByUserId) return settleInvalidEffectOutcome(decision); const run = await db.select({ responsibleUserId: heartbeatRuns.responsibleUserId }).from(heartbeatRuns).where(eq(heartbeatRuns.id, decision.originRunId)).then((rows) => rows[0] ?? null); const metadata = decision.metadata as Record; if (metadata.kind === "attention_archive_proposal") { @@ -618,7 +619,7 @@ export function decisionService(db: Db, options: DecisionServiceOptions) { targetSnapshots: decision.targetSnapshots as Record, }), decision.signedSpec)) { await writeDecisionChange(db, async tx => tx.update(decisions).set({ executionStatus: "failed", updatedAt: new Date(), metadata: { ...metadata, archiveProposalError: "invalid_signature", ...(decision.continuationPolicy === "wake_origin_agent" ? { continuationPending: true } : {}) } }) - .where(and(eq(decisions.id, decision.id), eq(decisions.executionStatus, "running")))); + .where(and(eq(decisions.id, decision.id), eq(decisions.executionStatus, "running"))), { companyId: decision.companyId, sourceId: decision.id }); return outcome(decision.id); } if (option.id === "archive") { @@ -629,7 +630,7 @@ export function decisionService(db: Db, options: DecisionServiceOptions) { const manifestHash = manifest ? hashAttentionArchiveManifest(manifest) : null; if (!manifest || manifestHash !== metadata.manifestHash) { await writeDecisionChange(db, async tx => tx.update(decisions).set({ executionStatus: "failed", updatedAt: new Date(), metadata: { ...metadata, archiveProposalError: "manifest_mismatch", ...(decision.continuationPolicy === "wake_origin_agent" ? { continuationPending: true } : {}) } }) - .where(and(eq(decisions.id, decision.id), eq(decisions.executionStatus, "running")))); + .where(and(eq(decisions.id, decision.id), eq(decisions.executionStatus, "running"))), { companyId: decision.companyId, sourceId: decision.id }); return outcome(decision.id); } const originActor: AuthorizationActor = { @@ -651,7 +652,7 @@ export function decisionService(db: Db, options: DecisionServiceOptions) { } catch (error) { const message = error instanceof Error ? error.message : "archive_proposal_failed"; await writeDecisionChange(db, async tx => tx.update(decisions).set({ executionStatus: "failed", updatedAt: new Date(), metadata: { ...metadata, archiveProposalError: message, ...(decision.continuationPolicy === "wake_origin_agent" ? { continuationPending: true } : {}) } }) - .where(and(eq(decisions.id, decision.id), eq(decisions.executionStatus, "running")))); + .where(and(eq(decisions.id, decision.id), eq(decisions.executionStatus, "running"))), { companyId: decision.companyId, sourceId: decision.id }); return outcome(decision.id); } } @@ -659,7 +660,7 @@ export function decisionService(db: Db, options: DecisionServiceOptions) { executionStatus: "succeeded", updatedAt: new Date(), metadata: { ...metadata, ...(decision.continuationPolicy === "wake_origin_agent" ? { continuationPending: true } : {}) }, - }).where(and(eq(decisions.id, decision.id), eq(decisions.executionStatus, "running")))); + }).where(and(eq(decisions.id, decision.id), eq(decisions.executionStatus, "running"))), { companyId: decision.companyId, sourceId: decision.id }); return outcome(decision.id); } for (let index = 0; index < option.effects.length; index += 1) await executeEffect(decision, option.effects[index]!, index, userActor, decision.decidedByUserId, run?.responsibleUserId ?? null); @@ -667,7 +668,7 @@ export function decisionService(db: Db, options: DecisionServiceOptions) { const successful = rows.filter((row) => row.status === "executed").length; const status = rows.every((row) => row.status === "executed") ? "succeeded" : successful ? "partial" : "failed"; await writeDecisionChange(db, async tx => tx.update(decisions).set({ executionStatus: status, updatedAt: new Date(), metadata: { ...decision.metadata, - ...(decision.continuationPolicy === "wake_origin_agent" ? { continuationPending: true } : {}) } }).where(and(eq(decisions.id, decision.id), eq(decisions.executionStatus, "running")))); + ...(decision.continuationPolicy === "wake_origin_agent" ? { continuationPending: true } : {}) } }).where(and(eq(decisions.id, decision.id), eq(decisions.executionStatus, "running"))), { companyId: decision.companyId, sourceId: decision.id }); return outcome(decision.id); } @@ -706,7 +707,7 @@ export function decisionService(db: Db, options: DecisionServiceOptions) { decideIdempotencyKey: input.idempotencyKey ?? null, ...(current.continuationPolicy === "wake_origin_agent" ? { continuationPending: true } : {}), ...(input.dismissed ? { dismissed: true, dismissReason: input.dismissReason ?? null } : {}) } }) - .where(and(eq(decisions.id, current.id), eq(decisions.status, "open"))).returning()); + .where(and(eq(decisions.id, current.id), eq(decisions.status, "open"))).returning(), { companyId: current.companyId, sourceId: current.id }); if (!claimed) throw conflict("decision_already_resolved", { code: "decision_already_resolved" }); const run = await db.select({ responsibleUserId: heartbeatRuns.responsibleUserId }).from(heartbeatRuns).where(eq(heartbeatRuns.id, claimed.originRunId)).then((rows) => rows[0] ?? null); await logActivity(db, { companyId: claimed.companyId, actorType: "system", actorId: "decision-executor", agentId: claimed.originAgentId, runId: claimed.originRunId, @@ -724,7 +725,7 @@ export function decisionService(db: Db, options: DecisionServiceOptions) { return (await get(id))!; } const [updated] = await writeDecisionChange(db, async tx => tx.update(decisions).set({ status: "cancelled", updatedAt: new Date(), metadata: { ...current.metadata, - ...(current.continuationPolicy === "wake_origin_agent" ? { continuationPending: true } : {}) } }).where(and(eq(decisions.id, id), eq(decisions.status, "open"))).returning()); + ...(current.continuationPolicy === "wake_origin_agent" ? { continuationPending: true } : {}) } }).where(and(eq(decisions.id, id), eq(decisions.status, "open"))).returning(), { companyId: current.companyId, sourceId: id }); if (!updated) throw conflict("decision_already_resolved", { code: "decision_already_resolved" }); await logActivity(db, { companyId: updated.companyId, actorType: actor.actorType, actorId: actor.actorId, runId: actor.runId, action: "decision.cancelled", entityType: "decision", entityId: id }); await continuations.deliver(updated.id); @@ -738,7 +739,7 @@ export function decisionService(db: Db, options: DecisionServiceOptions) { if (empty) return decide({ id, optionId: empty.id, decidedByUserId: userId, userActor, dismissed: true, dismissReason: reason }); const [updated] = await writeDecisionChange(db, async tx => tx.update(decisions).set({ status: "decided", executionStatus: "succeeded", chosenOptionId: "dismissed", decidedByUserId: userId, decidedAt: new Date(), updatedAt: new Date(), metadata: { ...current.metadata, dismissed: true, dismissReason: reason ?? null, - ...(current.continuationPolicy === "wake_origin_agent" ? { continuationPending: true } : {}) } }).where(and(eq(decisions.id, id), eq(decisions.status, "open"))).returning()); + ...(current.continuationPolicy === "wake_origin_agent" ? { continuationPending: true } : {}) } }).where(and(eq(decisions.id, id), eq(decisions.status, "open"))).returning(), { companyId: current.companyId, sourceId: id }); if (!updated) throw conflict("decision_already_resolved", { code: "decision_already_resolved" }); await logActivity(db, { companyId: updated.companyId, actorType: "system", actorId: "decision-executor", agentId: updated.originAgentId, runId: updated.originRunId, responsibleUserIdOverride: userId, action: "decision.dismissed", entityType: "decision", entityId: updated.id, diff --git a/server/src/services/delivery-work-notifications.ts b/server/src/services/delivery-work-notifications.ts index 598fa555708..db41058a201 100644 --- a/server/src/services/delivery-work-notifications.ts +++ b/server/src/services/delivery-work-notifications.ts @@ -47,6 +47,7 @@ export const DELIVERY_QUEUES = { executionDisposition: "execution-disposition", environmentCleanup: "environment-cleanup", approvalQueueSeeding: "approval-queue-seeding", + attentionRetention: "attention-retention", approvalRetention: "approval-retention", joinRequestRetention: "join-request-retention", decisionTargets: "decision-targets", diff --git a/server/src/services/execution-control-notifications.ts b/server/src/services/execution-control-notifications.ts index b00582c0f92..fa00196bf41 100644 --- a/server/src/services/execution-control-notifications.ts +++ b/server/src/services/execution-control-notifications.ts @@ -1,3 +1,4 @@ +import { notifyAttentionRetentionWork } from "./retention-work-notifications.js"; import { notifyStatusCardIssueChange } from "./status-card-notifications.js"; import { notifyDeferredExecutionEligibility } from "./deferred-execution-notifications.js"; import { notifyRunDispatch } from "./run-dispatch-notifications.js"; @@ -8,6 +9,7 @@ export type ExecutionWorkScope = { companyId: string; agentId?: string; runId?: export async function notifyExecutionRunWork(transaction: object, scope?: ExecutionWorkScope) { if (scope) { + await notifyAttentionRetentionWork(transaction, scope); await notifyStatusCardIssueChange(transaction, scope.companyId); await notifyRunDispatch(transaction, scope); await notifyDeferredExecutionEligibility(transaction, scope); @@ -25,6 +27,7 @@ export function writeExecutionControl( scope: ExecutionWorkScope | undefined, ): Promise { return db.transaction(async transaction => { + if (scope) await notifyAttentionRetentionWork(transaction, scope); if (scope) await notifyStatusCardIssueChange(transaction, scope.companyId); await notifyRunDispatch(transaction, scope); if (scope) await notifyDeferredExecutionEligibility(transaction, scope); diff --git a/server/src/services/grok-bot/workspace-inspection.ts b/server/src/services/grok-bot/workspace-inspection.ts index f1db9a4a851..73cad27c8aa 100644 --- a/server/src/services/grok-bot/workspace-inspection.ts +++ b/server/src/services/grok-bot/workspace-inspection.ts @@ -1,3 +1,4 @@ +import { notifyAttentionRetentionWork } from "../retention-work-notifications.js"; import { and, eq, inArray, isNotNull, or, sql } from "drizzle-orm"; import { dotRunnerAssignments, environmentLeases, heartbeatRuns, issueRecoveryActions, issues, nativeRunFinalizations, type Db } from "@paperclipai/db"; import { authorizeExternalBotOperator } from "../external-bot-authority.js"; @@ -74,6 +75,7 @@ export async function inspectGrokWorkspaceObligations(db: Db, companyId: string, return { ...rest, externalWorkspaceInspection: inspection }; }; await tx.update(heartbeatRuns).set({ resultJson: retireFileObligation(run.resultJson!) }).where(eq(heartbeatRuns.id, run.id)); + if (actions.length) await notifyAttentionRetentionWork(tx, { companyId, sourceKind: "recovery_action" }); for (const action of actions) { await tx.update(issueRecoveryActions).set({ evidence: retireFileObligation(action.evidence), updatedAt: new Date(), nextAction: "Inspect Grok and verify its connection before sending a new task message. The old work will not be replayed.", diff --git a/server/src/services/heartbeat-run-events.ts b/server/src/services/heartbeat-run-events.ts index ebce78a8266..ced43f5376f 100644 --- a/server/src/services/heartbeat-run-events.ts +++ b/server/src/services/heartbeat-run-events.ts @@ -1,3 +1,4 @@ +import { notifyAttentionRetentionWork } from "./retention-work-notifications.js"; import { readNativeEventEvidenceOutcome } from "./native-runtime/evidence-classifier.js"; import { notifyNativeStatusReassessment } from "./native-status-reassessment-notifications.js"; import { isNativeResponseEvidence, notifyNativeResponsePresentation } from "./native-response-presentation-notifications.js"; @@ -9,6 +10,8 @@ import type { Db } from "@paperclipai/db"; import { heartbeatRunEvents, heartbeatRuns } from "@paperclipai/db"; import { nativeSha256 } from "./native-runtime/canonical.js"; +export const RETRY_EXHAUSTION_MESSAGE_PREFIX = "Bounded retry exhausted"; + export interface AppendHeartbeatRunEventInput { companyId: string; runId: string; @@ -74,10 +77,11 @@ export async function appendHeartbeatRunEvent( db: Db, input: AppendHeartbeatRunEventInput, ): Promise { + const projectsRetention = input.eventType === "lifecycle" && Boolean(input.message?.startsWith(RETRY_EXHAUSTION_MESSAGE_PREFIX)); const projectsReassessment = readNativeEventEvidenceOutcome(input.payload?.prpEvent) !== null; const projectsProgress = isSafeNativeChatProgressEventType(input.eventType); const projectsResponse = isNativeResponseEvidence(input.eventType, input.payload?.prpEvent); - if (!input.nativeSource && !input.retryExhaustion && !projectsProgress && !projectsResponse && !projectsReassessment) { + if (!input.nativeSource && !input.retryExhaustion && !projectsProgress && !projectsResponse && !projectsReassessment && !projectsRetention) { const allocated = db.$with("allocated_run_event", {}).as(sql` update ${heartbeatRuns} set next_event_seq = ${heartbeatRuns.nextEventSeq} + 1, @@ -129,7 +133,7 @@ export async function appendHeartbeatRunEvent( eq(heartbeatRunEvents.runId, input.runId), eq(heartbeatRunEvents.agentId, input.agentId), eq(heartbeatRunEvents.eventType, "lifecycle"), - sql`${heartbeatRunEvents.message} like 'Bounded retry exhausted%'`, + sql`${heartbeatRunEvents.message} like ${`${RETRY_EXHAUSTION_MESSAGE_PREFIX}%`}`, sql`${heartbeatRunEvents.payload} @> ${JSON.stringify(input.retryExhaustion)}::jsonb`, )) .limit(1) @@ -187,6 +191,7 @@ export async function appendHeartbeatRunEvent( if (projectsResponse && run.runtimeMode === "native") { await notifyNativeResponsePresentation(tx, { companyId: run.companyId, runId: input.runId }); } + if (projectsRetention) await notifyAttentionRetentionWork(tx, { companyId: input.companyId, sourceKind: "failed_run" }); const seq = await allocateHeartbeatRunEventSeq( tx as unknown as Db, input.runId, diff --git a/server/src/services/heartbeat/agent-status.ts b/server/src/services/heartbeat/agent-status.ts index f034cd06594..23cef4b2f87 100644 --- a/server/src/services/heartbeat/agent-status.ts +++ b/server/src/services/heartbeat/agent-status.ts @@ -1,3 +1,4 @@ +import { notifyAttentionRetentionWork } from "../retention-work-notifications.js"; import { DELIVERY_QUEUES, notifyDeliveryWork } from "../delivery-work-notifications.js"; import { agents, heartbeatRuns, type Db } from "@paperclipai/db"; import { and, asc, desc, eq, lte, sql } from "drizzle-orm"; @@ -91,6 +92,7 @@ export function createHeartbeatAgentStatus(db: Db) { )).limit(1); const nextStatus = running ? "running" : settledStatus(outcome, options?.keepIdleOnFailure); const now = new Date(); + await notifyAttentionRetentionWork(tx, { companyId: existing.companyId }); await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers); const [updated] = await tx.update(agents).set({ status: nextStatus, @@ -140,6 +142,7 @@ export function createHeartbeatAgentStatus(db: Db) { eq(heartbeatRuns.agentId, candidate.id))) .orderBy(desc(heartbeatRuns.finishedAt), desc(heartbeatRuns.createdAt), desc(heartbeatRuns.id)).limit(1); const nextStatus = settledStatus(lastRun?.status); + await notifyAttentionRetentionWork(tx, { companyId: candidate.companyId }); const [agent] = await tx.update(agents).set({ status: nextStatus, errorReason: nextStatus === "error" ? errorReason(lastRun?.error) : null, updatedAt: now, }).where(and(eq(agents.id, candidate.id), eq(agents.companyId, candidate.companyId), diff --git a/server/src/services/heartbeat/retries.ts b/server/src/services/heartbeat/retries.ts index cbf6054710a..f09aad46dec 100644 --- a/server/src/services/heartbeat/retries.ts +++ b/server/src/services/heartbeat/retries.ts @@ -1,3 +1,4 @@ +import { notifyAttentionRetentionWork } from "../retention-work-notifications.js"; import { notifyStatusCardIssueChange } from "../status-card-notifications.js"; import { notifyScheduledRetryDispatch } from "../run-dispatch-notifications.js"; import { deriveTaskKeyWithHeartbeatFallback } from "./run-state.js"; @@ -999,6 +1000,7 @@ export function createHeartbeatRetries(db: Db, dependencies: HeartbeatRetryDepen retryContextSnapshot.previousRunId = continuation.previousRunId; } + await notifyAttentionRetentionWork(tx, { companyId: run.companyId }); await notifyScheduledRetryDispatch(tx, { companyId: run.companyId }); const wakeupRequest = await tx .insert(agentWakeupRequests) diff --git a/server/src/services/issue-recovery-actions.ts b/server/src/services/issue-recovery-actions.ts index 316eb413afe..e276dc88f89 100644 --- a/server/src/services/issue-recovery-actions.ts +++ b/server/src/services/issue-recovery-actions.ts @@ -1,3 +1,4 @@ +import { notifyAttentionRetentionWork } from "./retention-work-notifications.js"; import { notifyIssueRecovery } from "./issue-recovery-notifications.js"; import { notifyNativeRecoveryNotices } from "./native-recovery-notice-notifications.js"; import { notifyUnsafeWorkspaceExportRecovery } from "./unsafe-workspace-export-notifications.js"; @@ -323,6 +324,7 @@ export function issueRecoveryActionService(db: DbOrTransaction) { ): Promise { try { const created = await db.transaction(async (tx) => { + await notifyAttentionRetentionWork(tx, { companyId: input.companyId }); await notifyIssueRecovery(tx, { companyId: input.companyId }); await notifyNativeRecoveryNotices(tx, { companyId: input.companyId, issueId: input.sourceIssueId }); await notifyDeliveryWork(tx, DELIVERY_QUEUES.executionDisposition); diff --git a/server/src/services/issue-thread-interactions.test.ts b/server/src/services/issue-thread-interactions.test.ts index 24e1489006f..5ad905a8a10 100644 --- a/server/src/services/issue-thread-interactions.test.ts +++ b/server/src/services/issue-thread-interactions.test.ts @@ -1,5 +1,6 @@ import { beforeEach, describe, expect, it, vi } from "vitest"; -import { getTableName } from "drizzle-orm"; +import { PgDialect } from "drizzle-orm/pg-core"; +import { getTableName, type SQL } from "drizzle-orm"; const mockCreateChild = vi.fn(); @@ -48,8 +49,18 @@ function createFakeDb(args: { const toolActionRequestUpdates: Array> = []; const inserts: Array<{ table: string; values: Record }> = []; let selectCallCount = 0; + let transactionDepth = 0; const db: any = { + execute: vi.fn(async (query: SQL) => { + const statement = new PgDialect().sqlToQuery(query); + expect(transactionDepth).toBeGreaterThan(0); + expect(statement.params).toContain(interactionRow.companyId); + expect(statement.params).toContain(interactionRow.issueId); + if (statement.sql.includes("origin_kind")) return [{ id: interactionRow.issueId, originKind: null, originId: null }]; + if (statement.sql.includes("with recursive affected")) return [{ issueId: interactionRow.issueId }]; + throw new Error(`Unexpected interaction fixture query: ${statement.sql}`); + }), select: vi.fn(() => { selectCallCount += 1; const rows = selectCallCount === 1 ? [interactionRow] : (args.parentRows ?? []); @@ -84,7 +95,10 @@ function createFakeDb(args: { inserts.push({ table: getTableName(table as never), values }); }, })), - transaction: async (callback: (tx: typeof db) => Promise) => callback(db), + transaction: async (callback: (tx: typeof db) => Promise) => { + transactionDepth++; + try { return await callback(db); } finally { transactionDepth--; } + }, }; return { @@ -177,6 +191,7 @@ describe("issueThreadInteractionService", () => { }; let selectCallCount = 0; + let transactionDepth = 0; const db: any = { select: vi.fn(() => { selectCallCount += 1; diff --git a/server/src/services/issue-title.ts b/server/src/services/issue-title.ts index 51e360269d7..4894a7b070e 100644 --- a/server/src/services/issue-title.ts +++ b/server/src/services/issue-title.ts @@ -1,3 +1,4 @@ +import { notifyAttentionRetentionWork } from "./retention-work-notifications.js"; import { notifyStatusCardIssueChange } from "./status-card-notifications.js"; import { and, eq } from "drizzle-orm"; import { heartbeatRuns, issues, type Db } from "@paperclipai/db"; @@ -77,6 +78,7 @@ export async function setIssueTitle( || (issue.title === title && !issue.titleNeedsGeneration)) { return { result: await saveReceipt({ id: issue.id, title: issue.title, titleNeedsGeneration: issue.titleNeedsGeneration, changed: false }), publication: null }; } + await notifyAttentionRetentionWork(tx, { companyId }); await notifyStatusCardIssueChange(tx, companyId); await tx.update(issues).set({ title, titleNeedsGeneration: false, updatedAt: new Date() }).where(predicate); await issueReferenceService(tx).syncIssue(issueId, tx); diff --git a/server/src/services/issue-visibility.ts b/server/src/services/issue-visibility.ts index 44a04a089e6..b54832f29ee 100644 --- a/server/src/services/issue-visibility.ts +++ b/server/src/services/issue-visibility.ts @@ -1,5 +1,5 @@ import { nonIdleSlackIssueCondition } from "./slack-conversation-state.js"; -import { and, isNull, type SQL } from "drizzle-orm"; +import { and, isNull, isNotNull, or, type SQL } from "drizzle-orm"; import { issues } from "@paperclipai/db"; export function visibleIssueCondition(): SQL { @@ -14,3 +14,9 @@ export function visibleIssueSql(alias = "issues") { export function executionIssueCondition(): SQL { return and(visibleIssueCondition(), isNull(issues.conversationAgentId), nonIdleSlackIssueCondition())!; } + +export function attentionIssueCondition(includeConversations = false): SQL { + return includeConversations + ? or(executionIssueCondition(), and(visibleIssueCondition(), isNotNull(issues.conversationAgentId)))! + : executionIssueCondition(); +} diff --git a/server/src/services/issues.ts b/server/src/services/issues.ts index da0fb9a2a98..d44926cfad5 100644 --- a/server/src/services/issues.ts +++ b/server/src/services/issues.ts @@ -1,3 +1,4 @@ +import { notifyIssueAttentionRetentionWork } from "./retention-work-notifications.js"; import { notifyStatusCardIssueChange } from "./status-card-notifications.js"; import { notifyDeferredExecutionEligibility } from "./deferred-execution-notifications.js"; import { notifyIssueRecovery } from "./issue-recovery-notifications.js"; @@ -2518,120 +2519,87 @@ export async function listUnfinalizedExecutionWorkspaceIds( return unfinalized; } +/** Workspace-wide facts are shared by blockers, rather than rereading retained + * operation history for every issue pointing at the same workspace. The ordered + * OFFSET boundary keeps the rare finalize filter from turning into a company + * history scan; the existing workspace/started-at index can stream candidates. */ +function blockerWorkspaceFacts(companyId: string, workspaceScope: SQL) { + return sql`( + with candidate_workspaces as materialized ( + select distinct workspace_id from (${workspaceScope}) scoped_workspaces(workspace_id) + ), workspace_facts as materialized ( + select candidate.workspace_id, finalized.started_at as finalized_at, + legacy.phase, legacy.status, legacy.started_at + from candidate_workspaces candidate + left join lateral ( + select started_at from ( + select phase, status, started_at from workspace_operations + where company_id = ${companyId}::uuid + and execution_workspace_id = candidate.workspace_id + order by started_at desc offset 0 + ) workspace_history + where phase = 'workspace_finalize' and status = 'succeeded' + order by started_at desc limit 1 + ) finalized on true + left join lateral ( + select phase, status, started_at from workspace_operations + where company_id = ${companyId}::uuid + and execution_workspace_id = candidate.workspace_id and issue_id is null + order by started_at desc limit 1 + ) legacy on true + ) select * from workspace_facts + ) as blocker_workspace_facts`; +} + +/** An issue's own operation takes precedence over the legacy NULL fallback. + * Only a strictly later successful finalize clears either barrier. */ +function pendingFinalizeBlockerCondition(companyId: string, issueId: SQL, workspaceId: SQL) { + return sql`${workspaceId} is not null and exists ( + select 1 from (values (1)) seed(value) + left join lateral ( + select phase, status, started_at from workspace_operations blocker_operation + where blocker_operation.company_id = ${companyId}::uuid + and blocker_operation.execution_workspace_id = ${workspaceId} + and blocker_operation.issue_id = ${issueId} + order by blocker_operation.started_at desc limit 1 + ) latest_blocker_operation on true + where coalesce(latest_blocker_operation.started_at, blocker_workspace_facts.started_at) is not null + and not (coalesce(latest_blocker_operation.phase, blocker_workspace_facts.phase) = 'workspace_finalize' + and coalesce(latest_blocker_operation.status, blocker_workspace_facts.status) = 'succeeded') + and (blocker_workspace_facts.finalized_at is null + or blocker_workspace_facts.finalized_at <= coalesce(latest_blocker_operation.started_at, blocker_workspace_facts.started_at)) + )`; +} + async function listPendingFinalizeBlockerIssueIds( dbOrTx: Pick, companyId: string, - blockerWorkspacePairs: Array<{ - blockerIssueId: string; - executionWorkspaceId: string; - }>, + blockerWorkspacePairs: Array<{ blockerIssueId: string; executionWorkspaceId: string }>, lockSources = false, ): Promise> { const pending = new Set(); - const blockerIssueIds = [ - ...new Set(blockerWorkspacePairs.map((pair) => pair.blockerIssueId)), - ]; - const executionWorkspaceIds = [ - ...new Set(blockerWorkspacePairs.map((pair) => pair.executionWorkspaceId)), - ]; - if (blockerIssueIds.length === 0 || executionWorkspaceIds.length === 0) - return pending; - const blockerWorkspaceKeys = new Set( - blockerWorkspacePairs.map( - (pair) => `${pair.blockerIssueId}:${pair.executionWorkspaceId}`, - ), - ); - + const pairs = [...new Map(blockerWorkspacePairs.map(pair => [`${pair.blockerIssueId}:${pair.executionWorkspaceId}`, pair])).values()]; if (lockSources) { - - - await dbOrTx.select({ id: executionWorkspaces.id }).from(executionWorkspaces) - .where(and(eq(executionWorkspaces.companyId, companyId), inArray(executionWorkspaces.id, executionWorkspaceIds))) - .for("update", { noWait: true }); - } - const query = dbOrTx - .select({ - issueId: workspaceOperations.issueId, - executionWorkspaceId: workspaceOperations.executionWorkspaceId, - phase: workspaceOperations.phase, - status: workspaceOperations.status, - startedAt: workspaceOperations.startedAt, - }) - .from(workspaceOperations) - .where( - and( - eq(workspaceOperations.companyId, companyId), - inArray( - workspaceOperations.executionWorkspaceId, - executionWorkspaceIds, - ), - ), - ); - - const rows = await (lockSources ? query.for("share", { noWait: true }) : query); - - const latestAttributedByBlockerWorkspace = new Map< - string, - { phase: string; status: string; startedAt: Date } - >(); - const latestUnattributedByWorkspace = new Map< - string, - { phase: string; status: string; startedAt: Date } - >(); - const latestSuccessfulFinalizeByWorkspace = new Map(); - for (const row of rows) { - if (!row.executionWorkspaceId) continue; - if (row.phase === "workspace_finalize" && row.status === "succeeded") { - const current = latestSuccessfulFinalizeByWorkspace.get( - row.executionWorkspaceId, - ); - if (!current || row.startedAt > current) { - latestSuccessfulFinalizeByWorkspace.set( - row.executionWorkspaceId, - row.startedAt, - ); - } - } - if (row.issueId) { - const key = `${row.issueId}:${row.executionWorkspaceId}`; - if (!blockerWorkspaceKeys.has(key)) continue; - const current = latestAttributedByBlockerWorkspace.get(key); - if (!current || row.startedAt > current.startedAt) { - latestAttributedByBlockerWorkspace.set(key, { - phase: row.phase, - status: row.status, - startedAt: row.startedAt, - }); - } - continue; - } - - const current = latestUnattributedByWorkspace.get(row.executionWorkspaceId); - if (!current || row.startedAt > current.startedAt) { - latestUnattributedByWorkspace.set(row.executionWorkspaceId, { - phase: row.phase, - status: row.status, - startedAt: row.startedAt, - }); + const workspaceIds = [...new Set(pairs.map(pair => pair.executionWorkspaceId))]; + for (const chunk of chunkList(workspaceIds, ISSUE_LIST_RELATED_QUERY_CHUNK_SIZE)) { + await dbOrTx.select({ id: executionWorkspaces.id }).from(executionWorkspaces) + .where(and(eq(executionWorkspaces.companyId, companyId), inArray(executionWorkspaces.id, chunk))) + .for("update", { noWait: true }); + await dbOrTx.select({ id: workspaceOperations.id }).from(workspaceOperations) + .where(and(eq(workspaceOperations.companyId, companyId), inArray(workspaceOperations.executionWorkspaceId, chunk))) + .for("share", { noWait: true }); } } - - for (const pair of blockerWorkspacePairs) { - const latest = - latestAttributedByBlockerWorkspace.get( - `${pair.blockerIssueId}:${pair.executionWorkspaceId}`, - ) ?? latestUnattributedByWorkspace.get(pair.executionWorkspaceId); - if (!latest) continue; // no ops recorded -> nothing to finalize for this blocker - if (latest.phase === "workspace_finalize" && latest.status === "succeeded") - continue; - const laterSuccessfulFinalize = latestSuccessfulFinalizeByWorkspace.get( - pair.executionWorkspaceId, - ); - if (laterSuccessfulFinalize && laterSuccessfulFinalize > latest.startedAt) - continue; - pending.add(pair.blockerIssueId); + // Each pair occurs in both the source and the workspace scope. + for (const chunk of chunkList(pairs, ISSUE_LIST_RELATED_QUERY_CHUNK_SIZE / 2)) { + const values = sql.join(chunk.map(pair => sql`(${pair.blockerIssueId}::uuid, ${pair.executionWorkspaceId}::uuid)`), sql`, `); + const rows = await dbOrTx.select({ id: sql`blocker_source.issue_id` }) + .from(sql`(values ${values}) as blocker_source(issue_id, workspace_id)`) + .leftJoin(blockerWorkspaceFacts(companyId, sql`select workspace_id from (values ${values}) workspace_scope(issue_id, workspace_id)`), + sql`blocker_workspace_facts.workspace_id = blocker_source.workspace_id`) + .where(pendingFinalizeBlockerCondition(companyId, sql`blocker_source.issue_id`, sql`blocker_source.workspace_id`)); + for (const row of rows) pending.add(row.id); } - return pending; } @@ -3436,6 +3404,7 @@ type IssueBlockerAttentionEdge = { blockerIssueId: string; }; type IssueBlockerAttentionQueryRow = IssueBlockerAttentionNode & { + executionWorkspaceId?: string | null; issueId: string | null; blockerIssueId: string; }; @@ -3824,11 +3793,15 @@ async function terminalExplicitBlockersByRoot( return terminalByRoot; } -async function listIssueBlockerAttentionMap( +async function listIssueBlockerAttentionMap(dbOrTx: any, companyId: string, issueRows: IssueBlockerAttentionInputNode[]) { + return (await readIssueBlockerAttentionBatch(dbOrTx, companyId, issueRows)).attention; +} + +async function readIssueBlockerAttentionBatch( dbOrTx: any, companyId: string, issueRows: IssueBlockerAttentionInputNode[], -): Promise> { +) { const roots = issueRows.filter( (row) => row.companyId === companyId && row.status === "blocked", ); @@ -3838,42 +3811,44 @@ async function listIssueBlockerAttentionMap( attentionMap.set(row.id, createIssueBlockerAttention()); } } - if (roots.length === 0) return attentionMap; + const deferredRootIds = new Set(roots.slice(BLOCKER_ATTENTION_MAX_NODES).map(root => root.id)); + let edgeCount = 0, fetchedRows = 0; + const result = () => ({ attention: attentionMap, deferredRootIds, nodeCount: nodesById.size, edgeCount, fetchedRows }); + if (roots.length === 0) return { attention: attentionMap, deferredRootIds, nodeCount: 0, edgeCount, fetchedRows }; const nodesById = new Map(); const edgesByIssueId = new Map(); - for (const root of roots) nodesById.set(root.id, { ...root }); + // Returned rows are already bounded by the residual fetch budget. Keep their + // positive live-work evidence even when they cannot enter the traversal graph. + const observedNodes = new Map(); + const observedChildrenByParent = new Map>(); + const admittedRoots = roots.slice(0, BLOCKER_ATTENTION_MAX_NODES); + for (const root of admittedRoots) nodesById.set(root.id, { ...root }); - let frontier = roots.map((root) => root.id); - let truncated = false; + type Traversal = { rootId: string; nodes: Set; expanded: Set; frontier: string[]; truncated: boolean }; + const traversals = new Map(admittedRoots.map(root => [root.id, { + rootId: root.id, nodes: new Set([root.id]), expanded: new Set(), frontier: [root.id], truncated: false, + }])); + const expanded = new Set(); + const queried = new Set(); const pendingFinalizeBlockerIssueIds = new Set(); - for ( - let depth = 0; - frontier.length > 0 && depth < BLOCKER_ATTENTION_MAX_DEPTH; - depth += 1 - ) { - const nextFrontier = new Set(); + for (let depth = 0; depth < BLOCKER_ATTENTION_MAX_DEPTH; depth += 1) { + const active = [...traversals.values()].filter(traversal => !traversal.truncated && traversal.frontier.length > 0); + if (active.length === 0) break; + const unread = new Set(); + for (const traversal of active) for (const id of traversal.frontier) if (!expanded.has(id)) unread.add(id); + const frontier = [...unread]; for (const chunk of chunkList( - [...new Set(frontier)], + frontier, ISSUE_LIST_RELATED_QUERY_CHUNK_SIZE, )) { - const readinessByIssueId = await listIssueDependencyReadinessMap( - dbOrTx, - companyId, - chunk, - ); - for (const readiness of readinessByIssueId.values()) { - for (const blockerIssueId of readiness.pendingFinalizeBlockerIssueIds) { - pendingFinalizeBlockerIssueIds.add(blockerIssueId); - } - } - const explicitBlockerRowsPromise: Promise< - IssueBlockerAttentionQueryRow[] - > = dbOrTx + const remaining = Math.min(BLOCKER_ATTENTION_MAX_NODES - nodesById.size, + BLOCKER_ATTENTION_MAX_NODES * 2 - edgeCount); + if (remaining <= 0) break; + const rowLimit = remaining + 1; + const explicitBlocker = dbOrTx .select({ - issueId: issueRelations.relatedIssueId, - blockerIssueId: issues.id, id: issues.id, companyId: issues.companyId, parentId: issues.parentId, @@ -3881,19 +3856,55 @@ async function listIssueBlockerAttentionMap( title: issues.title, status: issues.status, executionRunId: issues.executionRunId, + executionWorkspaceId: issues.executionWorkspaceId, assigneeAgentId: issues.assigneeAgentId, assigneeUserId: issues.assigneeUserId, }) + // Resolve the globally unique relation target first. The company guard + // below remains authoritative, without a repeated company-index scan. + .from(issues).where(eq(issues.id, issueRelations.issueId)) + .limit(1).as("explicit_blocker"); + const workspaceScope = sql`select distinct candidate.execution_workspace_id + from issue_relations workspace_relation + inner join lateral ( + select company_id, status, execution_workspace_id from issues + where id = workspace_relation.issue_id limit 1 + ) candidate on true + where workspace_relation.company_id = ${companyId}::uuid + and workspace_relation.type = 'blocks' + and workspace_relation.related_issue_id in (${sql.join(chunk.map(id => sql`${id}::uuid`), sql`, `)}) + and candidate.company_id = ${companyId}::uuid + and candidate.status = 'done' and candidate.execution_workspace_id is not null`; + const explicitBlockerRowsPromise: Promise< + IssueBlockerAttentionQueryRow[] + > = dbOrTx + .select({ + issueId: issueRelations.relatedIssueId, + blockerIssueId: explicitBlocker.id, + id: explicitBlocker.id, + companyId: explicitBlocker.companyId, + parentId: explicitBlocker.parentId, + identifier: explicitBlocker.identifier, + title: explicitBlocker.title, + status: explicitBlocker.status, + executionRunId: explicitBlocker.executionRunId, + executionWorkspaceId: explicitBlocker.executionWorkspaceId, + assigneeAgentId: explicitBlocker.assigneeAgentId, + assigneeUserId: explicitBlocker.assigneeUserId, + }) .from(issueRelations) - .innerJoin(issues, eq(issueRelations.issueId, issues.id)) + .innerJoinLateral(explicitBlocker, sql`true`) + .leftJoin(blockerWorkspaceFacts(companyId, workspaceScope), + sql`blocker_workspace_facts.workspace_id = ${explicitBlocker.executionWorkspaceId}`) .where( and( eq(issueRelations.companyId, companyId), eq(issueRelations.type, "blocks"), inArray(issueRelations.relatedIssueId, chunk), - eq(issues.companyId, companyId), + eq(explicitBlocker.companyId, companyId), + or(ne(explicitBlocker.status, "done"), pendingFinalizeBlockerCondition(companyId, sql`${explicitBlocker.id}`, sql`${explicitBlocker.executionWorkspaceId}`)), ), - ); + ).limit(rowLimit); const childRowsPromise: Promise = dbOrTx .select({ issueId: issues.parentId, @@ -3918,68 +3929,83 @@ async function listIssueBlockerAttentionMap( BLOCKER_ATTENTION_CHILD_TERMINAL_STATUSES, ), ), - ); + ).limit(rowLimit); const [explicitBlockerRows, childRows] = await Promise.all([ explicitBlockerRowsPromise, childRowsPromise, ]); + for (const id of chunk) queried.add(id); + fetchedRows += explicitBlockerRows.length + childRows.length; + const pendingFinalize = await listPendingFinalizeBlockerIssueIds(dbOrTx, companyId, + explicitBlockerRows.flatMap(row => row.status === "done" && row.executionWorkspaceId + ? [{ blockerIssueId: row.blockerIssueId, executionWorkspaceId: row.executionWorkspaceId }] : [])); + for (const id of pendingFinalize) pendingFinalizeBlockerIssueIds.add(id); + let complete = explicitBlockerRows.length < rowLimit && childRows.length < rowLimit; const unresolvedExplicitBlockerRows = explicitBlockerRows.filter( (row) => row.status !== "done" || pendingFinalizeBlockerIssueIds.has(row.blockerIssueId), ); - appendBlockerAttentionEdges(edgesByIssueId, [ - ...unresolvedExplicitBlockerRows - .filter( - (row): row is IssueBlockerAttentionQueryRow & { issueId: string } => - row.issueId !== null, - ) - .map((row) => ({ - issueId: row.issueId, - blockerIssueId: row.blockerIssueId, - })), - ...childRows - .filter( - (row): row is IssueBlockerAttentionQueryRow & { issueId: string } => - row.issueId !== null, - ) - .map((row) => ({ - issueId: row.issueId, - blockerIssueId: row.blockerIssueId, - })), - ]); - for (const row of [...unresolvedExplicitBlockerRows, ...childRows]) { - if (!row.issueId || nodesById.has(row.blockerIssueId)) continue; - nodesById.set(row.blockerIssueId, { - id: row.blockerIssueId, - companyId: row.companyId, - parentId: row.parentId, - identifier: row.identifier, - title: row.title, - status: row.status, - executionRunId: row.executionRunId, - assigneeAgentId: row.assigneeAgentId, - assigneeUserId: row.assigneeUserId, - }); - nextFrontier.add(row.blockerIssueId); + if (!row.issueId) continue; + observedNodes.set(row.blockerIssueId, row); + const observed = observedChildrenByParent.get(row.issueId) ?? new Set(); + observed.add(row.blockerIssueId); + observedChildrenByParent.set(row.issueId, observed); + if (!nodesById.has(row.blockerIssueId)) { + if (nodesById.size >= BLOCKER_ATTENTION_MAX_NODES) { complete = false; continue; } + nodesById.set(row.blockerIssueId, { + id: row.blockerIssueId, companyId: row.companyId, parentId: row.parentId, identifier: row.identifier, + title: row.title, status: row.status, executionRunId: row.executionRunId, + assigneeAgentId: row.assigneeAgentId, assigneeUserId: row.assigneeUserId, + }); + } + const edges = edgesByIssueId.get(row.issueId) ?? []; + if (edges.some(edge => edge.blockerIssueId === row.blockerIssueId)) continue; + if (edgeCount >= BLOCKER_ATTENTION_MAX_NODES * 2) { complete = false; continue; } + edges.push({ issueId: row.issueId, blockerIssueId: row.blockerIssueId }); + edgesByIssueId.set(row.issueId, edges); + edgeCount++; } + if (complete) for (const id of chunk) expanded.add(id); + } - if (nodesById.size > BLOCKER_ATTENTION_MAX_NODES) { - truncated = true; - break; + for (const traversal of active) { + if (traversal.frontier.some(id => !expanded.has(id))) { + for (const id of traversal.frontier) if (queried.has(id)) { + traversal.expanded.add(id); + for (const edge of edgesByIssueId.get(id) ?? []) if (nodesById.has(edge.blockerIssueId)) traversal.nodes.add(edge.blockerIssueId); + } + traversal.truncated = true; + traversal.frontier = []; + deferredRootIds.add(traversal.rootId); + continue; + } + const nextFrontier = new Set(); + for (const id of traversal.frontier) { + traversal.expanded.add(id); + for (const edge of edgesByIssueId.get(id) ?? []) { + if (traversal.nodes.has(edge.blockerIssueId) || !nodesById.has(edge.blockerIssueId)) continue; + traversal.nodes.add(edge.blockerIssueId); + nextFrontier.add(edge.blockerIssueId); + } + } + traversal.truncated = traversal.nodes.size > BLOCKER_ATTENTION_MAX_NODES; + traversal.frontier = traversal.truncated ? [] : [...nextFrontier]; } - frontier = [...nextFrontier]; } - if (frontier.length > 0) truncated = true; + for (const traversal of traversals.values()) { + if (traversal.frontier.length > 0) traversal.truncated = true; + } - const nodeIds = [...nodesById.keys()]; + const evidenceNodes = new Map([...nodesById, ...observedNodes]); + const nodeIds = [...evidenceNodes.keys()]; const activeIssueIds = new Set(); const agentIds = new Set(); const issueIdByExecutionRunId = new Map(); - for (const node of nodesById.values()) { + for (const node of evidenceNodes.values()) { if (node.assigneeAgentId) agentIds.add(node.assigneeAgentId); if (node.executionRunId) issueIdByExecutionRunId.set(node.executionRunId, node.id); @@ -4037,7 +4063,7 @@ async function listIssueBlockerAttentionMap( } } - const explicitWaitCandidateIds = [...nodesById.values()] + const explicitWaitCandidateIds = [...evidenceNodes.values()] .filter((node) => node.status !== "done") .map((node) => node.id); const explicitWaitingIssueIds = new Set(); @@ -4112,23 +4138,26 @@ async function listIssueBlockerAttentionMap( ownerType: string; ownerAgentId: string | null; ownerUserId: string | null; - }> = await dbOrTx - .select({ - id: issueRecoveryActions.id, - sourceIssueId: issueRecoveryActions.sourceIssueId, - status: issueRecoveryActions.status, - ownerType: issueRecoveryActions.ownerType, - ownerAgentId: issueRecoveryActions.ownerAgentId, - ownerUserId: issueRecoveryActions.ownerUserId, - }) - .from(issueRecoveryActions) - .where( - and( - eq(issueRecoveryActions.companyId, companyId), - inArray(issueRecoveryActions.status, ["active", "escalated"]), - inArray(issueRecoveryActions.sourceIssueId, explicitWaitCandidateIds), - ), - ); + }> = []; + for (const chunk of chunkList(explicitWaitCandidateIds, ISSUE_LIST_RELATED_QUERY_CHUNK_SIZE)) { + recoveryActionRows.push(...await dbOrTx + .select({ + id: issueRecoveryActions.id, + sourceIssueId: issueRecoveryActions.sourceIssueId, + status: issueRecoveryActions.status, + ownerType: issueRecoveryActions.ownerType, + ownerAgentId: issueRecoveryActions.ownerAgentId, + ownerUserId: issueRecoveryActions.ownerUserId, + }) + .from(issueRecoveryActions) + .where( + and( + eq(issueRecoveryActions.companyId, companyId), + inArray(issueRecoveryActions.status, ["active", "escalated"]), + inArray(issueRecoveryActions.sourceIssueId, chunk), + ), + )); + } const recoveryActionIds = recoveryActionRows.map((row) => row.id); const liveRecoveryActionIds = new Set(); for (const chunk of chunkList( @@ -4191,22 +4220,11 @@ async function listIssueBlockerAttentionMap( } } - const agentRows: IssueBlockerAttentionAgentRow[] = - agentIds.size > 0 - ? await dbOrTx - .select({ - id: agents.id, - companyId: agents.companyId, - status: agents.status, - }) - .from(agents) - .where( - and( - eq(agents.companyId, companyId), - inArray(agents.id, [...agentIds]), - ), - ) - : []; + const agentRows: IssueBlockerAttentionAgentRow[] = []; + for (const chunk of chunkList([...agentIds], ISSUE_LIST_RELATED_QUERY_CHUNK_SIZE)) { + agentRows.push(...await dbOrTx.select({ id: agents.id, companyId: agents.companyId, status: agents.status }) + .from(agents).where(and(eq(agents.companyId, companyId), inArray(agents.id, chunk)))); + } const agentsById = new Map(agentRows.map((agent) => [agent.id, agent])); type PathClassification = { @@ -4219,6 +4237,7 @@ async function listIssueBlockerAttentionMap( const classifyPath = ( nodeId: string, seen: Set, + truncated: boolean, ): PathClassification => { const sample = blockerSampleIdentifier(nodesById.get(nodeId)); if (truncated || seen.has(nodeId)) { @@ -4323,7 +4342,7 @@ async function listIssueBlockerAttentionMap( const nextSeen = new Set(seen); nextSeen.add(nodeId); const classified = downstream.map((edge) => - classifyPath(edge.blockerIssueId, nextSeen), + classifyPath(edge.blockerIssueId, nextSeen, truncated), ); const stalledChild = classified.find( (result) => result.stalled || result.sampleStalledBlockerIdentifier, @@ -4389,18 +4408,22 @@ async function listIssueBlockerAttentionMap( }; }; - const pathHasLiveWork = (nodeId: string, seen: Set): boolean => { - if (seen.has(nodeId)) return false; + const nodeHasLiveWork = (node: IssueBlockerAttentionNode) => + node.status === "in_progress" || activeIssueIds.has(node.id) || + explicitWaitingIssueIds.has(node.id) || Boolean(node.assigneeUserId); + const observedChildHasLiveWork = (parentId: string) => + [...observedChildrenByParent.get(parentId) ?? []].some(id => { + const node = observedNodes.get(id); + return node?.companyId === companyId && nodeHasLiveWork(node); + }); + const pathHasLiveWork = (nodeId: string, seen: Set, traversal: Traversal): boolean => { + if (seen.has(nodeId) || !traversal.nodes.has(nodeId)) return false; const node = nodesById.get(nodeId); if (!node || node.companyId !== companyId) return false; - if ( - node.status === "in_progress" || - activeIssueIds.has(node.id) || - explicitWaitingIssueIds.has(node.id) || - Boolean(node.assigneeUserId) - ) - return true; + if (nodeHasLiveWork(node)) return true; + if (!traversal.expanded.has(nodeId)) return false; + if (observedChildHasLiveWork(nodeId)) return true; const nextSeen = new Set(seen); nextSeen.add(nodeId); return (edgesByIssueId.get(node.id) ?? []).some((edge) => { @@ -4410,7 +4433,7 @@ async function listIssueBlockerAttentionMap( !pendingFinalizeBlockerIssueIds.has(edge.blockerIssueId) ) return false; - return pathHasLiveWork(edge.blockerIssueId, nextSeen); + return pathHasLiveWork(edge.blockerIssueId, nextSeen, traversal); }); }; @@ -4423,6 +4446,7 @@ async function listIssueBlockerAttentionMap( }; for (const root of roots) { + const traversal = traversals.get(root.id)!; const topLevelEdges = (edgesByIssueId.get(root.id) ?? []).filter((edge) => { const blocker = nodesById.get(edge.blockerIssueId); return ( @@ -4444,7 +4468,7 @@ async function listIssueBlockerAttentionMap( const classified = topLevelEdges.map((edge) => ({ edge, - result: classifyPath(edge.blockerIssueId, new Set([root.id])), + result: classifyPath(edge.blockerIssueId, new Set([root.id]), traversal.truncated), })); const coveredBlockerCount = classified.filter( (entry) => entry.result.covered, @@ -4526,8 +4550,8 @@ async function listIssueBlockerAttentionMap( stalledEntry?.result.sampleStalledBlockerIdentifier ?? sampleStalledFromChain ?? null, - blockingTreeLive: topLevelEdges.some((edge) => - pathHasLiveWork(edge.blockerIssueId, new Set([root.id])), + blockingTreeLive: observedChildHasLiveWork(root.id) || topLevelEdges.some((edge) => + pathHasLiveWork(edge.blockerIssueId, new Set([root.id]), traversal), ), directBlockerIssueId: sampleEntry?.edge.blockerIssueId ?? null, terminalBlockerIssueId, @@ -4542,7 +4566,7 @@ async function listIssueBlockerAttentionMap( ); } - return attentionMap; + return result(); } type IssueReviewAttentionInput = Pick; @@ -7602,7 +7626,7 @@ export function issueService(db: Db) { companyId: string, blockedByIssueIds: string[], actor: { agentId?: string | null; userId?: string | null } = {}, - dbOrTx: any = db, + dbOrTx: any, ) { const deduped = [...new Set(blockedByIssueIds)]; if (deduped.some((candidate) => candidate === issueId)) { @@ -7631,6 +7655,7 @@ export function issueService(db: Db) { await assertNoBlockingCycles(companyId, issueId, deduped, dbOrTx); } + await notifyIssueAttentionRetentionWork(dbOrTx, { companyId, issueId }); await notifyIssueRecovery(dbOrTx, { companyId }); await notifyDeferredExecutionEligibility(dbOrTx, { companyId }); await dbOrTx @@ -9380,6 +9405,9 @@ export function issueService(db: Db) { return listIssueDependencyReadinessMap(dbOrTx, companyId, issueIds); }, + listBlockerAttentionBatch: async (companyId: string, issueRows: IssueBlockerAttentionInputNode[], dbOrTx: any = db) => + readIssueBlockerAttentionBatch(dbOrTx, companyId, issueRows), + listBlockerAttention: async ( companyId: string, issueRows: IssueBlockerAttentionInputNode[], @@ -9684,6 +9712,8 @@ export function issueService(db: Db) { if (blockParentUntilDone) { await db.transaction(async tx => { + await tx.select({ id: issues.id }).from(issues).where(and( + eq(issues.companyId, parent.companyId), eq(issues.id, parent.id))).for("update"); const existingBlockers = await tx .select({ blockerIssueId: issueRelations.issueId }) .from(issueRelations) @@ -10620,6 +10650,8 @@ export function issueService(db: Db) { await notifyRunDispatch(tx, { companyId: values.companyId }); await notifyDeferredExecutionEligibility(tx, { companyId: values.companyId }); await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers); + values.id ??= randomUUID(); + await notifyIssueAttentionRetentionWork(tx, { companyId, issueId: values.id, parentId: values.parentId, originKind: values.originKind, originId: values.originId }); await notifyStatusCardIssueChange(tx, companyId); const [issue] = await tx.insert(issues).values(values).returning(); await attachOwnedDraftImages(tx, issue, issue.description, { userId: issueData.createdByUserId, agentId: issueData.createdByAgentId }); @@ -11509,6 +11541,7 @@ export function issueService(db: Db) { || issueData.parentId !== undefined || issueData.executionWorkspaceId !== undefined) { await notifyTerminalWorkspaceWork(tx, `issue:${id}`); } + await notifyIssueAttentionRetentionWork(tx, { companyId: receiptExisting.companyId, issueId: id, parentId: patch.parentId, originKind: patch.originKind, originId: patch.originId }); if (patch.status !== undefined || patch.assigneeAgentId !== undefined || patch.assigneeUserId !== undefined || patch.hiddenAt !== undefined || patch.conversationAgentId !== undefined || patch.parentId !== undefined || patch.executionState !== undefined || patch.executionRunId !== undefined || patch.checkoutRunId !== undefined @@ -11977,6 +12010,7 @@ export function issueService(db: Db) { await requestRestorePointDeletion(tx, { sourceIssueId: id }); const [owner] = await tx.select({ companyId: issues.companyId }).from(issues).where(eq(issues.id, id)); if (owner) { + await notifyIssueAttentionRetentionWork(tx, { companyId: owner.companyId, issueId: id }); await notifyExternalConversationSettlement(tx, { companyId: owner.companyId }); await notifyApprovalWork(tx, owner.companyId); await notifyStatusCardIssueChange(tx, owner.companyId); @@ -12112,6 +12146,7 @@ export function issueService(db: Db) { ) : isNull(issues.executionRunId); const updated = await db.transaction(async tx => { + await notifyIssueAttentionRetentionWork(tx, { companyId: issueCompany.companyId, issueId: id }); await notifyStatusCardIssueChange(tx, issueCompany.companyId); await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors); await notifyRunDispatch(tx, { companyId: issueCompany.companyId }); @@ -12247,6 +12282,7 @@ export function issueService(db: Db) { } const staleExecutionRunId = current.executionRunId; const adopted = await db.transaction(async tx => { + await notifyIssueAttentionRetentionWork(tx, { companyId: issueCompany.companyId, issueId: id }); await notifyStatusCardIssueChange(tx, issueCompany.companyId); await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors); await notifyRunDispatch(tx, { companyId: issueCompany.companyId }); @@ -12505,6 +12541,7 @@ export function issueService(db: Db) { existing.status === "in_progress" ? "todo" : existing.status; await notifyTerminalWorkspaceWork(tx, `issue:${id}`); await notifyNativeStatusReassessment(tx, { companyId: existing.companyId, issueId: id }); + await notifyIssueAttentionRetentionWork(tx, { companyId: existing.companyId, issueId: id }); await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors); await notifyRunDispatch(tx, { companyId: existing.companyId }); await notifyDeferredExecutionEligibility(tx, { companyId: existing.companyId }); @@ -12559,6 +12596,7 @@ export function issueService(db: Db) { patch.assigneeAgentId = null; } + await notifyIssueAttentionRetentionWork(tx, { companyId: existing.companyId, issueId: id }); await notifyStatusCardIssueChange(tx, existing.companyId); await notifyTerminalWorkspaceWork(tx, `issue:${id}`); if (existing.checkoutRunId || existing.executionRunId || options.clearAssignee) { diff --git a/server/src/services/native-finalization-notifications.ts b/server/src/services/native-finalization-notifications.ts index 72d4cbcac49..1feb1ba9703 100644 --- a/server/src/services/native-finalization-notifications.ts +++ b/server/src/services/native-finalization-notifications.ts @@ -1,3 +1,4 @@ +import { notifyAttentionRetentionWork } from "./retention-work-notifications.js"; import { notifyNativeEnvironmentSettlement } from "./native-environment-settlement-notifications.js"; import { notifyExternalAgentSettlement } from "../modules/external-agents/index.js"; import { notifyExternalConversationSettlement } from "./external-conversation-settlement-notifications.js"; @@ -14,6 +15,7 @@ import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notificatio export type NativeFinalizationWorkScope = { companyId: string; runId?: string; issueId?: string }; export async function notifyNativeFinalizationWork(transaction: object, scope: NativeFinalizationWorkScope) { + await notifyAttentionRetentionWork(transaction, { companyId: scope.companyId }); if (scope.issueId || scope.runId) { await notifyNativeEnvironmentSettlement(transaction, scope); await notifyExternalAgentSettlement(transaction, scope); diff --git a/server/src/services/native-runtime/native-provider-capacity-retry.ts b/server/src/services/native-runtime/native-provider-capacity-retry.ts index 99575395f52..cb05729dcc5 100644 --- a/server/src/services/native-runtime/native-provider-capacity-retry.ts +++ b/server/src/services/native-runtime/native-provider-capacity-retry.ts @@ -1,3 +1,4 @@ +import { notifyAttentionRetentionWork } from "../retention-work-notifications.js"; import { notifyScheduledRetryDispatch } from "../run-dispatch-notifications.js"; import { randomUUID } from "node:crypto"; import { and, eq } from "drizzle-orm"; @@ -50,6 +51,7 @@ export async function materializeNativeProviderCapacityRetry(input: { scheduledRetryAt: dueAt.toISOString(), executionRetryAccounting: accountingForScheduledRetry(run, NATIVE_PROVIDER_OVERLOADED_CODE, attempt), }; + await notifyAttentionRetentionWork(input.tx, { companyId: input.companyId }); await notifyScheduledRetryDispatch(input.tx, { companyId: run.companyId }); const [wake] = await input.tx.insert(agentWakeupRequests).values({ companyId: run.companyId, agentId: run.agentId, source: "automation", triggerDetail: "system", diff --git a/server/src/services/native-runtime/native-review-participant.ts b/server/src/services/native-runtime/native-review-participant.ts index 6391d506d5d..16787a140b4 100644 --- a/server/src/services/native-runtime/native-review-participant.ts +++ b/server/src/services/native-runtime/native-review-participant.ts @@ -1,3 +1,4 @@ +import { notifyAttentionRetentionWork } from "../retention-work-notifications.js"; import { notifyStatusCardIssueChange } from "../status-card-notifications.js"; import { agents, @@ -241,6 +242,7 @@ export async function claimNativeReviewExecutionLock(db: Db, input: { if (!run || !["queued", "running"].includes(run.status) || !admitted || !expected || admitted.nativeReviewInteractionId !== expected.nativeReviewInteractionId || admitted.nativeReviewDecisionId !== expected.nativeReviewDecisionId) return false; + await notifyAttentionRetentionWork(tx, { companyId: input.companyId }); await notifyStatusCardIssueChange(tx, input.companyId); const claimed = await tx.update(issues).set({ executionRunId: input.runId, diff --git a/server/src/services/native-runtime/native-session-executor.test.ts b/server/src/services/native-runtime/native-session-executor.test.ts index f8684817dd9..dc82054a0eb 100644 --- a/server/src/services/native-runtime/native-session-executor.test.ts +++ b/server/src/services/native-runtime/native-session-executor.test.ts @@ -4492,6 +4492,17 @@ describe("retained native cleanup activation", () => { releaseCommit = release; }); const db = { + execute: async (query: SQL) => { + // Retention hints read the fixture's run/issue relationship inside the + // same cleanup transaction; this fixture has one issue and no ancestors. + const statement = new PgDialect().sqlToQuery(query); + expect(transactionOpen).toBe(true); + expect(statement.params).toContain(run.companyId); + if (statement.sql.includes("from heartbeat_runs")) return [{ id: run.nativeIssueId }]; + if (statement.sql.includes("origin_kind")) return [{ id: run.nativeIssueId, originKind: null, originId: null }]; + if (statement.sql.includes("with recursive affected")) return [{ issueId: run.nativeIssueId }]; + throw new Error(`Unexpected cleanup fixture query: ${statement.sql}`); + }, select: () => ({ from: (table: unknown) => { const rows = diff --git a/server/src/services/native-runtime/native-workspace-reconstruction.ts b/server/src/services/native-runtime/native-workspace-reconstruction.ts index 33e37704389..d616ed266ed 100644 --- a/server/src/services/native-runtime/native-workspace-reconstruction.ts +++ b/server/src/services/native-runtime/native-workspace-reconstruction.ts @@ -1,3 +1,4 @@ +import { notifyAttentionRetentionWork } from "../retention-work-notifications.js"; import { notifyStatusCardIssueChange } from "../status-card-notifications.js"; import { DELIVERY_QUEUES, notifyDeliveryWork } from "../delivery-work-notifications.js"; import { notifyUnsafeWorkspaceExportRecovery } from "../unsafe-workspace-export-notifications.js"; @@ -180,6 +181,7 @@ export async function reconstructNativeWorkspace(input: { ...(source.interveningExecutions.length ? { interveningExecutions: source.interveningExecutions } : {}) } }; await markExecutionReconciliation(db, source.action, decision, input.actorId); const now = new Date(); + await notifyAttentionRetentionWork(tx, { companyId: input.companyId }); await notifyUnsafeWorkspaceExportRecovery(tx, { companyId: input.companyId, issueId: input.issueId }); await tx.update(issueRecoveryActions).set({ status: "resolved", outcome: "handed_back", resolvedAt: now, updatedAt: now, resolutionNote: "workspace_reconstruction_authorized", nextAction: "Continue from saved sources. The original unexported edits are lost; verify the selected attachments before rebuilding. Do not repeat completed external actions or reset the original work limits.", diff --git a/server/src/services/native-runtime/paperclip-runner-tool-authority.ts b/server/src/services/native-runtime/paperclip-runner-tool-authority.ts index 69a8625892b..a6681db000f 100644 --- a/server/src/services/native-runtime/paperclip-runner-tool-authority.ts +++ b/server/src/services/native-runtime/paperclip-runner-tool-authority.ts @@ -1,5 +1,6 @@ import { notifyStatusCardIssueChange } from "../status-card-notifications.js"; import { nativeRunKeyScope, nativeSkillTestToolAllowed, assertNativeSkillTestTool } from "./native-skill-test-scope.js"; +import { notifyAttentionRetentionWork } from "../retention-work-notifications.js"; import { requiredQualityReviewSchema } from "@paperclipai/shared"; import { priorRecoveryApiReceipt } from "../recovery-effect-receipts.js"; import { DELIVERY_QUEUES, notifyDeliveryWork } from "../delivery-work-notifications.js"; @@ -2080,6 +2081,7 @@ export class PaperclipRunnerToolAuthority { replayMonitor = summarizeIssueMonitor(target, normalizeIssueExecutionPolicy(target.executionPolicy)); return { taskId: target.id, status: target.status, monitor: replayMonitor }; } + await notifyAttentionRetentionWork(tx, { companyId: this.binding.companyId }); const patch = prepareIssueMonitorUpdate(target, input.monitor, actor); await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors); await notifyStatusCardIssueChange(tx, this.binding.companyId); diff --git a/server/src/services/native-runtime/superseded-external-review-requests.ts b/server/src/services/native-runtime/superseded-external-review-requests.ts index f04db57f7d7..9697b436ed3 100644 --- a/server/src/services/native-runtime/superseded-external-review-requests.ts +++ b/server/src/services/native-runtime/superseded-external-review-requests.ts @@ -1,3 +1,4 @@ +import { notifyAttentionRetentionWork } from "../retention-work-notifications.js"; import { and, eq, isNull, ne, sql } from "drizzle-orm"; import { completionContracts, heartbeatRuns, issues, issueThreadInteractions, statusDecisionEffects, statusDecisions, workAssessments, type Db } from "@paperclipai/db"; import { hasPublishedGitHubReview } from "../chat-channels/github/review-wait.js"; @@ -70,6 +71,7 @@ export async function supersedeExternalReviewRequestsInTransaction( publications: ActivityPublication[], ) { for (const { interaction } of candidates) { + await notifyAttentionRetentionWork(tx, { companyId: binding.companyId }); const reason = "A later published review no longer requests human input."; const [retired] = await tx.update(issueThreadInteractions).set({ status: "cancelled", result: { version: 1, outcome: "withdrawn", reason }, resolvedAt: new Date(), updatedAt: new Date(), diff --git a/server/src/services/native-status-reassessment-notifications.ts b/server/src/services/native-status-reassessment-notifications.ts index 0b7ed9e1fdf..c49f600ce5f 100644 --- a/server/src/services/native-status-reassessment-notifications.ts +++ b/server/src/services/native-status-reassessment-notifications.ts @@ -1,3 +1,4 @@ +import { notifyIssueAttentionRetentionWork } from "./retention-work-notifications.js"; import { notifyStatusCardIssueChange } from "./status-card-notifications.js"; import { notifyIssueRecovery } from "./issue-recovery-notifications.js"; import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js"; @@ -5,6 +6,7 @@ import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notificatio export type NativeStatusReassessmentScope = { companyId: string; issueId?: string; runId?: string; agentId?: string }; export async function notifyNativeStatusReassessment(tx: object, scope: NativeStatusReassessmentScope) { + await notifyIssueAttentionRetentionWork(tx, { companyId: scope.companyId, issueId: scope.issueId, runId: scope.runId }); await notifyStatusCardIssueChange(tx, scope.companyId); await notifyIssueRecovery(tx, scope ? { companyId: scope.companyId } : undefined); return notifyDeliveryWork(tx, DELIVERY_QUEUES.nativeStatusReassessment, diff --git a/server/src/services/plugin-managed-agents.ts b/server/src/services/plugin-managed-agents.ts index e2e2c965e8c..91a81fb5120 100644 --- a/server/src/services/plugin-managed-agents.ts +++ b/server/src/services/plugin-managed-agents.ts @@ -1,3 +1,4 @@ +import { notifyAttentionRetentionWork } from "./retention-work-notifications.js"; import { createAgentLifecycle } from "./agent-lifecycle.js"; import { and, eq, isNull, ne } from "drizzle-orm"; import type { Db } from "@paperclipai/db"; @@ -340,6 +341,7 @@ export function pluginManagedAgentService( // and its canonical history. Transfer only its proven stable-key marker to // the replacement installation, atomically with the new scoped bindings. await upsertBinding(companyId, declaration, agentId, {}, adapterType, tx); + await notifyAttentionRetentionWork(tx, { companyId, sourceKind: "agent_error_alert" }); await tx.update(agents).set({ metadata: managedMetadata(options.pluginId, options.pluginKey, declaration, agent.metadata), updatedAt: new Date(), diff --git a/server/src/services/retention-work-notifications.ts b/server/src/services/retention-work-notifications.ts index ffedb0b1a4d..79cffd624a8 100644 --- a/server/src/services/retention-work-notifications.ts +++ b/server/src/services/retention-work-notifications.ts @@ -1,14 +1,89 @@ import { notifyApprovalWork } from "./approval-work-notifications.js"; +import { RECOVERY_ORIGIN_KINDS, parseIssueGraphLivenessIncidentKey } from "./recovery/origins.js"; +import { sql } from "drizzle-orm"; import type { Db } from "@paperclipai/db"; import type { AttentionSourceKind } from "@paperclipai/shared"; import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js"; -export async function notifyRetentionWork(transaction: object, kind: AttentionSourceKind) { +export const ATTENTION_RETENTION_MAX_ISSUE_IDS = 100; +export type AttentionRetentionScope = { companyId?: string; sourceKind?: AttentionSourceKind; + issueId?: string; issueIds?: string[]; sourceId?: string }; + +export async function notifyAttentionRetentionWork(transaction: object, scope: AttentionRetentionScope = {}) { + const issueIds = [...new Set([...(scope.issueIds ?? []), ...(scope.issueId ? [scope.issueId] : [])])].sort(); + await notifyDeliveryWork(transaction, DELIVERY_QUEUES.attentionRetention, JSON.stringify({ + companyId: scope.companyId, sourceKind: scope.sourceKind, sourceId: scope.sourceId, + issueId: issueIds.length === 1 ? issueIds[0] : undefined, + issueIds: issueIds.length > 1 && issueIds.length <= ATTENTION_RETENTION_MAX_ISSUE_IDS ? issueIds : undefined, + })); +} + +export async function notifyIssueAttentionRetentionWork(transaction: object, + scope: { companyId: string; issueId?: string; parentId?: string | null; runId?: string; originKind?: string | null; originId?: string | null }) { + const database = transaction as Db; + const seeds = [scope.issueId, scope.parentId].filter((id): id is string => Boolean(id)); + if (!seeds.length && scope.runId) { + const rows = await database.execute(sql`select issue_id::text as id from heartbeat_runs + where company_id = ${scope.companyId}::uuid and id = ${scope.runId}::uuid and issue_id is not null + union select native_issue_id::text from heartbeat_runs + where company_id = ${scope.companyId}::uuid and id = ${scope.runId}::uuid and native_issue_id is not null + union select issue.id::text from heartbeat_runs run inner join issues issue + on issue.company_id = run.company_id + and issue.id::text = coalesce(run.context_snapshot->>'issueId', run.context_snapshot->>'taskId') + where run.company_id = ${scope.companyId}::uuid and run.id = ${scope.runId}::uuid`); + for (const row of rows) seeds.push(row.id as string); + } + if (!seeds.length) return notifyAttentionRetentionWork(transaction, { companyId: scope.companyId }); + const origins = await database.execute(sql`select id::text as id, origin_kind as "originKind", origin_id as "originId" from issues + where company_id = ${scope.companyId}::uuid + and id in (${sql.join([...new Set(seeds)].map(id => sql`${id}::uuid`), sql`, `)})`); + const current = origins.find(source => source.id === scope.issueId); + origins.push({ originKind: scope.originKind ?? current?.originKind, originId: scope.originId ?? current?.originId }); + const references: string[] = []; + for (const source of origins) { + if (source.originKind === RECOVERY_ORIGIN_KINDS.strandedIssueRecovery && typeof source.originId === "string") { + references.push(source.originId); + } + const incident = source.originKind === RECOVERY_ORIGIN_KINDS.issueGraphLivenessEscalation + ? parseIssueGraphLivenessIncidentKey(source.originId as string | null) : null; + if (incident?.companyId === scope.companyId) references.push(incident.issueId, incident.leafIssueId); + } + if (references.length) { + const trusted = await database.execute(sql`select id::text as id from issues + where company_id = ${scope.companyId}::uuid + and id::text in (${sql.join([...new Set(references)].map(id => sql`${id}`), sql`, `)})`); + for (const row of trusted) seeds.push(row.id as string); + } + // UNION deduplicates visited UUIDs (including cycles). Each relation edge is + // unique by schema, so 101 neighbours guarantee overflow of a 100-ID scope. + // Bound that adjacency before materialization and stream at most 101 unique + // IDs to the caller; an inner UNION DISTINCT would consume the full fan-in. + const affected = await database.execute(sql`with recursive affected(id) as ( + select unnest(array[${sql.join([...new Set(seeds)].map(id => sql`${id}::uuid`), sql`, `)}]) + union + select parent.id from affected current cross join lateral ( + (select related_issue_id as id from issue_relations where company_id = ${scope.companyId}::uuid + and issue_id = current.id and type = 'blocks' limit ${ATTENTION_RETENTION_MAX_ISSUE_IDS + 1}) + union all select parent_id from issues where company_id = ${scope.companyId}::uuid + and id = current.id and parent_id is not null + ) parent + ) select id::text as "issueId" from affected limit ${ATTENTION_RETENTION_MAX_ISSUE_IDS + 1}`); + // The sentinel means the affected set is incomplete. Reconstruct the whole + // company through the existing paged authority instead of dropping ancestors. + await notifyAttentionRetentionWork(transaction, { companyId: scope.companyId, + issueIds: affected.length <= ATTENTION_RETENTION_MAX_ISSUE_IDS ? affected.map(row => row.issueId as string) : undefined, + }); +} + +export async function notifyRetentionWork(transaction: object, kind: AttentionSourceKind, scope: { companyId?: string; sourceId?: string } = {}) { if (kind === "approval") await notifyDeliveryWork(transaction, DELIVERY_QUEUES.approvalRetention); if (kind === "join_request") await notifyDeliveryWork(transaction, DELIVERY_QUEUES.joinRequestRetention); + if (kind !== "approval" && kind !== "join_request") await notifyAttentionRetentionWork(transaction, { ...scope, sourceKind: kind, + sourceId: kind === "blocker_attention" ? undefined : scope.sourceId }); } export async function notifyCompanyRetentionWork(transaction: object, companyId: string) { + await notifyAttentionRetentionWork(transaction, { companyId }); await notifyApprovalWork(transaction, companyId); await notifyRetentionWork(transaction, "join_request"); } diff --git a/server/src/services/slack-conversation-lifecycle.ts b/server/src/services/slack-conversation-lifecycle.ts index 9e784ff005d..b7754387c00 100644 --- a/server/src/services/slack-conversation-lifecycle.ts +++ b/server/src/services/slack-conversation-lifecycle.ts @@ -1,3 +1,4 @@ +import { notifyAttentionRetentionWork } from "./retention-work-notifications.js"; import { notifyStatusCardIssueChange } from "./status-card-notifications.js"; import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js"; import { notifyUnsafeWorkspaceExportRecovery } from "./unsafe-workspace-export-notifications.js"; @@ -127,6 +128,7 @@ export async function settleExternalConversation(db: Db, companyId: string, issu const ready = await readSettlement(tx, companyId, issueId, true); if (!ready) return false; const { binding, issue, run } = ready; + await notifyAttentionRetentionWork(tx, { companyId }); const changed = await tx.update(chatConversations).set({ state: "waiting", updatedAt: new Date() }) .where(and(eq(chatConversations.id, binding.conversation.id), eq(chatConversations.companyId, companyId), eq(chatConversations.state, "active"))) .returning({ id: chatConversations.id }); diff --git a/server/src/services/slack-conversation-state.ts b/server/src/services/slack-conversation-state.ts index 08f6c9c038b..622149badeb 100644 --- a/server/src/services/slack-conversation-state.ts +++ b/server/src/services/slack-conversation-state.ts @@ -1,3 +1,4 @@ +import { notifyAttentionRetentionWork } from "./retention-work-notifications.js"; import { notifyStatusCardIssueChange } from "./status-card-notifications.js"; import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js"; import { notifyExternalConversationSettlement } from "./external-conversation-settlement-notifications.js"; @@ -46,6 +47,7 @@ export function nonIdleSlackIssueCondition() { /** Called inside message admission while its transaction owns the issue lock. */ export async function resumeExternalConversation(tx: Db, companyId: string, issueId: string) { + await notifyAttentionRetentionWork(tx, { companyId }); await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers); await notifyExternalConversationSettlement(tx, { companyId, issueId }); // Only the verified idle convention may leave In Review automatically. diff --git a/server/src/services/workspace-operations.ts b/server/src/services/workspace-operations.ts index 7bd7e2ab658..67ff166b8d0 100644 --- a/server/src/services/workspace-operations.ts +++ b/server/src/services/workspace-operations.ts @@ -1,3 +1,4 @@ +import { notifyAttentionRetentionWork, notifyIssueAttentionRetentionWork } from "./retention-work-notifications.js"; import { notifyNativeResultCommit } from "./native-result-commit-notifications.js"; import { notifyNativeWorkspaceFinalization } from "./native-workspace-finalization-notifications.js"; import { notifyNativeStatusReassessment } from "./native-status-reassessment-notifications.js"; @@ -480,8 +481,14 @@ export function workspaceOperationService(db: Db) { async recordOperation(recordInput) { async function writeState(write: (database: Db) => Promise): Promise { - if (recordInput.phase !== "workspace_finalize" || !input.heartbeatRunId) return write(db); + if (recordInput.phase !== "workspace_finalize" || (!input.heartbeatRunId && !executionWorkspaceId && !input.issueId)) return write(db); return db.transaction(async tx => { + if (!input.heartbeatRunId) { + // An export clears barriers for every issue sharing this workspace. + if (executionWorkspaceId) await notifyAttentionRetentionWork(tx, { companyId: input.companyId, sourceKind: "blocker_attention" }); + else await notifyIssueAttentionRetentionWork(tx, { companyId: input.companyId, issueId: input.issueId ?? undefined }); + return write(tx as unknown as Db); + } await notifyNativeResultCommit(tx, { companyId: input.companyId, runId: input.heartbeatRunId!, issueId: input.issueId ?? undefined }); await notifyNativeWorkspaceFinalization(tx, { companyId: input.companyId,