diff --git a/doc/plans/2026-11-13-thousand-concurrent-agents.md b/doc/plans/2026-11-13-thousand-concurrent-agents.md index a17a2113b99..0d40f80967f 100644 --- a/doc/plans/2026-11-13-thousand-concurrent-agents.md +++ b/doc/plans/2026-11-13-thousand-concurrent-agents.md @@ -276,6 +276,23 @@ turn, so memory per session counts for idle agents too. | 1g | Size the shared limiters (log uploads, checkpoint pool, recovery pass) from the active run count | no fixed slot count below the run count | | 1h | Keep one frozen, parsed tool catalog per catalog hash and company, shared by the sessions that use it, in place of two private copies per session (`assigned-mcp-tools.ts:146`) | server memory per live or warm session from about 5–11 MB (measured at 400–800 tools) to near zero; at 1,000 agents that is 5–11 GB | +### Background recovery sweeps + +Run events used to send a company recovery hint, so the stranded-assignment +and assigned-work queues re-checked every open task of the company on each +run event (#21552). Local fixture: 1,500 open tasks, 60 agents, process agents +that fail at once, 180-second window that includes one 5-minute backstop pass. + +| Build | Runs created (/s) | Stranded checks per window | Assigned-work checks per window | `agent_task_sessions` reads (/s) | Statements (/s) | +| --- | --- | --- | --- | --- | --- | +| Before #21552 (54c3616ad9) | 1.09 | 4,327 | 245 | 26.5 | 1,431 | +| #21552 at dcd170e29f | 1.00 | 702 | 276 | 7.2 | 1,060 | +| master at 340157d1d6 | 0.24 | 330 | 72 | 2.5 | 409 | + +The master row has much less run churn than the earlier rows, so compare the +per-window totals with care. In that window the one backstop pass loaded 260 +tasks into the stranded queue; run events loaded 147. + ### Phase 2: several controller processes (target: 1,000) | # | Change | diff --git a/server/src/__tests__/attention-retention-work.test.ts b/server/src/__tests__/attention-retention-work.test.ts index cb08a3de0e8..fbbae5a232e 100644 --- a/server/src/__tests__/attention-retention-work.test.ts +++ b/server/src/__tests__/attention-retention-work.test.ts @@ -9,7 +9,8 @@ import { attentionRetentionService, registerAttentionRetentionWork } from "../se import { workspaceOperationService } from "../services/workspace-operations.js"; import * as workspaceOperationLogs from "../services/workspace-operation-log-store.js"; import { budgetService, flushBudgetEvaluationWork } from "../services/budgets.js"; -import { writeExecutionControl } from "../services/execution-control-notifications.js"; +import { notifyExecutionRunWork, writeExecutionControl } from "../services/execution-control-notifications.js"; +import { createHeartbeatAgentStatus } from "../services/heartbeat/agent-status.js"; import { DELIVERY_QUEUES } from "../services/delivery-work-notifications.js"; import { attentionService } from "../services/attention.js"; import { companyService } from "../services/companies.js"; @@ -26,6 +27,7 @@ import { idleWorkSnapshot } from "../services/task-admission.js"; import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js"; +import { createFullDatabaseResponseCapture, installDatabaseResponseCapture } from "./helpers/full-database-response-capture.js"; const externalUrl = resolveExternalTestDatabaseUrl("PAPERCLIP_ATTENTION_RETENTION_TEST_DATABASE_URL"); const support = externalUrl ? { supported: true } : await getEmbeddedPostgresTestSupport(); @@ -249,6 +251,45 @@ const delay = DEFAULT_DECISION_ARCHIVE_DAYS * 86_400_000; expect(await db.select().from(decisionArchiveNotificationOutbox)).toHaveLength(0); }); + it("reads only the fields its decisions use from large runs, reviews and agents", async () => { + const payload = "saved-execution-input".repeat(2_000); + await db.update(agents).set({ adapterConfig: { instructions: payload }, runtimeConfig: { notes: payload } }).where(eq(agents.id, agentId)); + const origins = await db.insert(issues).values(Array.from({ length: 50 }, () => ({ companyId, + title: randomUUID(), status: "in_review", assigneeAgentId: agentId, description: payload, updatedAt: old() }))).returning(); + const runs = await db.insert(heartbeatRuns).values(origins.map(origin => ({ companyId, agentId, status: "failed", + contextSnapshot: { issueId: origin.id, prompt: payload }, runnerProfileJson: { instructions: payload }, + stdoutExcerpt: payload, createdAt: old(), finishedAt: old() }))).returning({ id: heartbeatRuns.id }); + await db.insert(heartbeatRunEvents).values(runs.map(run => ({ companyId, agentId, runId: run.id, + seq: 1, eventType: "lifecycle", message: "Bounded retry exhausted: byte fixture" }))); + const [paused] = await db.insert(agents).values({ companyId, name: "Paused", role: "engineer", status: "paused", + adapterConfig: { instructions: payload } }).returning({ id: agents.id }); + await db.insert(issueThreadInteractions).values(origins.slice(0, 10).map(row => ({ companyId, issueId: row.id, + kind: "ask_user_questions" as const, addresseeAgentId: paused.id, payload: { version: 1 as const, questions: [] } }))); + const observed = createDb(externalUrl ?? temporary!.connectionString), events: Array> = [], now = Date.now(); + const capture = createFullDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event)); + installDatabaseResponseCapture(observed, { heartbeatRuns, issues, agents }, capture); + const decisions = new Map(); + try { + const service = attentionRetentionService(observed); + for (const sourceKind of ["failed_run", "review", "issue_thread_interaction"] as const) { + for (let page = 0; page < 3; page++) { + const loaded = await service.load({ companyId, sourceKind }); + for (const entry of loaded.rows ?? []) decisions.set(entry.id, `${entry.sourceKind}:${entry.dueAt}:${entry.activityAt}`); + if (loaded.complete) break; + } + } + capture.flush(); + } finally { capture.flush(); await observed.$client.end(); } + const bytes = (table: string) => events.filter(event => event.table === table && event.operation === "select") + .reduce((total, event) => total + Number(event.responseValueBytes), 0); + const kinds = [...decisions.values()].reduce((counts, value) => { + const kind = value.split(":")[0]; counts[kind] = (counts[kind] ?? 0) + 1; return counts; }, {} as Record); + expect(kinds).toEqual({ failed_run: 50, review: 40, issue_thread_interaction: 10 }); + expect(bytes("heartbeat_runs")).toBeLessThan(20_000); + expect(bytes("agents")).toBeLessThan(5_000); + expect(bytes("issues")).toBeLessThan(60_000); + }); + it("batches 100 failed runs and rechecks late visibility and supersession before archive", async () => { const origins = await db.insert(issues).values(Array.from({ length: 100 }, () => ({ companyId, title: randomUUID(), status: "todo", updatedAt: old() }))).returning(); @@ -877,6 +918,24 @@ const delay = DEFAULT_DECISION_ARCHIVE_DAYS * 86_400_000; expect(hints).toEqual([scoped, scoped]); }); + it("scopes run lifecycle attention hints to the run's task graph and agent status to agent alerts", async () => { + const parent = await insertIssue("blocked"); + const issue = await insertIssue("todo", { parentId: parent.id }); + const [run] = await db.insert(heartbeatRuns).values({ companyId, agentId, status: "failed", + contextSnapshot: { issueId: issue.id } }).returning(); + const hints: string[] = []; + const unsubscribe = subscribeDatabaseWork(db, DELIVERY_QUEUES.attentionRetention, (event, values) => { + if (event === "settled") hints.push(...values ?? []); + }); + try { + await writeExecutionControl(db, DELIVERY_QUEUES.executionFinalization, async () => undefined, { companyId, runId: run.id }); + await db.transaction(tx => notifyExecutionRunWork(tx, { companyId, runId: run.id })); + await createHeartbeatAgentStatus(db).finalizeAgentStatus(agentId, "failed"); + } finally { unsubscribe(); } + const scoped = JSON.stringify({ companyId, issueIds: [issue.id, parent.id].sort() }); + expect(hints).toEqual([scoped, scoped, JSON.stringify({ companyId, sourceKind: "agent_error_alert" })]); + }); + it("accepts ordinary liveness incident creation and completion with a none leaf sentinel", async () => { const source = await insertIssue("blocked", { responsibleUserId: "board" }); const service = issueService(db); diff --git a/server/src/services/attention-retention.ts b/server/src/services/attention-retention.ts index 568247e329f..930d27eeba6 100644 --- a/server/src/services/attention-retention.ts +++ b/server/src/services/attention-retention.ts @@ -1,8 +1,8 @@ -import { and, eq, getTableColumns, inArray, isNull, sql, type SQL } from "drizzle-orm"; +import { and, eq, inArray, isNull, sql, type SQL } from "drizzle-orm"; import { agents, decisionRetention, decisions, heartbeatRuns, issueApprovals, issueRecoveryActions, issueThreadInteractions, issues, approvals, type Db } from "@paperclipai/db"; import { evaluateAgentInvokability } from "./agent-invokability.js"; -import { HUMAN_RECOVERY_OWNER_TYPES, ATTENTION_BUDGET_WARNING_PERCENT, ATTENTION_AGENT_ERROR_STATUS, collapsePendingConfirmationsToNewest, listFailedRunsWithNewerRun, loadNewestSettledConfirmationAt, readRunIssueId, requiresReviewAttention, terminalBlockerRequiresAttention } from "./attention-source-eligibility.js"; +import { HUMAN_RECOVERY_OWNER_TYPES, ATTENTION_BUDGET_WARNING_PERCENT, ATTENTION_AGENT_ERROR_STATUS, collapsePendingConfirmationsToNewest, listFailedRunsWithNewerRun, loadNewestSettledConfirmationAt, requiresReviewAttention, terminalBlockerRequiresAttention } from "./attention-source-eligibility.js"; import { decisionRetentionService, DEFAULT_DECISION_ARCHIVE_DAYS } from "./decision-retention.js"; import type { createDeliveryWorkCoordinator } from "./delivery-work-coordinator.js"; import { DELIVERY_QUEUES } from "./delivery-work-notifications.js"; @@ -23,12 +23,22 @@ type Candidate = { sourceKind: Kind; sourceId: string; companyId: string; activi type Entry = Candidate & { id: string; dueAt: number; rootId: string; variant?: "owner" | "terminal" }; type Cursor = { sourceKind: string; sourceId: string }; -type PageIssue = typeof issues.$inferSelect & { activityAt: string; deadlineAt: number; +const pageIssueColumns = { id: issues.id, companyId: issues.companyId, parentId: issues.parentId, identifier: issues.identifier, + title: issues.title, status: issues.status, assigneeAgentId: issues.assigneeAgentId, assigneeUserId: issues.assigneeUserId, + executionRunId: issues.executionRunId, unblockDescriptor: issues.unblockDescriptor, blockedTransitionAt: issues.blockedTransitionAt }; +const contextSnapshot = heartbeatRuns.contextSnapshot; +const runProbeColumns = { id: heartbeatRuns.id, agentId: heartbeatRuns.agentId, createdAt: heartbeatRuns.createdAt, + issueId: sql`case when jsonb_typeof(${contextSnapshot}->'issueId') is not null and jsonb_typeof(${contextSnapshot}->'issueId') <> 'null' + then case when jsonb_typeof(${contextSnapshot}->'issueId') = 'string' then nullif(${contextSnapshot}->>'issueId', '') end + else case when jsonb_typeof(${contextSnapshot}->'taskId') = 'string' then nullif(${contextSnapshot}->>'taskId', '') end end` }; +type VisibleIssue = { [K in keyof typeof pageIssueColumns]: (typeof issues.$inferSelect)[K] }; +type PageIssue = VisibleIssue & { activityAt: string; deadlineAt: number; blockedActivityAt: string | null; blockedDeadlineAt: number | null }; +type RunProbe = { id: string; agentId: string; createdAt: Date; issueId: string | null }; type PageEvidence = { decisions: Map; - recoveries: Map; - runs: Map; + recoveries: Map; + runs: Map; agents: Set; visibleIssues: Map; newerRuns: Set; @@ -127,7 +137,7 @@ export function attentionRetentionService(db: Db) { const candidates = listAttentionRetentionCandidates; async function visibleIssue(database: Db, companyId: string, issueId: string, conversations = false) { - const [row] = await database.select().from(issues).where(and(eq(issues.companyId, companyId), eq(issues.id, issueId), + const [row] = await database.select(pageIssueColumns).from(issues).where(and(eq(issues.companyId, companyId), eq(issues.id, issueId), attentionIssueCondition(conversations))); return row; } @@ -149,7 +159,8 @@ export function attentionRetentionService(db: Db) { const visible = await database.select({ id: issues.id }).from(issues).where(and(eq(issues.companyId, companyId), inArray(issues.id, rows.map(row => row.issueId)), attentionIssueCondition(true))); const visibleIds = new Set(visible.map(row => row.id)); - const roster = await database.select().from(agents).where(eq(agents.companyId, companyId)); + const roster = await database.select({ id: agents.id, companyId: agents.companyId, name: agents.name, + reportsTo: agents.reportsTo, status: agents.status }).from(agents).where(eq(agents.companyId, companyId)); const boardAddressed = (row: typeof issueThreadInteractions.$inferSelect) => !row.addresseeAgentId || !evaluateAgentInvokability(roster.find(agent => agent.id === row.addresseeAgentId), roster).invokable; const confirmationIssueIds = rows.filter(row => row.kind === "request_confirmation").map(row => row.issueId); @@ -175,7 +186,8 @@ export function attentionRetentionService(db: Db) { byCompany.set(candidate.companyId, ids); } for (const [companyId, ids] of byCompany) { - const rows = await database.select().from(issues).where(and(eq(issues.companyId, companyId), + const rows = await database.select({ id: issues.id, companyId: issues.companyId, status: issues.status, + executionState: issues.executionState, assigneeUserId: issues.assigneeUserId }).from(issues).where(and(eq(issues.companyId, companyId), inArray(issues.id, ids), attentionIssueCondition(false))); const pendingApprovals = await database.select({ issueId: issueApprovals.issueId }).from(issueApprovals) .innerJoin(approvals, eq(approvals.id, issueApprovals.approvalId)).where(and( @@ -209,14 +221,15 @@ export function attentionRetentionService(db: Db) { const decisionIds = ids("decision"), recoveryIds = ids("recovery_action"), runIds = ids("failed_run"), blockerIds = ids("blocker_attention"); const decisionRows = decisionIds.length ? await database.select({ id: decisions.id, originIssueId: decisions.originIssueId }) .from(decisions).where(and(eq(decisions.companyId, companyId), inArray(decisions.id, decisionIds), eq(decisions.status, "open"))) : []; - const recoveryRows = recoveryIds.length ? await database.select().from(issueRecoveryActions).where(and( + const recoveryRows = recoveryIds.length ? await database.select({ id: issueRecoveryActions.id, + sourceIssueId: issueRecoveryActions.sourceIssueId }).from(issueRecoveryActions).where(and( eq(issueRecoveryActions.companyId, companyId), inArray(issueRecoveryActions.id, recoveryIds))) : []; - const runRows = runIds.length ? await database.select().from(heartbeatRuns).where(and( + const runRows: RunProbe[] = runIds.length ? await database.select(runProbeColumns).from(heartbeatRuns).where(and( eq(heartbeatRuns.companyId, companyId), inArray(heartbeatRuns.id, runIds))) : []; const runAgents = runRows.length ? await database.select({ id: agents.id }).from(agents).where(and( eq(agents.companyId, companyId), inArray(agents.id, [...new Set(runRows.map(row => row.agentId))]), sql`${agents.status} <> 'terminated'`)) : []; - const readIssues = async (issueIds: string[]) => issueIds.length ? database.select({ ...getTableColumns(issues), + const readIssues = async (issueIds: string[]) => issueIds.length ? database.select({ ...pageIssueColumns, activityAt: sql`${issues.updatedAt}::text`, deadlineAt: sql`ceil(extract(epoch from ${issues.updatedAt}) * 1000 + ${archiveDelay})`.mapWith(Number), blockedActivityAt: sql`${issues.blockedTransitionAt}::text`, @@ -224,7 +237,7 @@ export function attentionRetentionService(db: Db) { }).from(issues).where(and(eq(issues.companyId, companyId), inArray(issues.id, [...new Set(issueIds)]), attentionIssueCondition(false))) : []; const issueIds = [...blockerIds, ...decisionRows.map(row => row.originIssueId), ...recoveryRows.map(row => row.sourceIssueId), - ...runRows.map(row => readRunIssueId(row.contextSnapshot))].filter((id): id is string => Boolean(id)); + ...runRows.map(row => row.issueId)].filter((id): id is string => Boolean(id)); const visibleRows = await readIssues(issueIds); const visible = new Map(visibleRows.map(row => [row.id, row])); const blockerRoots = visibleRows.filter(row => blockerIds.includes(row.id)); @@ -238,7 +251,7 @@ export function attentionRetentionService(db: Db) { decisions: new Map(decisionRows.map(row => [row.id, row])), recoveries: new Map(recoveryRows.map(row => [row.id, row])), runs: new Map(runRows.map(row => [row.id, row])), agents: new Set(runAgents.map(row => row.id)), visibleIssues: visible, newerRuns: await listFailedRunsWithNewerRun(database, companyId, - runRows.map(run => ({ ...run, issueId: readRunIssueId(run.contextSnapshot) }))), blockers, deferredRoots: blockerBatch.deferredRootIds, + runRows), blockers, deferredRoots: blockerBatch.deferredRootIds, }); } return result; @@ -261,7 +274,8 @@ export function attentionRetentionService(db: Db) { const attention = interactionAttention ?? await interactionEligibility(database, [candidate]); if (!attention.get(sourceId)) return null; } else if (candidate.sourceKind === "recovery_action") { - const row = evidence ? evidence.recoveries.get(sourceId) : (await database.select().from(issueRecoveryActions).where(and(eq(issueRecoveryActions.companyId, candidate.companyId), eq(issueRecoveryActions.id, sourceId))))[0]; + const row = evidence ? evidence.recoveries.get(sourceId) : (await database.select({ id: issueRecoveryActions.id, + sourceIssueId: issueRecoveryActions.sourceIssueId }).from(issueRecoveryActions).where(and(eq(issueRecoveryActions.companyId, candidate.companyId), eq(issueRecoveryActions.id, sourceId))))[0]; if (!row || !(evidence ? evidence.visibleIssues.has(row.sourceIssueId) : await visibleIssue(database, candidate.companyId, row.sourceIssueId))) return null; } else if (candidate.sourceKind === "review") { @@ -297,14 +311,14 @@ export function attentionRetentionService(db: Db) { eq(decisionRetention.sourceKind, candidate.sourceKind), eq(decisionRetention.sourceId, sourceId))); if (retention?.keep || retention?.archivedAt) return null; } else if (candidate.sourceKind === "failed_run") { - const run = evidence ? evidence.runs.get(sourceId) : (await database.select().from(heartbeatRuns).where(and(eq(heartbeatRuns.companyId, candidate.companyId), eq(heartbeatRuns.id, sourceId))))[0]; + const run: RunProbe | undefined = evidence ? evidence.runs.get(sourceId) : (await database.select(runProbeColumns).from(heartbeatRuns).where(and(eq(heartbeatRuns.companyId, candidate.companyId), eq(heartbeatRuns.id, sourceId))))[0]; if (!run) return null; const agent = evidence ? evidence.agents.has(run.agentId) : (await database.select({ id: agents.id }).from(agents).where(and(eq(agents.companyId, candidate.companyId), eq(agents.id, run.agentId), sql`${agents.status} <> 'terminated'`)))[0]; - const issueId = readRunIssueId(run.contextSnapshot); + const issueId = run.issueId; if (!agent || issueId && !(evidence ? evidence.visibleIssues.has(issueId) : await visibleIssue(database, candidate.companyId, issueId))) return null; - if ((evidence ? evidence.newerRuns : await listFailedRunsWithNewerRun(database, candidate.companyId, [{ ...run, issueId }])).has(run.id)) return null; + if ((evidence ? evidence.newerRuns : await listFailedRunsWithNewerRun(database, candidate.companyId, [run])).has(run.id)) return null; } return { ...candidate, sourceId, activityAt, rootId: candidate.sourceId, variant, id: `${candidate.sourceKind}:${candidate.sourceId}:${variant ?? "source"}`, dueAt }; diff --git a/server/src/services/execution-control-notifications.ts b/server/src/services/execution-control-notifications.ts index a94ac36ec91..d01592d1da4 100644 --- a/server/src/services/execution-control-notifications.ts +++ b/server/src/services/execution-control-notifications.ts @@ -10,7 +10,7 @@ import { DELIVERY_QUEUES, notifyDeliveryWork, type DeliveryQueue } from "./deliv export type ExecutionWorkScope = { companyId: string; agentId?: string; runId?: string; issueId?: string }; async function notifyExecutionAttentionRetention(transaction: object, scope: ExecutionWorkScope) { - if (scope.issueId) await notifyIssueAttentionRetentionWork(transaction, { companyId: scope.companyId, issueId: scope.issueId }); + if (scope.issueId || scope.runId) await notifyIssueAttentionRetentionWork(transaction, { companyId: scope.companyId, issueId: scope.issueId, runId: scope.runId }); else await notifyAttentionRetentionWork(transaction, scope); } diff --git a/server/src/services/heartbeat/agent-status.ts b/server/src/services/heartbeat/agent-status.ts index 622e796b700..0d03f6ac828 100644 --- a/server/src/services/heartbeat/agent-status.ts +++ b/server/src/services/heartbeat/agent-status.ts @@ -94,7 +94,7 @@ export function createHeartbeatAgentStatus(db: Db) { const nextStatus = running ? "running" : settledStatus(outcome, options?.keepIdleOnFailure); const now = new Date(); await notifyStrandedAgentStatus(tx, { companyId: existing.companyId, agentId }); - await notifyAttentionRetentionWork(tx, { companyId: existing.companyId }); + await notifyAttentionRetentionWork(tx, { companyId: existing.companyId, sourceKind: "agent_error_alert" }); await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers); const [updated] = await tx.update(agents).set({ status: nextStatus, @@ -152,7 +152,7 @@ export function createHeartbeatAgentStatus(db: Db) { eq(heartbeatRuns.agentId, candidate.id))) .orderBy(desc(heartbeatRuns.finishedAt), desc(heartbeatRuns.createdAt), desc(heartbeatRuns.id)).limit(1); const nextStatus = settledStatus(lastRun?.status); - await notifyAttentionRetentionWork(tx, { companyId: candidate.companyId }); + await notifyAttentionRetentionWork(tx, { companyId: candidate.companyId, sourceKind: "agent_error_alert" }); const [agent] = await tx.update(agents).set({ status: nextStatus, errorReason: nextStatus === "error" ? errorReason(lastRun?.error) : null, updatedAt: now, }).where(and(eq(agents.id, candidate.id), eq(agents.companyId, candidate.companyId), diff --git a/server/src/services/heartbeat/retries.ts b/server/src/services/heartbeat/retries.ts index c463a458a5b..7425f402af0 100644 --- a/server/src/services/heartbeat/retries.ts +++ b/server/src/services/heartbeat/retries.ts @@ -3,7 +3,7 @@ import { runOwnerFenceHeldSql } from "../remote-workspace-writer-hold.js"; import { AGENT_DIRECTORY_BUSY_ERROR_CODE, AGENT_DIRECTORY_BUSY_MAX_DEFERRALS, AGENT_DIRECTORY_BUSY_RETRY_REASON, AGENT_DIRECTORY_BUSY_RETRY_WAKE_REASON, agentDirectoryBusyRetryDelayMs, type AgentDirectoryPreparationDeferral } from "./agent-directory-busy.js"; import { isInterruptedNativePreparation } from "../cancelled-native-startup.js"; import { isPreDispatchExecutionWaitVerified } from "../pre-dispatch-execution-wait.js"; -import { notifyAttentionRetentionWork } from "../retention-work-notifications.js"; +import { notifyIssueAttentionRetentionWork } from "../retention-work-notifications.js"; import { normalizeAgentNameKey } from "../agent-name-key.js"; import { notifyStatusCardIssueChange } from "../status-card-notifications.js"; import { notifyScheduledRetryDispatch } from "../run-dispatch-notifications.js"; @@ -1209,7 +1209,7 @@ export function createHeartbeatRetries(db: Db, dependencies: HeartbeatRetryDepen } } - await notifyAttentionRetentionWork(tx, { companyId: run.companyId }); + await notifyIssueAttentionRetentionWork(tx, { companyId: run.companyId, runId: run.id }); await notifyScheduledRetryDispatch(tx, { companyId: run.companyId, runId: run.id }); const wakeupRequest = await tx .insert(agentWakeupRequests) diff --git a/server/src/services/native-runtime/paperclip-runner-tool-authority.ts b/server/src/services/native-runtime/paperclip-runner-tool-authority.ts index 61f790329f4..8b61d8fcb8a 100644 --- a/server/src/services/native-runtime/paperclip-runner-tool-authority.ts +++ b/server/src/services/native-runtime/paperclip-runner-tool-authority.ts @@ -10,7 +10,7 @@ import { createCostEventInTransaction } from "../costs.js"; import { withOperationAccountingTransaction } from "../accounting-transaction.js"; import { notifyStatusCardIssueChange } from "../status-card-notifications.js"; import { nativeRunKeyScope, nativeSkillTestToolAllowed, assertNativeSkillTestTool } from "./native-skill-test-scope.js"; -import { notifyAttentionRetentionWork } from "../retention-work-notifications.js"; +import { notifyIssueAttentionRetentionWork } from "../retention-work-notifications.js"; import { requiredQualityReviewSchema, type RequestCheckboxConfirmationOption } from "@paperclipai/shared"; import { priorRecoveryApiReceipt } from "../recovery-effect-receipts.js"; import { DELIVERY_QUEUES, notifyDeliveryWork } from "../delivery-work-notifications.js"; @@ -2560,7 +2560,7 @@ export class PaperclipRunnerToolAuthority { replayMonitor = summarizeIssueMonitor(target, normalizeIssueExecutionPolicy(target.executionPolicy)); return { taskId: target.id, status: target.status, monitor: replayMonitor }; } - await notifyAttentionRetentionWork(tx, { companyId: this.binding.companyId }); + await notifyIssueAttentionRetentionWork(tx, { companyId: this.binding.companyId, issueId: target.id }); const patch = prepareIssueMonitorUpdate(target, input.monitor, actor); await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors); await notifyStatusCardIssueChange(tx, this.binding.companyId); diff --git a/server/src/services/retention-work-notifications.ts b/server/src/services/retention-work-notifications.ts index 79cffd624a8..e395e5fe4b4 100644 --- a/server/src/services/retention-work-notifications.ts +++ b/server/src/services/retention-work-notifications.ts @@ -6,6 +6,7 @@ import type { AttentionSourceKind } from "@paperclipai/shared"; import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js"; export const ATTENTION_RETENTION_MAX_ISSUE_IDS = 100; +const UUID_PATTERN = "^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$"; export type AttentionRetentionScope = { companyId?: string; sourceKind?: AttentionSourceKind; issueId?: string; issueIds?: string[]; sourceId?: string }; @@ -27,9 +28,10 @@ export async function notifyIssueAttentionRetentionWork(transaction: object, where company_id = ${scope.companyId}::uuid and id = ${scope.runId}::uuid and issue_id is not null union select native_issue_id::text from heartbeat_runs where company_id = ${scope.companyId}::uuid and id = ${scope.runId}::uuid and native_issue_id is not null - union select issue.id::text from heartbeat_runs run inner join issues issue - on issue.company_id = run.company_id - and issue.id::text = coalesce(run.context_snapshot->>'issueId', run.context_snapshot->>'taskId') + union select issue.id::text from heartbeat_runs run + cross join lateral (select coalesce(run.context_snapshot->>'issueId', run.context_snapshot->>'taskId') as saved) context + inner join issues issue on issue.company_id = run.company_id + and issue.id = case when context.saved ~* ${UUID_PATTERN} then context.saved::uuid end where run.company_id = ${scope.companyId}::uuid and run.id = ${scope.runId}::uuid`); for (const row of rows) seeds.push(row.id as string); }