From dd49318d554252ed0cd52134f60caa571aa2cf96 Mon Sep 17 00:00:00 2001 From: Dotta Date: Fri, 30 Oct 2026 16:01:08 -0500 Subject: [PATCH 01/74] fix(server): settle routine status from committed work Co-Authored-By: Paperclip --- doc/plans/2026-10-30-routine-status-sleep.md | 54 +++++ .../routine-execution-status-work.test.ts | 206 ++++++++++++++++++ .../routine-execution-status-worker.test.ts | 86 ++++++++ .../routine-status-reconciliation.test.ts | 6 +- server/src/index.ts | 17 +- .../services/delivery-work-notifications.ts | 1 + .../execution-control-notifications.ts | 5 +- .../src/services/heartbeat/run-lifecycle.ts | 3 +- .../issue-review-state-notifications.ts | 2 + server/src/services/issues.ts | 11 + .../native-routine-status.test.ts | 6 +- .../routine-execution-status-notifications.ts | 5 + .../services/routine-execution-status-work.ts | 18 ++ server/src/services/routine-run-status.ts | 68 +++++- server/src/services/routines.ts | 41 +--- 15 files changed, 475 insertions(+), 54 deletions(-) create mode 100644 doc/plans/2026-10-30-routine-status-sleep.md create mode 100644 server/src/__tests__/routine-execution-status-work.test.ts create mode 100644 server/src/__tests__/routine-execution-status-worker.test.ts create mode 100644 server/src/services/routine-execution-status-notifications.ts create mode 100644 server/src/services/routine-execution-status-work.ts diff --git a/doc/plans/2026-10-30-routine-status-sleep.md b/doc/plans/2026-10-30-routine-status-sleep.md new file mode 100644 index 00000000000..ea6f9364999 --- /dev/null +++ b/doc/plans/2026-10-30-routine-status-sleep.md @@ -0,0 +1,54 @@ +# Routine execution status settlement + +The Cloud routine status worker repairs a routine invocation when its linked +execution task and saved routine status disagree. The previous heartbeat callback +scanned every five minutes, even with no routines or only completed history. + +## Ownership and trigger + +`routine-run-status.ts` owns the projection, custody checks and repair transaction. +The worker loads actual mismatches at startup. It accepts issue hints from issue +creation, status/origin changes, checkout/release, routine issue association, +execution control writes and review state writes. It uses the existing +transaction-aware work notification protocol. An outer transaction owns the hint; +a savepoint release does not wake the repair worker before commit. + +A standard execution must name the same routine, invocation, company and linked +task. Plugin-managed executions may use their declared custom origin. Repair +locks the routine and task and rechecks the linked invocation before writing. +It preserves skipped/coalesced history and unrelated dispatch failures. A task +that resumes clears an execution-task transient failure through the same owner +used by native completion and issue routes. + +`activeIssue` remains derived from open tasks and live heartbeat runs. A terminal +heartbeat with an open task does not create a completion obligation. The worker +does not retry merely because a run is active or a routine exists. + +## Lifecycle and writer contract + +`registerScopedDeadlineWork` owns coalescing, fair batches, failure backoff, drain +and shutdown. A failed owner retries with bounded backoff. Successful settlement +removes that owner. Settled configuration, healthy active tasks and completed +history leave no recurring worker SQL. The existing Cloud and heartbeat scheduler +enablement gates remain in place. Ordinary routine scheduling is unchanged. + +Supported app writers register `notifyRoutineExecutionStatus(tx, issueId)` before +their mutation. Common execution writers without an issue hint request debt +reconstruction. Backups and independent SQL import/repair tools must restore +before startup or call that same transaction contract; process-local hints do +not observe independent SQL or another process. No permanent listener or periodic +configured-routine scan is added. + +The in-process retry deadline does not establish external Cloud wake coverage. +This change preserves the existing host sleep safety checks and does not claim +measured host or database suspension. + +## Acceptance + +Tests exercise ordinary issue writes and a real heartbeat lifecycle transition +against PostgreSQL with the production coordinator. Provider wakeup is simulated. +They inspect durable routine rows and the normal routine detail read separately. +Coverage includes empty/configured/history quiet periods, first producer, +terminal outcomes and reopen, legacy mismatch reconstruction, skipped/coalesced +custody, outer commit/rollback, retry/restart, drain/resume and standby/disable. +Focused worker tests exercise in-flight invalidations, retry isolation and stop. diff --git a/server/src/__tests__/routine-execution-status-work.test.ts b/server/src/__tests__/routine-execution-status-work.test.ts new file mode 100644 index 00000000000..7415b63bfd5 --- /dev/null +++ b/server/src/__tests__/routine-execution-status-work.test.ts @@ -0,0 +1,206 @@ +import { randomUUID } from "node:crypto"; +import { eq, sql } from "drizzle-orm"; +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; +import { agents, companies, createDb, heartbeatRuns, issues, pluginManagedResources, plugins, routineRuns, type Db } from "@paperclipai/db"; +import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; +import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js"; +import { createDeliveryWorkCoordinator } from "../services/delivery-work-coordinator.js"; +import { notifyRoutineExecutionStatus } from "../services/routine-execution-status-notifications.js"; +import { registerRoutineExecutionStatusWork } from "../services/routine-execution-status-work.js"; +import { routineExecutionStatusService } from "../services/routine-run-status.js"; +import { routineService } from "../services/routines.js"; +import { issueService } from "../services/issues.js"; +import { idleWorkSnapshot } from "../services/task-admission.js"; +import { createHeartbeatLifecycle, type HeartbeatLifecycleDependencies } from "../services/heartbeat/run-lifecycle.js"; +import { createHeartbeatRunState } from "../services/heartbeat/run-state.js"; +import { createHeartbeatRunPreparation } from "../services/heartbeat/run-preparation.js"; + +const external = resolveExternalTestDatabaseUrl("PAPERCLIP_ROUTINE_STATUS_WORK_TEST_DATABASE_URL"); +const support = external ? { supported: true } : await getEmbeddedPostgresTestSupport(); +(support.supported ? describe : describe.skip)("routine execution status work with PostgreSQL", () => { + let db: Db, temporary: Awaited> | undefined; + let coordinator: ReturnType | undefined; + let enabled = true, draining = false, standby = false; + const errors: unknown[] = []; + beforeAll(async () => { + if (!external) temporary = await startEmbeddedPostgresTestDatabase("routine-status-work-"); + db = createDb(external ?? temporary!.connectionString, { maxConnections: 6 }); + }, 60_000); + beforeEach(() => { + enabled = true; draining = false; standby = false; errors.length = 0; + vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"], shouldAdvanceTime: true }); + }); + afterEach(async () => { + await coordinator?.stop(); coordinator = undefined; + vi.useRealTimers(); vi.restoreAllMocks(); + await db.execute(sql`SET client_min_messages = warning; TRUNCATE companies CASCADE`); + }); + afterAll(async () => { await db?.$client.end(); await temporary?.cleanup(); }); + const canRun = () => enabled && !draining && !standby; + async function start(service = routineExecutionStatusService(db)) { + coordinator = createDeliveryWorkCoordinator({ owner: db, canRun, canReconcile: () => !standby, + onError: error => errors.push(error) }); + const worker = registerRoutineExecutionStatusWork(coordinator, service, canRun); + await worker.ready; + return worker; + } + async function settled() { await vi.waitFor(() => expect(idleWorkSnapshot().active).toBe(0)); } + async function quiet() { + await settled(); + const select = vi.spyOn(db, "select"), execute = vi.spyOn(db, "execute"), transaction = vi.spyOn(db, "transaction"); + try { + await vi.advanceTimersByTimeAsync(20 * 60_000); + expect(select).not.toHaveBeenCalled(); expect(execute).not.toHaveBeenCalled(); expect(transaction).not.toHaveBeenCalled(); + } finally { select.mockRestore(); execute.mockRestore(); transaction.mockRestore(); } + } + async function fixture() { + const companyId = randomUUID(), agentId = randomUUID(); + await db.insert(companies).values({ id: companyId, name: "Routine status work", issuePrefix: `R${companyId.slice(0, 6)}`, + defaultResponsibleUserId: randomUUID(), requireBoardApprovalForNewAgents: false }); + await db.insert(agents).values({ id: agentId, companyId, name: "Routine worker", status: "active", lifecycleState: "ready", adapterType: "codex_local" }); + const service = routineService(db, { heartbeat: { wakeup: async () => null }, runtimeEnv: {} }); + const routine = await service.create(companyId, { title: "Status projection", assigneeAgentId: agentId, + priority: "medium", status: "active", concurrencyPolicy: "always_enqueue", catchUpPolicy: "skip_missed" }, {}); + return { companyId, agentId, routine, service }; + } + async function dispatch(f: Awaited>) { + return f.service.runRoutine(f.routine.id, { source: "manual", idempotencyKey: randomUUID() }); + } + async function saved(id: string) { return (await db.select().from(routineRuns).where(eq(routineRuns.id, id)))[0]; } + async function projected(id: string, status: string) { + await vi.waitFor(async () => expect((await saved(id)).status).toBe(status)); + await settled(); + } + + it("settles empty startup, configured idle, healthy active and retained terminal history without recurring SQL", async () => { + await start(); await quiet(); + const f = await fixture(); await quiet(); + const run = await dispatch(f); await quiet(); + await issueService(db).update(run.linkedIssueId!, { status: "done" }); + await projected(run.id, "completed"); + const before = await saved(run.id); await quiet(); + expect(await saved(run.id)).toEqual(before); expect(errors).toEqual([]); + }); + it.each(["done", "blocked", "cancelled"])("reactivates from the first ordinary %s transition and preserves input", async status => { + await start(); await quiet(); + const f = await fixture(), run = await dispatch(f); + await issueService(db).update(run.linkedIssueId!, { status }); + await projected(run.id, status === "done" ? "completed" : "failed"); + expect((await saved(run.id)).completedAt).toBeInstanceOf(Date); + await quiet(); expect(errors).toEqual([]); + if (status !== "done") { + await issueService(db).update(run.linkedIssueId!, { status: "in_progress" }); + await projected(run.id, "issue_created"); + expect(await saved(run.id)).toMatchObject({ completedAt: null, failureReason: null }); + await quiet(); + } + }); + it("reconstructs legacy terminal debt and reopened transient failures but leaves skipped and coalesced history unchanged", async () => { + const f = await fixture(), first = await dispatch(f), second = await dispatch(f); + await db.update(issues).set({ status: "done" }).where(eq(issues.id, first.linkedIssueId!)); + await db.update(routineRuns).set({ status: "failed", failureReason: "Execution issue moved to blocked", completedAt: new Date() }).where(eq(routineRuns.id, second.id)); + const terminal = await db.insert(routineRuns).values(["skipped", "coalesced"].map(status => ({ companyId: f.companyId, + routineId: f.routine.id, source: "manual", linkedIssueId: first.linkedIssueId, status, completedAt: new Date() }))).returning(); + await start(); await projected(first.id, "completed"); await projected(second.id, "issue_created"); + for (const row of terminal) expect(await saved(row.id)).toEqual(row); + await quiet(); expect(errors).toEqual([]); + }); + it("settles the declared custom origin of a real plugin-managed dispatch and preserves other companies", async () => { + const f = await fixture(), other = await fixture(); + const pluginKey = `routine-status-${randomUUID()}`, originId = `operation:${randomUUID()}`; + const [plugin] = await db.insert(plugins).values({ pluginKey, packageName: pluginKey, version: "0.1.0", + manifestJson: { id: pluginKey, apiVersion: 1, version: "0.1.0", displayName: "Routine test", description: "Routine test", + author: "Paperclip", categories: ["automation"], capabilities: ["routines.managed"], entrypoints: { worker: "./worker.js" } } }).returning(); + await db.insert(pluginManagedResources).values({ companyId: f.companyId, pluginId: plugin.id, pluginKey, + resourceKind: "routine", resourceKey: "status", resourceId: f.routine.id, defaultsJson: { issueTemplate: { originId } } }); + const run = await dispatch(f), otherRun = await dispatch(other); + expect((await db.select().from(issues).where(eq(issues.id, run.linkedIssueId!)))[0].originId).toBe(originId); + await start(); await quiet(); + await issueService(db).update(run.linkedIssueId!, { status: "done" }); await projected(run.id, "completed"); + const before = await saved(run.id); + await db.transaction(async tx => { await notifyRoutineExecutionStatus(tx, run.linkedIssueId!); await notifyRoutineExecutionStatus(tx, run.linkedIssueId!); }); + await settled(); await quiet(); + expect(await saved(run.id)).toEqual(before); expect((await saved(otherRun.id)).status).toBe("issue_created"); expect(errors).toEqual([]); + }); + it("waits for the real outer transaction and forgets a rolled-back nested status write", async () => { + const f = await fixture(), run = await dispatch(f); await start(); await quiet(); + let release!: () => void, entered!: () => void; + const gate = new Promise(resolve => { release = resolve; }); + const written = new Promise(resolve => { entered = resolve; }); + const writing = db.transaction(async tx => { + await issueService(db).update(run.linkedIssueId!, { status: "done" }, tx); + entered(); await gate; + }); + try { await written; expect((await saved(run.id)).status).toBe("issue_created"); } + finally { release(); await writing; } + await projected(run.id, "completed"); await quiet(); + const other = await dispatch(f); + await expect(db.transaction(async tx => { + await issueService(db).update(other.linkedIssueId!, { status: "cancelled" }, tx); + throw new Error("outer rollback"); + })).rejects.toThrow("outer rollback"); + await settled(); expect((await saved(other.id)).status).toBe("issue_created"); await quiet(); expect(errors).toEqual([]); + }); + it("keeps the first write during drain and reconstructs on resume without passive SQL", async () => { + const f = await fixture(), run = await dispatch(f); await start(); await quiet(); + draining = true; + await issueService(db).update(run.linkedIssueId!, { status: "cancelled" }); + await settled(); await quiet(); expect((await saved(run.id)).status).toBe("issue_created"); + draining = false; await vi.advanceTimersByTimeAsync(30_000); await projected(run.id, "failed"); + await quiet(); expect(errors).toEqual([]); + }); + it.each(["standby", "disabled"])("does no owner SQL while %s and reconstructs when enabled", async mode => { + const f = await fixture(), run = await dispatch(f); + await issueService(db).update(run.linkedIssueId!, { status: "done" }); + if (mode === "standby") standby = true; else enabled = false; + await start(); await quiet(); expect((await saved(run.id)).status).toBe("issue_created"); + standby = false; enabled = true; await vi.advanceTimersByTimeAsync(30_000); + await projected(run.id, "completed"); await quiet(); expect(errors).toEqual([]); + }); + it("retries failed exact owners, retains the retry across drain and reconstructs after restart", async () => { + const f = await fixture(), run = await dispatch(f), service = routineExecutionStatusService(db); + const settle = vi.spyOn(service, "settle").mockRejectedValueOnce(new Error("projection unavailable")); + await start(service); + await issueService(db).update(run.linkedIssueId!, { status: "done" }); + await vi.waitFor(() => expect(errors).toHaveLength(1)); + expect((await saved(run.id)).status).toBe("issue_created"); + await vi.advanceTimersByTimeAsync(29_000); expect(settle).toHaveBeenCalledTimes(1); + draining = true; await vi.advanceTimersByTimeAsync(60_000); await quiet(); + draining = false; await vi.advanceTimersByTimeAsync(30_000); await projected(run.id, "completed"); await quiet(); + await coordinator!.stop(); coordinator = undefined; + const other = await dispatch(f); await issueService(db).update(other.linkedIssueId!, { status: "blocked" }); + await start(); await projected(other.id, "failed"); await quiet(); + }); + it("preserves notifications during a sweep, coalesces duplicate hints and never overlaps", async () => { + const f = await fixture(), first = await dispatch(f), second = await dispatch(f), service = routineExecutionStatusService(db); + let entered!: () => void, release!: () => void; + const enteredGate = new Promise(resolve => { entered = resolve; }); + const releaseGate = new Promise(resolve => { release = resolve; }); + const load = service.load; + vi.spyOn(service, "load").mockImplementationOnce(async issueId => { const rows = await load(issueId); entered(); await releaseGate; return rows; }); + const starting = start(service); await enteredGate; + await issueService(db).update(second.linkedIssueId!, { status: "done" }); + await db.transaction(async tx => { await notifyRoutineExecutionStatus(tx, second.linkedIssueId!); await notifyRoutineExecutionStatus(tx, second.linkedIssueId!); }); + release(); await starting; + await projected(second.id, "completed"); expect((await saved(first.id)).status).toBe("issue_created"); + await quiet(); expect(errors).toEqual([]); + }); + it.each(["succeeded", "failed", "cancelled", "interrupted", "timed_out"])("uses real %s run lifecycle changes for activeIssue without an open-issue retry", async status => { + const f = await fixture(), run = await dispatch(f); await start(); + const [execution] = await db.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: f.agentId, + status: "running", contextSnapshot: { issueId: run.linkedIssueId } }).returning(); + await db.update(issues).set({ executionRunId: execution.id }).where(eq(issues.id, run.linkedIssueId!)); + expect((await f.service.getDetail(f.routine.id))?.activeIssue?.id).toBe(run.linkedIssueId); + const deps = { + ...createHeartbeatRunState(db), ...createHeartbeatRunPreparation(db), + issuesSvc: { addComment: vi.fn(), listReviewAttention: vi.fn() }, + recovery: { escalateStrandedAssignedIssue: vi.fn() }, companySkills: { completeTestRunForIssue: vi.fn() }, + budgets: { getInvocationBlock: vi.fn(async () => null) }, treeControlSvc: { getActivePauseHoldGate: vi.fn(async () => null) }, + enqueueWakeup: vi.fn(), getCurrentUserRedactionOptions: vi.fn(async () => ({})), + getAgentInvokability: vi.fn(async () => ({ invokable: true })), emitTerminalAgentTaskRun: vi.fn(), budgetHooks: {}, + } satisfies HeartbeatLifecycleDependencies; + await createHeartbeatLifecycle(db, deps).setRunStatusIfRunning(execution.id, status, { finishedAt: new Date() }); + expect((await f.service.getDetail(f.routine.id))?.activeIssue).toBeNull(); + expect((await saved(run.id)).status).toBe("issue_created"); await quiet(); expect(errors).toEqual([]); + }); +}); diff --git a/server/src/__tests__/routine-execution-status-worker.test.ts b/server/src/__tests__/routine-execution-status-worker.test.ts new file mode 100644 index 00000000000..b88babbbd98 --- /dev/null +++ b/server/src/__tests__/routine-execution-status-worker.test.ts @@ -0,0 +1,86 @@ +import { installDatabaseWorkSignals } from "../../../packages/db/src/work-signals.js"; +import { afterEach, beforeEach, expect, it, vi } from "vitest"; +import { createDeliveryWorkCoordinator } from "../services/delivery-work-coordinator.js"; +import { registerRoutineExecutionStatusWork } from "../services/routine-execution-status-work.js"; + +let coordinator: ReturnType; +let canRun = true; +const errors: unknown[] = []; +const entry = { id: "run", companyId: "company", routineId: "routine", issueId: "issue" }; +beforeEach(() => { + vi.useFakeTimers(); canRun = true; errors.length = 0; + const owner = installDatabaseWorkSignals({ execute: vi.fn(async () => []) }); + coordinator = createDeliveryWorkCoordinator({ owner, canRun: () => canRun, canReconcile: () => true, + onError: error => errors.push(error) }); +}); +afterEach(async () => { await coordinator.stop(); vi.useRealTimers(); }); + +it("loads actual debt once and leaves no recurring owner work", async () => { + const service = { load: vi.fn(async () => []), settle: vi.fn(async () => true) }; + await registerRoutineExecutionStatusWork(coordinator, service, () => canRun).ready; + expect(service.load).toHaveBeenCalledExactlyOnceWith(undefined); + expect(coordinator.nextWakeAt()).toBeNull(); + await vi.advanceTimersByTimeAsync(20 * 60_000); + expect(service.load).toHaveBeenCalledOnce(); expect(service.settle).not.toHaveBeenCalled(); expect(errors).toEqual([]); +}); + +it("reconstructs only an invalidated issue and coalesces duplicate requests", async () => { + const service = { load: vi.fn(async (_issueId?: string) => [] as typeof entry[]), settle: vi.fn(async () => true) }; + const worker = registerRoutineExecutionStatusWork(coordinator, service, () => canRun); await worker.ready; + service.load.mockResolvedValueOnce([entry]); + worker.invalidate("issue"); worker.invalidate("issue"); + await vi.advanceTimersByTimeAsync(1000); + expect(service.load).toHaveBeenLastCalledWith("issue"); expect(service.load).toHaveBeenCalledTimes(2); + expect(service.settle).toHaveBeenCalledExactlyOnceWith(expect.objectContaining(entry)); + expect(coordinator.nextWakeAt()).toBeNull(); expect(errors).toEqual([]); +}); + +it("retries only the failed owner at backoff and disarms after success", async () => { + const service = { load: vi.fn(async () => [entry]), settle: vi.fn(async () => true).mockRejectedValueOnce(new Error("locked")) }; + await registerRoutineExecutionStatusWork(coordinator, service, () => canRun).ready; + expect(errors).toHaveLength(1); + const retryAt = coordinator.nextWakeAt(); expect(retryAt).toBe(Date.now() + 30_000); + await vi.advanceTimersByTimeAsync(29_999); expect(service.settle).toHaveBeenCalledOnce(); + await vi.advanceTimersByTimeAsync(1); expect(service.settle).toHaveBeenCalledTimes(2); + expect(service.load).toHaveBeenCalledOnce(); expect(coordinator.nextWakeAt()).toBeNull(); + await vi.advanceTimersByTimeAsync(20 * 60_000); expect(service.settle).toHaveBeenCalledTimes(2); +}); + +it("retains startup and first work during drain without passive owner calls", async () => { + canRun = false; + const service = { load: vi.fn(async () => [entry]), settle: vi.fn(async () => true) }; + const worker = registerRoutineExecutionStatusWork(coordinator, service, () => canRun); await worker.ready; + worker.invalidate("issue"); + await vi.advanceTimersByTimeAsync(20 * 60_000); + expect(service.load).not.toHaveBeenCalled(); expect(service.settle).not.toHaveBeenCalled(); + canRun = true; await vi.advanceTimersByTimeAsync(30_000); + expect(service.settle).toHaveBeenCalledOnce(); expect(coordinator.nextWakeAt()).toBeNull(); expect(errors).toEqual([]); +}); + +it("keeps a failed owner across drain and retries after resume", async () => { + const service = { load: vi.fn(async () => [entry]), settle: vi.fn(async () => true).mockRejectedValueOnce(new Error("write failed")) }; + await registerRoutineExecutionStatusWork(coordinator, service, () => canRun).ready; + canRun = false; await vi.advanceTimersByTimeAsync(20 * 60_000); + expect(service.settle).toHaveBeenCalledOnce(); expect(service.load).toHaveBeenCalledOnce(); + canRun = true; await vi.advanceTimersByTimeAsync(30_000); + expect(service.settle).toHaveBeenCalledTimes(2); expect(coordinator.nextWakeAt()).toBeNull(); +}); + +it("retains a notification that arrives while the startup reconstruction is in flight", async () => { + let release!: () => void; + const gate = new Promise(resolve => { release = resolve; }); + const service = { load: vi.fn(async (_issueId?: string) => [] as typeof entry[]), settle: vi.fn(async () => true) }; + service.load.mockImplementationOnce(async () => { await gate; return []; }); + const worker = registerRoutineExecutionStatusWork(coordinator, service, () => canRun); + await vi.waitFor(() => expect(service.load).toHaveBeenCalledOnce()); + worker.invalidate("issue"); service.load.mockResolvedValueOnce([entry]); release(); await worker.ready; + await vi.advanceTimersByTimeAsync(1000); + expect(service.settle).toHaveBeenCalledOnce(); expect(coordinator.nextWakeAt()).toBeNull(); expect(errors).toEqual([]); +}); + +it("cancels retries and subscriptions at shutdown", async () => { + const service = { load: vi.fn(async () => [entry]), settle: vi.fn(async () => true).mockRejectedValue(new Error("offline")) }; + const worker = registerRoutineExecutionStatusWork(coordinator, service, () => canRun); await worker.ready; + await worker.stop(); worker.invalidate("issue"); await vi.advanceTimersByTimeAsync(20 * 60_000); + expect(service.settle).toHaveBeenCalledOnce(); expect(coordinator.nextWakeAt()).toBeNull(); +}); diff --git a/server/src/__tests__/routine-status-reconciliation.test.ts b/server/src/__tests__/routine-status-reconciliation.test.ts index eba0718ce25..964e43d6035 100644 --- a/server/src/__tests__/routine-status-reconciliation.test.ts +++ b/server/src/__tests__/routine-status-reconciliation.test.ts @@ -1,3 +1,4 @@ +import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js"; import { randomUUID } from "node:crypto"; import { eq, sql } from "drizzle-orm"; import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest"; @@ -5,11 +6,12 @@ import { companies, createDb, issues, routineRuns, routines } from "@paperclipai import { routineService } from "../services/routines.js"; import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; -const support = await getEmbeddedPostgresTestSupport(); +const external = resolveExternalTestDatabaseUrl("PAPERCLIP_ROUTINE_STATUS_TEST_DATABASE_URL"); +const support = external ? { supported: true, reason: undefined } : await getEmbeddedPostgresTestSupport(); if (!support.supported) console.warn(`Skipping routine status database tests: ${support.reason}`); (support.supported ? describe : describe.skip)("routine status reconciliation", () => { let db: ReturnType, database: Awaited>; - beforeAll(async () => { database = await startEmbeddedPostgresTestDatabase("routine-status-"); db = createDb(database.connectionString); }, 60_000); + beforeAll(async () => { if (!external) database = await startEmbeddedPostgresTestDatabase("routine-status-"); db = createDb(external ?? database.connectionString, { maxConnections: 6 }); }, 60_000); afterEach(async () => { vi.restoreAllMocks(); await db.execute(sql`SET client_min_messages = warning; TRUNCATE companies CASCADE`); }); afterAll(async () => { await db?.$client.end(); await database?.cleanup(); }); async function seed(status = "blocked") { diff --git a/server/src/index.ts b/server/src/index.ts index ae229f03518..8484b58054f 100644 --- a/server/src/index.ts +++ b/server/src/index.ts @@ -1,3 +1,5 @@ +import { registerRoutineExecutionStatusWork } from "./services/routine-execution-status-work.js"; +import { routineExecutionStatusService } from "./services/routine-run-status.js"; import { registerProviderTraceRetentionWork } from "./services/provider-trace-retention-work.js"; import { registerRunEvidenceMigrationWork } from "./services/run-evidence-migration-work.js"; import { registerSkillVersionRetentionWork } from "./services/skill-version-retention-work.js"; @@ -1560,15 +1562,6 @@ async function startServerWithDatabaseTeardown( heartbeat.drainActiveRunExecutions(); prepareHotRestartShutdown = heartbeat.prepareHotRestartShutdown; const routines = routineService(db as any, { pluginWorkerManager }); - let nextRoutineStatusReconcile = 0; - const scheduleRoutineStatusReconcile = () => { - if (!isCloudManagedInstance() || isWarmStandby() || isIdleTaskDrainActive() || heartbeatSchedulerStopped || - Date.now() < nextRoutineStatusReconcile) return; - nextRoutineStatusReconcile = Date.now() + 5 * 60_000; - trackHeartbeatSchedulerWork(routines.reconcileExecutionRunStatuses().then(result => { - if (result.settled > 0) logger.info(result, "reconciled routine execution issue status"); - }).catch(() => logger.warn("routine execution status reconciliation deferred"))); - }; const statusCards = statusCardService(db as any); const issues = issueService(db as any); const mergedPullRequestConfirmations = issueThreadInteractionService(db as any, { @@ -1989,7 +1982,6 @@ async function startServerWithDatabaseTeardown( // captures the authoritative set of running heartbeat rows. trackHeartbeatSchedulerWork((async () => { if (isIdleTaskDrainActive() || heartbeatSchedulerStopped) return; - scheduleRoutineStatusReconcile(); trackHeartbeatSchedulerWork(runRetentionSweep().catch((err: unknown) => { logger.error({ err }, "decision retention sweep failed"); })); @@ -2097,6 +2089,11 @@ async function startServerWithDatabaseTeardown( })); }; const startHeartbeatScheduler = () => { + if (isCloudManagedInstance()) { + const status = registerRoutineExecutionStatusWork(deliveryWork, routineExecutionStatusService(db as any), + () => !isIdleTaskDrainActive() && !isWarmStandby() && !heartbeatSchedulerStopped); + trackHeartbeatSchedulerWork(status.ready); + } const scheduled = registerScheduledWork(deliveryWork, { retryMs: config.heartbeatSchedulerIntervalMs, tickAgentTimers: async signal => { diff --git a/server/src/services/delivery-work-notifications.ts b/server/src/services/delivery-work-notifications.ts index 9195f1fcb12..22171c435cf 100644 --- a/server/src/services/delivery-work-notifications.ts +++ b/server/src/services/delivery-work-notifications.ts @@ -114,6 +114,7 @@ export const DELIVERY_QUEUES = { pluginLifecycle: "plugin-resource-lifecycle", toolAction: "tool-action-receipts", terminalWorkspaces: "terminal-workspaces", + routineExecutionStatus: "routine-execution-status", routineSchedules: "routine-schedules", issueMonitors: "issue-monitors", agentTimers: "agent-timers", diff --git a/server/src/services/execution-control-notifications.ts b/server/src/services/execution-control-notifications.ts index 22c8913bbcd..33a99774a48 100644 --- a/server/src/services/execution-control-notifications.ts +++ b/server/src/services/execution-control-notifications.ts @@ -1,7 +1,9 @@ +import { notifyRoutineExecutionStatus } from "./routine-execution-status-notifications.js"; import type { Db } from "@paperclipai/db"; import { DELIVERY_QUEUES, notifyDeliveryWork, type DeliveryQueue } from "./delivery-work-notifications.js"; -export async function notifyExecutionRunWork(transaction: object) { +export async function notifyExecutionRunWork(transaction: object, issueId?: string) { + await notifyRoutineExecutionStatus(transaction, issueId); for (const queue of [DELIVERY_QUEUES.executionStatus, DELIVERY_QUEUES.executionFinalization, DELIVERY_QUEUES.executionReplacement, DELIVERY_QUEUES.executionDisposition, DELIVERY_QUEUES.issueMonitors]) { await notifyDeliveryWork(transaction, queue); @@ -14,6 +16,7 @@ export function writeExecutionControl( write: (tx: Db) => PromiseLike, ): Promise { return db.transaction(async transaction => { + await notifyRoutineExecutionStatus(transaction); await notifyDeliveryWork(transaction, DELIVERY_QUEUES.issueMonitors); for (const topic of typeof queue === "string" ? [queue] : queue) await notifyDeliveryWork(transaction, topic); return write(transaction as unknown as Db); diff --git a/server/src/services/heartbeat/run-lifecycle.ts b/server/src/services/heartbeat/run-lifecycle.ts index 00a27c1faf2..f796da03a5e 100644 --- a/server/src/services/heartbeat/run-lifecycle.ts +++ b/server/src/services/heartbeat/run-lifecycle.ts @@ -1044,7 +1044,8 @@ export function createHeartbeatLifecycle(db: Db, dependencies: HeartbeatLifecycl if (previous && isHeartbeatRunTerminalStatus(status)) await notifyRetainedNativeSessionCleanup(tx, { companyId: previous.companyId, runId: previous.id }); if (isHeartbeatRunTerminalStatus(status)) await notifyDeliveryWork(tx, DELIVERY_QUEUES.accounting, companyId); - await notifyExecutionRunWork(tx); + await notifyExecutionRunWork(tx, previous?.nativeIssueId ?? previous?.issueId + ?? readNonEmptyString(parseObject(previous?.contextSnapshot).issueId) ?? undefined); if (previous) await notifyChatRunMilestones(tx, { companyId: previous.companyId, runId: previous.id, issueId: previous.nativeIssueId ?? previous.issueId ?? readNonEmptyString(parseObject(previous.contextSnapshot).issueId) ?? undefined }); return write(tx); diff --git a/server/src/services/issue-review-state-notifications.ts b/server/src/services/issue-review-state-notifications.ts index 679b7e37475..15bc2a24287 100644 --- a/server/src/services/issue-review-state-notifications.ts +++ b/server/src/services/issue-review-state-notifications.ts @@ -1,9 +1,11 @@ +import { notifyRoutineExecutionStatus } from "./routine-execution-status-notifications.js"; import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js"; import { notifyExternalConversationSettlement } from "./external-conversation-settlement-notifications.js"; import { notifyNativeReviewResponse } from "./native-review-response-notifications.js"; import { notifyNativeStatusReassessment } from "./native-status-reassessment-notifications.js"; export async function notifyIssueReviewState(tx: object, scope: { companyId: string; issueId?: string; runId?: string; agentId?: string }) { + await notifyRoutineExecutionStatus(tx, scope.issueId); await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors); await notifyExternalConversationSettlement(tx, scope); await notifyNativeReviewResponse(tx, scope); diff --git a/server/src/services/issues.ts b/server/src/services/issues.ts index 42c65bb5d02..eeebf326267 100644 --- a/server/src/services/issues.ts +++ b/server/src/services/issues.ts @@ -1,3 +1,4 @@ +import { notifyRoutineExecutionStatus } from "./routine-execution-status-notifications.js"; import { notifyExternalConversationSettlement } from "./external-conversation-settlement-notifications.js"; import { notifyNativeRecoveryNotices } from "./native-recovery-notice-notifications.js"; import { notifyNativeRunProjection } from "./native-run-projection-notifications.js"; @@ -10606,6 +10607,7 @@ export function issueService(db: Db) { } if (values.monitorNextCheckAt) await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors); await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers); + if (values.originKind === "routine_execution") await notifyRoutineExecutionStatus(tx, values.id); const [issue] = await tx.insert(issues).values(values).returning(); await attachOwnedDraftImages(tx, issue, issue.description, { userId: issueData.createdByUserId, agentId: issueData.createdByAgentId }); if (issue.visibility === "private" && !inheritedPrivateSubtree && issueData.createdByAgentId) { @@ -11494,6 +11496,7 @@ export function issueService(db: Db) { if (patch.status !== undefined || patch.assigneeAgentId !== undefined || patch.assigneeUserId !== undefined || patch.hiddenAt !== undefined || patch.conversationAgentId !== undefined) { await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers); + await notifyRoutineExecutionStatus(tx, id); } if (patch.monitorNextCheckAt !== undefined || (receiptExisting.monitorNextCheckAt && (patch.status !== undefined || patch.assigneeAgentId !== undefined || patch.assigneeUserId !== undefined))) { @@ -11542,6 +11545,9 @@ export function issueService(db: Db) { await notifyNativeWorkspaceFinalization(tx, { companyId: receiptExisting.companyId, issueId: id }); await notifyNativeStatusReassessment(tx, { companyId: receiptExisting.companyId, issueId: id }); } + if (patch.originKind !== undefined || patch.originRunId !== undefined || patch.originId !== undefined) { + await notifyRoutineExecutionStatus(tx, id); + } const updated = await tx .update(issues) .set(patch) @@ -11958,6 +11964,7 @@ export function issueService(db: Db) { await notifyDecisionTargetIssueRemoval(tx, id); await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors); await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers); + await notifyRoutineExecutionStatus(tx, id); removedIssue = await tx .delete(issues) .where(eq(issues.id, id)) @@ -12086,6 +12093,7 @@ export function issueService(db: Db) { const updated = await db.transaction(async tx => { await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors); await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers); + await notifyRoutineExecutionStatus(tx, id); return tx .update(issues) .set({ @@ -12215,6 +12223,7 @@ export function issueService(db: Db) { const adopted = await db.transaction(async tx => { await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors); await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers); + await notifyRoutineExecutionStatus(tx, id); return tx .update(issues) .set(adoptionSet) @@ -12470,6 +12479,7 @@ export function issueService(db: Db) { await notifyNativeStatusReassessment(tx, { companyId: existing.companyId, issueId: id }); await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors); await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers); + await notifyRoutineExecutionStatus(tx, id); const updated = await tx .update(issues) .set({ @@ -12530,6 +12540,7 @@ export function issueService(db: Db) { } await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors); await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers); + await notifyRoutineExecutionStatus(tx, id); const updated = await tx .update(issues) .set(patch) diff --git a/server/src/services/native-runtime/native-routine-status.test.ts b/server/src/services/native-runtime/native-routine-status.test.ts index 668de83a4d5..415622ea2c3 100644 --- a/server/src/services/native-runtime/native-routine-status.test.ts +++ b/server/src/services/native-runtime/native-routine-status.test.ts @@ -1,3 +1,4 @@ +import { resolveExternalTestDatabaseUrl } from "../../__tests__/helpers/external-test-database.js"; import { randomUUID } from "node:crypto"; import { eq } from "drizzle-orm"; import { afterAll, beforeAll, describe, expect, it } from "vitest"; @@ -8,13 +9,14 @@ import { routineService } from "../routines.js"; import { commitNativeStatusDecision } from "./status-decision-committer.js"; import { NATIVE_STATUS_ARBITER_POLICY_VERSION, type NativeStatusDecision } from "./status-arbiter.js"; -const support = await getEmbeddedPostgresTestSupport(); +const external = resolveExternalTestDatabaseUrl("PAPERCLIP_NATIVE_ROUTINE_STATUS_TEST_DATABASE_URL"); +const support = external ? { supported: true, reason: undefined } : await getEmbeddedPostgresTestSupport(); if (!support.supported) console.warn(`Skipping native routine status tests: ${support.reason}`); (support.supported ? describe : describe.skip)("native routine history status projection", () => { let db: ReturnType, database: Awaited>; const companyId = randomUUID(), agentId = randomUUID(); beforeAll(async () => { - database = await startEmbeddedPostgresTestDatabase("native-routine-status-"); db = createDb(database.connectionString); + if (!external) database = await startEmbeddedPostgresTestDatabase("native-routine-status-"); db = createDb(external ?? database.connectionString, { maxConnections: 6 }); await db.insert(companies).values({ id: companyId, name: "Routine status fixture", issuePrefix: "NRS" }); await db.insert(agents).values({ id: agentId, companyId, name: "Routine worker", adapterType: "paperclip_runner", status: "running" }); }, 60_000); diff --git a/server/src/services/routine-execution-status-notifications.ts b/server/src/services/routine-execution-status-notifications.ts new file mode 100644 index 00000000000..cdd2be826c0 --- /dev/null +++ b/server/src/services/routine-execution-status-notifications.ts @@ -0,0 +1,5 @@ +import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js"; + +export function notifyRoutineExecutionStatus(tx: object, issueId?: string) { + return notifyDeliveryWork(tx, DELIVERY_QUEUES.routineExecutionStatus, issueId ?? "*"); +} diff --git a/server/src/services/routine-execution-status-work.ts b/server/src/services/routine-execution-status-work.ts new file mode 100644 index 00000000000..81c8e566a7c --- /dev/null +++ b/server/src/services/routine-execution-status-work.ts @@ -0,0 +1,18 @@ +import type { createDeliveryWorkCoordinator } from "./delivery-work-coordinator.js"; +import { DELIVERY_QUEUES } from "./delivery-work-notifications.js"; +import { registerScopedDeadlineWork } from "./scoped-deadline-work.js"; +import type { routineExecutionStatusService } from "./routine-run-status.js"; + +export function registerRoutineExecutionStatusWork( + coordinator: ReturnType, + service: ReturnType, + canRun: () => boolean, +) { + return registerScopedDeadlineWork(coordinator, { + queue: DELIVERY_QUEUES.routineExecutionStatus, + decodeScope: hint => hint === "*" ? undefined : hint, + matches: (entry, issueId?: string) => issueId === undefined || entry.issueId === issueId, + load: async (issueId?: string) => (await service.load(issueId)).map(row => ({ ...row, dueAt: Date.now() })), + check: async entry => { await service.settle(entry); return null; }, + }, canRun); +} diff --git a/server/src/services/routine-run-status.ts b/server/src/services/routine-run-status.ts index 77341f9cba4..79c1ff9a231 100644 --- a/server/src/services/routine-run-status.ts +++ b/server/src/services/routine-run-status.ts @@ -1,6 +1,7 @@ -import { and, eq } from "drizzle-orm"; +import { notifyRoutineExecutionStatus } from "./routine-execution-status-notifications.js"; +import { and, eq, inArray, or, sql } from "drizzle-orm"; import type { Db } from "@paperclipai/db"; -import { issues, routineRuns, routines } from "@paperclipai/db"; +import { issues, pluginManagedResources, routineRuns, routines } from "@paperclipai/db"; const EXECUTION_ISSUE_TRANSIENT_FAILURE_CODE = "execution_issue_status"; const EXECUTION_ISSUE_TRANSIENT_FAILURE_STATUSES = ["blocked", "cancelled"] as const; @@ -37,6 +38,7 @@ function legacyExecutionIssueTransientFailureStatus( } export async function finalizeRoutineRun(executor: Db, runId: string, patch: Partial) { + if (patch.linkedIssueId) await notifyRoutineExecutionStatus(executor, patch.linkedIssueId); return executor .update(routineRuns) .set({ @@ -67,14 +69,16 @@ export async function syncRoutineRunStatusForIssue(executor: Db, issueId: string status: routineRuns.status, failureReason: routineRuns.failureReason, triggerPayload: routineRuns.triggerPayload, + completedAt: routineRuns.completedAt, }) .from(routineRuns) .innerJoin(routines, and(eq(routines.id, routineRuns.routineId), eq(routines.companyId, issue.companyId))) .where(and(eq(routineRuns.id, issue.originRunId), eq(routineRuns.companyId, issue.companyId), eq(routineRuns.linkedIssueId, issue.id))) .then((rows) => rows[0] ?? null); - if (!run) return null; + if (!run || ["received", "skipped", "coalesced"].includes(run.status)) return null; if (issue.status === "done") { + if (run.status === "completed" && run.failureReason === null && run.completedAt) return null; const transientFailureStatus = executionIssueTransientFailureStatusFromPayload(run.triggerPayload) ?? legacyExecutionIssueTransientFailureStatus(run.failureReason); const transientFailureClearedAt = executionIssueTransientFailureClearedAtFromPayload(run.triggerPayload); @@ -99,6 +103,7 @@ export async function syncRoutineRunStatusForIssue(executor: Db, issueId: string } if (issue.status === "blocked" || issue.status === "cancelled") { const failureReason = executionIssueTransientFailureReason(issue.status); + if (run.status === "failed" && run.failureReason === failureReason && run.completedAt) return null; return finalizeRoutineRun(executor, issue.originRunId, { status: "failed", failureReason, @@ -135,3 +140,60 @@ export async function syncRoutineRunStatusForIssue(executor: Db, issueId: string return null; } + +export function routineExecutionOriginCondition() { + return or(sql`${issues.originId} = ${routineRuns.routineId}::text`, sql`exists ( + select 1 from ${pluginManagedResources} resource + where resource.company_id = ${routineRuns.companyId} + and resource.resource_kind = 'routine' and resource.resource_id = ${routineRuns.routineId} + and resource.defaults_json -> 'issueTemplate' ->> 'originId' = ${issues.originId} + )`); +} + +export function routineExecutionStatusDebtCondition() { + const transientFailure = sql`(${routineRuns.triggerPayload} -> 'transientFailure' ->> 'code' = ${EXECUTION_ISSUE_TRANSIENT_FAILURE_CODE} + or ${routineRuns.failureReason} in ('Execution issue moved to blocked', 'Execution issue moved to cancelled'))`; + return or( + and(eq(routineRuns.status, "issue_created"), inArray(issues.status, ["done", "blocked", "cancelled"])), + and(eq(routineRuns.status, "completed"), or(inArray(issues.status, ["blocked", "cancelled"]), + and(eq(issues.status, "done"), sql`(${routineRuns.completedAt} is null or ${routineRuns.failureReason} is not null)`))), + and(eq(routineRuns.status, "failed"), transientFailure, + sql`(${issues.status} not in ('blocked', 'cancelled') + or ${routineRuns.completedAt} is null + or ${routineRuns.failureReason} is distinct from 'Execution issue moved to ' || ${issues.status})`), + ); +} + +export function routineExecutionStatusService(db: Db) { + async function load(issueId?: string) { + return db.select({ id: routineRuns.id, companyId: routineRuns.companyId, + issueId: issues.id, routineId: routineRuns.routineId }) + .from(routineRuns) + .innerJoin(routines, and(eq(routines.id, routineRuns.routineId), eq(routines.companyId, routineRuns.companyId))) + .innerJoin(issues, and(eq(issues.id, routineRuns.linkedIssueId), eq(issues.companyId, routineRuns.companyId), + eq(issues.originKind, "routine_execution"), sql`${issues.originRunId} = ${routineRuns.id}::text`, routineExecutionOriginCondition())) + .where(and(routineExecutionStatusDebtCondition(), ...(issueId ? [eq(issues.id, issueId)] : []))) + .orderBy(routineRuns.id); + } + async function settle(candidate: Awaited>[number]) { + return db.transaction(async tx => { + await tx.execute(sql`SET LOCAL lock_timeout = '500ms'`); + await tx.execute(sql`SET LOCAL statement_timeout = '5s'`); + const [routine] = await tx.select({ id: routines.id }).from(routines).where(and( + eq(routines.id, candidate.routineId), eq(routines.companyId, candidate.companyId))).for("share"); + if (!routine) return false; + const [issue] = await tx.select().from(issues).where(and(eq(issues.id, candidate.issueId), + eq(issues.companyId, candidate.companyId), eq(issues.originKind, "routine_execution"), + eq(issues.originRunId, candidate.id))).for("update"); + if (!issue) return false; + const [run] = await tx.select({ id: routineRuns.id }).from(routineRuns) + .innerJoin(issues, eq(issues.id, routineRuns.linkedIssueId)) + .where(and(eq(routineRuns.id, candidate.id), eq(routineRuns.companyId, candidate.companyId), + eq(routineRuns.routineId, candidate.routineId), eq(routineRuns.linkedIssueId, issue.id), + routineExecutionOriginCondition(), routineExecutionStatusDebtCondition())).for("update", { of: routineRuns }); + if (!run) return false; + return (await syncRoutineRunStatusForIssue(tx as unknown as Db, issue.id)) !== null; + }); + } + return { load, settle }; +} diff --git a/server/src/services/routines.ts b/server/src/services/routines.ts index 23787b45249..c391207950b 100644 --- a/server/src/services/routines.ts +++ b/server/src/services/routines.ts @@ -73,7 +73,7 @@ import { secretService } from "./secrets.js"; import { getSecretProvider } from "../secrets/provider-registry.js"; import { parseCron, validateCron } from "./cron.js"; import { heartbeatService } from "./heartbeat.js"; -import { finalizeRoutineRun, syncRoutineRunStatusForIssue } from "./routine-run-status.js"; +import { finalizeRoutineRun, routineExecutionStatusService, syncRoutineRunStatusForIssue } from "./routine-run-status.js"; import { instanceSettingsService, isTruthyRuntimeEnvValue, @@ -2001,7 +2001,6 @@ export function routineService( return run; } - let statusReconcileCursor: string | null = null; async function syncRunStatusForIssue(issueId: string, executor: Db = db) { return syncRoutineRunStatusForIssue(executor, issueId); @@ -3245,41 +3244,13 @@ export function routineService( return { triggered }; }, - // Repair a missed issue-status hook through the same status transition as - // normal issue writes. Never discard queued input or infer run completion - // from age. Lock both owners and re-read the issue before applying repair. reconcileExecutionRunStatuses: async (): Promise<{ scanned: number; settled: number; deferred: number }> => { - const candidates = await db.select({ id: routineRuns.id, companyId: routineRuns.companyId, - issueId: issues.id, routineId: routineRuns.routineId }) - .from(routineRuns).innerJoin(routines, and(eq(routines.id, routineRuns.routineId), eq(routines.companyId, routineRuns.companyId))) - .innerJoin(issues, and(eq(issues.id, routineRuns.linkedIssueId), - eq(issues.companyId, routineRuns.companyId), eq(issues.originKind, "routine_execution"), - sql`${issues.originRunId} = ${routineRuns.id}::text`, sql`${issues.originId} = ${routineRuns.routineId}::text`)) - .where(and(eq(routineRuns.status, "issue_created"), inArray(issues.status, ["done", "cancelled", "blocked"]), - ...(statusReconcileCursor ? [sql`${routineRuns.id}::text > ${statusReconcileCursor}`] : []))) - .orderBy(asc(routineRuns.id)).limit(50); - statusReconcileCursor = candidates.length === 50 ? candidates.at(-1)!.id : null; - let settled = 0; - let deferred = 0; + const service = routineExecutionStatusService(db); + const candidates = await service.load(); + let settled = 0, deferred = 0; for (const candidate of candidates) { try { - const repaired = await db.transaction(async tx => { - await tx.execute(sql`SET LOCAL lock_timeout = '500ms'`); - await tx.execute(sql`SET LOCAL statement_timeout = '5s'`); - const [routine] = await tx.select({ id: routines.id }).from(routines).where(and( - eq(routines.id, candidate.routineId), eq(routines.companyId, candidate.companyId))).for("share"); - if (!routine) return false; - const [issue] = await tx.select().from(issues).where(and(eq(issues.id, candidate.issueId), - eq(issues.companyId, candidate.companyId), eq(issues.originKind, "routine_execution"), - eq(issues.originRunId, candidate.id), eq(issues.originId, candidate.routineId))).for("update"); - if (!issue || !["done", "cancelled", "blocked"].includes(issue.status)) return false; - const [run] = await tx.select().from(routineRuns).where(and(eq(routineRuns.id, candidate.id), - eq(routineRuns.companyId, candidate.companyId), eq(routineRuns.routineId, candidate.routineId), - eq(routineRuns.linkedIssueId, issue.id), eq(routineRuns.status, "issue_created"))).for("update"); - if (!run) return false; - return (await syncRunStatusForIssue(issue.id, tx as unknown as Db)) !== null; - }); - if (repaired) settled += 1; + if (await service.settle(candidate)) settled++; } catch (error) { let current: unknown = error; let retryable = false; @@ -3289,7 +3260,7 @@ export function routineService( current = cause.cause; } if (!retryable) throw error; - deferred += 1; + deferred++; } } return { scanned: candidates.length, settled, deferred }; From b762036c984be27ba8aa0c4395e6395741d1e848 Mon Sep 17 00:00:00 2001 From: Dotta Date: Fri, 30 Oct 2026 16:01:23 -0500 Subject: [PATCH 02/74] fix(server): schedule native backup expiration from retention deadlines Co-Authored-By: Paperclip --- .../2026-10-30-backup-expiration-sleep.md | 44 +++++ .../native-harness-backup-expiration.test.ts | 150 ++++++++++++++++++ server/src/index.ts | 36 +---- .../services/delivery-work-notifications.ts | 1 + server/src/services/issues.ts | 4 + ...ve-environment-settlement-notifications.ts | 2 + ...harness-backup-expiration-notifications.ts | 19 +++ .../native-harness-backup-expiration-work.ts | 53 +++++++ .../native-harness-backup-expiration.ts | 130 ++++++++++----- .../native-runtime/native-session-executor.ts | 3 + ...ed-native-session-cleanup-notifications.ts | 2 + 11 files changed, 372 insertions(+), 72 deletions(-) create mode 100644 doc/plans/2026-10-30-backup-expiration-sleep.md create mode 100644 server/src/services/native-harness-backup-expiration-notifications.ts create mode 100644 server/src/services/native-runtime/native-harness-backup-expiration-work.ts diff --git a/doc/plans/2026-10-30-backup-expiration-sleep.md b/doc/plans/2026-10-30-backup-expiration-sleep.md new file mode 100644 index 00000000000..47d41bbf581 --- /dev/null +++ b/doc/plans/2026-10-30-backup-expiration-sleep.md @@ -0,0 +1,44 @@ +# Backup expiration without idle polling + +The backup expiration duty retains its existing minimum age and safety policy. +It uses the shared scoped deadline coordinator. The archived runner evidence and +checkpoint expiration duties keep their separate registrations. + +The durable source is each controller-owned `failover-backups/current` directory, +its control-plane session identity, and the associated run, issue, finalization, +and environment lease rows. The next eligible instant is one millisecond after +the later of the backup mtime and every related issue's close time, plus the +configured retention duration. An open issue, live run or lease, unsettled +finalization, or cleanup activation has no time-derived expiry; its producer +must invalidate the source when it changes. A busy process or failed I/O leaves +real unresolved cleanup and uses the shared retry backoff. Dry-run reports an +eligible source once and waits for source invalidation or restart. + +Startup pages existing disk scopes. Roots without backup or cleanup tombstones +perform no backup SQL. After discovery and all obligations settle, there is no +recurring SQL or filesystem scan. Startup reconstructs future work; inspecting a +source never changes its business timestamps. Runtime task close/reopen and +retention timestamp changes, native environment and retained session writes, +local backup publication, and process release invalidate the company or digest. +Signals from database producers register before writes inside their owning +transaction, including caller-owned transactions and nested savepoints. + +The controller remains the only supported writer of its local backup files. +An independent process writing those files must use the same producer ingress; +manual disk edits require reconstruction on restart. Database writer ingress +continues to use the existing transaction-aware work signals. This change adds +no permanent database listener and does not add a polling table or migration. + +The existing issue share locks now remain held through the retirement rename. +This fences task reopen while the backup becomes unavailable. A failed database +transaction restores `.retiring-*`; a process interruption is recovered from +that existing file journal at startup. After database settlement, `.expired-*` +records outstanding physical removal so a partial delete retries idempotently. +Ownership, session identity, live run, lease and local execution safety checks +remain in the backup owner. No host-sleep blocker or expiry-on-use rule changes. +An in-process deadline does not register a Cloud wake alarm. + +Acceptance uses isolated real PostgreSQL for service writes, transaction signals, +safety checks, resume pointer deletion, and restart reconstruction. A temporary +filesystem is the controlled storage boundary. These tests do not claim cloud +object storage behavior, deployed sleep, or fleet savings. diff --git a/server/src/__tests__/native-harness-backup-expiration.test.ts b/server/src/__tests__/native-harness-backup-expiration.test.ts index 2437d9711b5..be38a9ae0d9 100644 --- a/server/src/__tests__/native-harness-backup-expiration.test.ts +++ b/server/src/__tests__/native-harness-backup-expiration.test.ts @@ -1,3 +1,7 @@ +import { registerNativeHarnessBackupExpirationWork } from "../services/native-runtime/native-harness-backup-expiration-work.js"; +import { noteNativeHarnessBackupChange, notifyNativeHarnessBackupExpiration } from "../services/native-harness-backup-expiration-notifications.js"; +import { writeEnvironmentCleanup } from "../services/environment-cleanup-notifications.js"; +import { issueService } from "../services/issues.js"; import { applyPendingMigrations } from "@paperclipai/db"; import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js"; import { registerNativeRunnerEvidenceRetentionWork } from "../services/native-runtime/native-runner-evidence-retention-work.js"; @@ -55,7 +59,10 @@ describeEmbeddedPostgres("native harness backup expiration", () => { let coordinator: ReturnType | undefined; let worker: ReturnType | undefined; let admitted = true; + let backupWorker: ReturnType | undefined; + const backupErrors: unknown[] = []; afterEach(async () => { + await backupWorker?.stop(); backupWorker = undefined; backupErrors.length = 0; worker?.unsubscribe(); await coordinator?.stop(); coordinator = undefined; worker = undefined; admitted = true; vi.useRealTimers(); vi.restoreAllMocks(); await db.delete(agentTaskSessions); @@ -558,6 +565,149 @@ describeEmbeddedPostgres("native harness backup expiration", () => { reads.mockRestore(); writes.mockRestore(); directories.mockRestore(); } + async function startBackups(options: Parameters[1] = {}) { + if (!stateBase) await emptyStateBase(); + if (!vi.isFakeTimers()) { + vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"], shouldAdvanceTime: true }); + vi.setSystemTime(now); + } + coordinator = createDeliveryWorkCoordinator({ owner: db, canRun: () => admitted, canReconcile: () => true, + onError: error => backupErrors.push(error) }); + backupWorker = registerNativeHarnessBackupExpirationWork(coordinator, db, + { stateBase, isSessionBusy: () => false, ...options }, () => admitted); + await backupWorker.ready; + } + + it.each([false, true])("backup deadline worker leaves empty and retained terminal history quiet (%s)", async history => { + await emptyStateBase(); + if (history) { + const session = await seedSession(); + await fs.rm(path.join(session.root, "failover-backups"), { recursive: true }); + for (let index = 0; index < 250; index++) await fs.mkdir(path.join(stateBase, + createHash("sha256").update(String(index)).digest("hex"))); + } + const reads = vi.spyOn(db, "select"), writes = vi.spyOn(db, "transaction"); + await startBackups({ maxEntries: 20 }); await settled(); + expect(reads).not.toHaveBeenCalled(); expect(writes).not.toHaveBeenCalled(); + reads.mockRestore(); writes.mockRestore(); await quiet(); + expect(backupErrors).toEqual([]); + }); + + it("ordinary task close arms exact backup retention, deletes physical bytes and resume pointer, then stays quiet", async () => { + const session = await seedSession({ issueStatus: "todo" }); + await startBackups(); await settled(); await quiet(); + const closed = await issueService(db).update(session.issueId, { status: "done", companyGuard: session.companyId }); + const deadline = new Date(closed!.completedAt!).getTime() + 7 * DAY_MS + 1; + await settled(deadline); await quiet(60_000); + await vi.advanceTimersByTimeAsync(deadline - Date.now() - 1); + expect(await exists(session.current)).toBe(true); + expect(await taskSessionCount(session.agentId)).toBe(1); + await vi.advanceTimersByTimeAsync(1); await settled(); + expect(await exists(session.current)).toBe(false); + expect(await taskSessionCount(session.agentId)).toBe(0); + expect(await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, session.runId))).toHaveLength(1); + expect(backupErrors).toEqual([]); await quiet(); + }); + + it("reopen cancels a saved deadline and a later ordinary close restores retention", async () => { + const session = await seedSession({ closedDaysAgo: 2 }); + await startBackups(); await settled(now.getTime() + 5 * DAY_MS + 1); + await issueService(db).update(session.issueId, { status: "todo" }); + await settled(); await quiet(8 * DAY_MS); expect(await exists(session.current)).toBe(true); + const closed = await issueService(db).update(session.issueId, { status: "done" }); + const due = new Date(closed!.completedAt!).getTime() + 7 * DAY_MS + 1; + await settled(due); await vi.advanceTimersByTimeAsync(due - Date.now()); await settled(); + expect(await exists(session.current)).toBe(false); await quiet(); + }); + + it("reconstructs a retained backup deadline on restart without changing business timestamps", async () => { + const session = await seedSession({ closedDaysAgo: 2 }); + const [before] = await db.select().from(issues).where(eq(issues.id, session.issueId)); + const due = now.getTime() + 5 * DAY_MS + 1; + await startBackups(); await settled(due); + await backupWorker!.stop(); await coordinator!.stop(); + await startBackups(); await settled(due); + const [after] = await db.select().from(issues).where(eq(issues.id, session.issueId)); + expect(after!.updatedAt).toEqual(before!.updatedAt); expect(after!.completedAt).toEqual(before!.completedAt); + await quiet(); + }); + + it("reports dry-run backups once and rechecks only after source invalidation", async () => { + const session = await seedSession(); const report = vi.spyOn(logger, "info"); + await startBackups({ dryRun: true }); await settled(); + expect(report.mock.calls.filter(call => call[1] === "native harness backup expiration")).toHaveLength(1); + expect(await taskSessionCount(session.agentId)).toBe(1); expect(await exists(session.current)).toBe(true); + await quiet(); + noteNativeHarnessBackupChange(db, { companyId: session.companyId, scopeDigest: session.scope }); + await settled(); + expect(report.mock.calls.filter(call => call[1] === "native harness backup expiration")).toHaveLength(2); + await quiet(); + }); + + it("retains live lease references and only removes the owning company's backup after lease release", async () => { + const first = await seedSession({ leaseStatus: "retained" }); + const other = await seedSession({ leaseStatus: "retained" }); + await startBackups(); await settled(); await quiet(); + await writeEnvironmentCleanup(db, tx => tx.update(environmentLeases).set({ status: "expired" }) + .where(eq(environmentLeases.companyId, first.companyId)), { companyId: first.companyId }); + await settled(); expect(await exists(first.current)).toBe(false); expect(await exists(other.current)).toBe(true); + expect(await taskSessionCount(other.agentId)).toBe(1); await quiet(); + }); + + it("retries a partial physical deletion from its tombstone without recreating a resume pointer", async () => { + const session = await seedSession(); const remove = fs.rm; let failed = false; + const fault = vi.spyOn(fs, "rm").mockImplementation(async (...args) => { + if (!failed && String(args[0]).includes(".expired-")) { failed = true; throw new Error("controlled filesystem delete failure"); } + return remove(...args); + }); + await startBackups(); + await vi.waitFor(async () => { await vi.advanceTimersByTimeAsync(100); expect(backupErrors).toHaveLength(1); }); + const retry = coordinator!.nextWakeAt()!; expect(retry - Date.now()).toBeGreaterThan(20_000); + expect(await taskSessionCount(session.agentId)).toBe(0); expect(await exists(session.current)).toBe(false); + fault.mockRestore(); await quiet(5000); + await vi.advanceTimersByTimeAsync(retry - Date.now()); await settled(); + expect(await fs.readdir(path.join(session.root, "failover-backups"))).toEqual([]); await quiet(); + }); + + it("retains a first ordinary close during drain and dispatches it after resume", async () => { + const session = await seedSession({ issueStatus: "todo" }); + await startBackups(); await settled(); admitted = false; + const closed = await issueService(db).update(session.issueId, { status: "done" }); + await vi.waitFor(async () => { await vi.advanceTimersByTimeAsync(1000); expect(idleWorkSnapshot().active).toBe(0); }); + await quiet(60_000); admitted = true; + const due = new Date(closed!.completedAt!).getTime() + 7 * DAY_MS + 1; + await settled(due); await vi.advanceTimersByTimeAsync(due - Date.now()); await settled(); + expect(await exists(session.current)).toBe(false); await quiet(); + }); + + it("honors outer rollback for nested ordinary close and activates only on committed work", async () => { + const session = await seedSession({ issueStatus: "todo" }); + await startBackups(); await settled(); + await expect(db.transaction(async outer => { + await outer.transaction(async nested => { + await issueService(db).update(session.issueId, { status: "done" }, nested); + }); + expect(await exists(session.current)).toBe(true); + throw new Error("rollback outer close"); + })).rejects.toThrow("rollback outer close"); + await settled(); await quiet(8 * DAY_MS); + expect(await exists(session.current)).toBe(true); + const [saved] = await db.select().from(issues).where(eq(issues.id, session.issueId)); + expect(saved!.status).toBe("todo"); + }); + + it("pages multiple company backups, preserves a busy one and wakes from process release", async () => { + const sessions = [await seedSession(), await seedSession(), await seedSession()]; let busy = true; + await startBackups({ maxEntries: 1, maxRemovals: 1, isSessionBusy: digest => busy && digest === sessions[1]!.scope }); + await vi.waitFor(async () => { await vi.advanceTimersByTimeAsync(1000); + expect(await exists(sessions[0]!.current)).toBe(false); expect(await exists(sessions[2]!.current)).toBe(false); + expect(backupErrors.length).toBeGreaterThan(0); + }); + expect(await exists(sessions[1]!.current)).toBe(true); busy = false; + noteNativeHarnessBackupChange(db, { companyId: sessions[1]!.companyId }); + await settled(); expect(await exists(sessions[1]!.current)).toBe(false); await quiet(); + }); + it("leaves empty evidence and retained backups quiet without invoking backup SQL", async () => { const session = await seedSession(); const reads = vi.spyOn(db, "select"), writes = vi.spyOn(db, "transaction"); diff --git a/server/src/index.ts b/server/src/index.ts index ae229f03518..919af75f14a 100644 --- a/server/src/index.ts +++ b/server/src/index.ts @@ -1,3 +1,4 @@ +import { registerNativeHarnessBackupExpirationWork } from "./services/native-runtime/native-harness-backup-expiration-work.js"; import { registerProviderTraceRetentionWork } from "./services/provider-trace-retention-work.js"; import { registerRunEvidenceMigrationWork } from "./services/run-evidence-migration-work.js"; import { registerSkillVersionRetentionWork } from "./services/skill-version-retention-work.js"; @@ -32,8 +33,6 @@ import { skillVersionRetentionService, } from "./services/skill-version-retention.js"; import { - NATIVE_HARNESS_BACKUP_EXPIRATION_INTERVAL_MS, - NATIVE_HARNESS_BACKUP_EXPIRATION_STARTUP_DELAY_MS, nativeHarnessBackupExpirationService, } from "./services/native-runtime/native-harness-backup-expiration.js"; import { registerNativeRunnerEvidenceRetentionWork } from "./services/native-runtime/native-runner-evidence-retention-work.js"; @@ -1290,30 +1289,6 @@ async function startServerWithDatabaseTeardown( evidenceRetentionDays: config.nativeRunnerEvidenceRetentionDays, shouldContinue: () => !heartbeatSchedulerStopped && !isIdleTaskDrainActive() && !isWarmStandby(), }); - let nativeHarnessBackupExpirationInFlight = false; - let nativeHarnessBackupExpirationContinuation: ReturnType | undefined; - const sweepNativeHarnessBackups = () => { - if (!config.nativeHarnessBackupExpirationEnabled || nativeHarnessBackupExpirationInFlight || isIdleTaskDrainActive() || heartbeatSchedulerStopped || isWarmStandby()) return; - nativeHarnessBackupExpirationInFlight = true; - trackHeartbeatSchedulerWork(nativeHarnessBackupExpiration.tick() - .then((result) => { - if (result.candidates > 0 || result.evidenceCandidates > 0 || result.failed > 0) { - logger.info({ ...result, dryRun: config.nativeHarnessBackupExpirationDryRun }, "native harness backup expiration"); - } - if (result.truncated && !heartbeatSchedulerStopped) { - nativeHarnessBackupExpirationContinuation = setTimeout(sweepNativeHarnessBackups, 1000); - nativeHarnessBackupExpirationContinuation.unref?.(); - } - }) - .catch((err) => { - logger.error({ err }, "native harness backup expiration failed"); - }) - .finally(() => { nativeHarnessBackupExpirationInFlight = false; })); - }; - const nativeHarnessBackupExpirationInterval = setInterval(sweepNativeHarnessBackups, NATIVE_HARNESS_BACKUP_EXPIRATION_INTERVAL_MS); - nativeHarnessBackupExpirationInterval.unref?.(); - const nativeHarnessBackupExpirationStartup = setTimeout(sweepNativeHarnessBackups, NATIVE_HARNESS_BACKUP_EXPIRATION_STARTUP_DELAY_MS); - nativeHarnessBackupExpirationStartup.unref?.(); const externalObjects = externalObjectService(db as any, { pluginWorkerManager, enabled: async () => (await instanceSettingsService(db).getExperimental()).enableExternalObjects === true, @@ -1359,6 +1334,12 @@ async function startServerWithDatabaseTeardown( const deliveryWork = app.locals.deliveryWork as ReturnType; if (config.nativeHarnessBackupExpirationEnabled) { nativeHarnessBackupExpiration.registerCheckpointExpiration(deliveryWork); + const backupExpirationWork = registerNativeHarnessBackupExpirationWork(deliveryWork, db as any, { + dryRun: config.nativeHarnessBackupExpirationDryRun, + closedForDays: config.nativeHarnessBackupExpirationDays, + }, () => !heartbeatSchedulerStopped && !isIdleTaskDrainActive() && !isWarmStandby()); + trackHeartbeatSchedulerWork(backupExpirationWork.ready); + server.on("close", backupExpirationWork.stop); } const taskDrainExpiryWork = registerTaskDrainExpiryWork(deliveryWork, () => { if (heartbeatSchedulerStopped || isWarmStandby()) return; @@ -2235,9 +2216,6 @@ async function startServerWithDatabaseTeardown( heartbeatSchedulerStopped = true; skillVersionRetentionWorker?.unsubscribe(); await taskDrainExpiryWork.stop(); - clearInterval(nativeHarnessBackupExpirationInterval); - clearTimeout(nativeHarnessBackupExpirationStartup); - clearTimeout(nativeHarnessBackupExpirationContinuation); nativeRunnerEvidenceRetentionWorker?.unsubscribe(); await skillSourceSync.stop(); unsubscribeChatCompletions(); diff --git a/server/src/services/delivery-work-notifications.ts b/server/src/services/delivery-work-notifications.ts index 9195f1fcb12..4a4d66edee1 100644 --- a/server/src/services/delivery-work-notifications.ts +++ b/server/src/services/delivery-work-notifications.ts @@ -118,6 +118,7 @@ export const DELIVERY_QUEUES = { issueMonitors: "issue-monitors", agentTimers: "agent-timers", caseHandoffs: "case-handoffs", + nativeHarnessBackupExpiration: "native-harness-backup-expiration", nativeCheckpointExpiration: "native-checkpoint-expiration", } as const; export type DeliveryQueue = typeof DELIVERY_QUEUES[keyof typeof DELIVERY_QUEUES]; diff --git a/server/src/services/issues.ts b/server/src/services/issues.ts index 42c65bb5d02..fdb02c5ea9e 100644 --- a/server/src/services/issues.ts +++ b/server/src/services/issues.ts @@ -1,3 +1,4 @@ +import { notifyNativeHarnessBackupExpiration } from "./native-harness-backup-expiration-notifications.js"; import { notifyExternalConversationSettlement } from "./external-conversation-settlement-notifications.js"; import { notifyNativeRecoveryNotices } from "./native-recovery-notice-notifications.js"; import { notifyNativeRunProjection } from "./native-run-projection-notifications.js"; @@ -11486,6 +11487,9 @@ export function issueService(db: Db) { (blockedByIssueIds !== undefined || issueData.unblockDescriptor !== undefined)))) { patch.statusVersion = sql`${issues.statusVersion} + 1` as unknown as number; } + if (patch.status !== undefined || patch.completedAt !== undefined || patch.cancelledAt !== undefined) { + await notifyNativeHarnessBackupExpiration(tx, { companyId: receiptExisting.companyId }); + } if ((patch.status !== undefined && patch.status !== receiptExisting.status) || patch.completedAt !== undefined || patch.cancelledAt !== undefined || issueData.parentId !== undefined || issueData.executionWorkspaceId !== undefined) { diff --git a/server/src/services/native-environment-settlement-notifications.ts b/server/src/services/native-environment-settlement-notifications.ts index 5769b81015a..427c43522b7 100644 --- a/server/src/services/native-environment-settlement-notifications.ts +++ b/server/src/services/native-environment-settlement-notifications.ts @@ -1,8 +1,10 @@ +import { notifyNativeHarnessBackupExpiration } from "./native-harness-backup-expiration-notifications.js"; import { notifySkillVersionRetention } from "./skill-version-retention-notifications.js"; import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js"; export type NativeEnvironmentSettlementScope = { companyId: string; runId?: string; issueId?: string }; export async function notifyNativeEnvironmentSettlement(tx: object, scope?: NativeEnvironmentSettlementScope) { + await notifyNativeHarnessBackupExpiration(tx, scope && { companyId: scope.companyId }); if (scope) await notifySkillVersionRetention(tx, scope.companyId); await notifyDeliveryWork(tx, DELIVERY_QUEUES.nativeEnvironmentSettlement, scope ? JSON.stringify({ companyId: scope.companyId, runId: scope.runId, issueId: scope.issueId }) : "*"); diff --git a/server/src/services/native-harness-backup-expiration-notifications.ts b/server/src/services/native-harness-backup-expiration-notifications.ts new file mode 100644 index 00000000000..602a7063acd --- /dev/null +++ b/server/src/services/native-harness-backup-expiration-notifications.ts @@ -0,0 +1,19 @@ +import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js"; + +export type NativeHarnessBackupExpirationScope = { companyId: string; scopeDigest?: string }; + +export function notifyNativeHarnessBackupExpiration(tx: object, scope?: NativeHarnessBackupExpirationScope) { + return notifyDeliveryWork(tx, DELIVERY_QUEUES.nativeHarnessBackupExpiration, scope ? JSON.stringify(scope) : "*"); +} + +const listeners = new WeakMap void>>(); +export function onNativeHarnessBackupChange(db: object, listener: (scope: NativeHarnessBackupExpirationScope) => void) { + let owned = listeners.get(db); + if (!owned) { owned = new Set(); listeners.set(db, owned); } + owned.add(listener); + return () => { owned.delete(listener); if (!owned.size) listeners.delete(db); }; +} + +export function noteNativeHarnessBackupChange(db: object, scope: NativeHarnessBackupExpirationScope) { + for (const listener of listeners.get(db) ?? []) listener(scope); +} diff --git a/server/src/services/native-runtime/native-harness-backup-expiration-work.ts b/server/src/services/native-runtime/native-harness-backup-expiration-work.ts new file mode 100644 index 00000000000..93c4f08e340 --- /dev/null +++ b/server/src/services/native-runtime/native-harness-backup-expiration-work.ts @@ -0,0 +1,53 @@ +import type { Db } from "@paperclipai/db"; +import { logger } from "../../middleware/logger.js"; +import type { createDeliveryWorkCoordinator } from "../delivery-work-coordinator.js"; +import { DELIVERY_QUEUES } from "../delivery-work-notifications.js"; +import { onNativeHarnessBackupChange, type NativeHarnessBackupExpirationScope } from "../native-harness-backup-expiration-notifications.js"; +import { onRetainedNativeSessionCleanupRelease } from "../retained-native-session-cleanup-notifications.js"; +import { registerScopedDeadlineWork } from "../scoped-deadline-work.js"; +import { nativeHarnessBackupExpirationService, type NativeHarnessBackupExpirationOptions } from "./native-harness-backup-expiration.js"; + +type Entry = { id: string; companyId: string; agentId: string; dueAt: number }; + +export function registerNativeHarnessBackupExpirationWork( + coordinator: ReturnType, db: Db, + options: NativeHarnessBackupExpirationOptions, canRun: () => boolean, +) { + const service = nativeHarnessBackupExpirationService(db, { ...options, duty: "backups" }); + const cursors = new Map(); + const worker = registerScopedDeadlineWork(coordinator, { + queue: DELIVERY_QUEUES.nativeHarnessBackupExpiration, batchSize: options.maxRemovals ?? 25, + decodeScope: (hint: string): NativeHarnessBackupExpirationScope | undefined => hint === "*" ? undefined : JSON.parse(hint), + matches: (entry: Entry, scope?: NativeHarnessBackupExpirationScope) => !scope || entry.companyId === scope.companyId + && (!scope.scopeDigest || entry.id === scope.scopeDigest), + load: async (scope, signal) => { + const key = scope ? JSON.stringify(scope) : "*"; + const page = scope?.scopeDigest ? { names: [scope.scopeDigest], complete: true } + : await service.backupScopes(cursors.get(key), options.maxEntries ?? 100); + const rows: Entry[] = []; + for (const name of page.names) { + if (signal?.aborted || !canRun()) throw new Error("Backup expiration discovery paused"); + const entry = await service.inspectBackup(name); + if (entry && (!scope || entry.companyId === scope.companyId)) rows.push(entry); + } + if (page.complete) cursors.delete(key); else cursors.set(key, page.names.at(-1)!); + return { rows, complete: page.complete }; + }, + check: async (entry, signal) => { + if (signal?.aborted || !canRun()) throw new Error("Backup expiration paused"); + const current = await service.inspectBackup(entry.id); + if (!current || current.companyId !== entry.companyId) return null; + if (current.dueAt > Date.now()) return current; + const result = await service.expireBackup(entry.id); + if (result.candidates || result.failed) logger.info({ ...result, dryRun: options.dryRun ?? false }, "native harness backup expiration"); + if (options.dryRun) return null; + const remaining = await service.inspectBackup(entry.id); + if (!remaining) return null; + if (result.skipped.session_active) throw new Error("Backup session is active"); + return remaining; + }, + }, canRun); + const unsubscribe = onNativeHarnessBackupChange(db, scope => worker.invalidate(JSON.stringify(scope))); + const release = onRetainedNativeSessionCleanupRelease(db, scope => worker.invalidate(JSON.stringify({ companyId: scope.companyId }))); + return { ...worker, async stop() { unsubscribe(); release(); await worker.stop(); } }; +} diff --git a/server/src/services/native-runtime/native-harness-backup-expiration.ts b/server/src/services/native-runtime/native-harness-backup-expiration.ts index 8ea9a242629..2d45b12135f 100644 --- a/server/src/services/native-runtime/native-harness-backup-expiration.ts +++ b/server/src/services/native-runtime/native-harness-backup-expiration.ts @@ -170,7 +170,7 @@ type Tx = Parameters[0]>[0]; async function sessionRetentionBlocker( tx: Tx, input: { scopeDigest: string | null; identity: SessionIdentity; closedBefore: Date; checkpoint?: boolean }, -): Promise<{ reason: string } | { companyId: string; agentId: string; runIds: string[] }> { +): Promise<{ reason: string } | { companyId: string; agentId: string; runIds: string[]; issueIds: string[]; closedAt: number }> { const [owner] = await tx .select({ companyId: heartbeatRuns.companyId, agentId: heartbeatRuns.agentId }) .from(heartbeatRuns) @@ -246,7 +246,7 @@ async function sessionRetentionBlocker( .where(and(inArray(environmentLeases.status, LIVE_LEASE_STATUSES), or(...leaseReferences))) .limit(1); if (lease) return { reason: "live_lease" }; - return { ...owner, runIds }; + return { ...owner, runIds, issueIds, closedAt: Math.max(...issueRows.map(issue => new Date(issue.closedAt!).getTime())) }; } async function hasLiveRun(db: Db, owner: { companyId: string; agentId: string }, sessionId: string) { @@ -379,54 +379,48 @@ export function nativeHarnessBackupExpirationService( const identity = await readSessionIdentity(root); if (!identity) return skip("identity_unknown"); - const decision = await db.transaction(async (tx) => { - const blocker = await sessionRetentionBlocker(tx, { - scopeDigest, - identity, - closedBefore: options.closedBefore, - }); - if ("reason" in blocker) return blocker; - if (options.dryRun) return { owner: blocker, cleared: 0 }; - await notifyExternalConversationSettlement(tx, { companyId: blocker.companyId }); - const cleared = await tx - .delete(agentTaskSessions) - .where( - and( - eq(agentTaskSessions.companyId, blocker.companyId), - eq(agentTaskSessions.agentId, blocker.agentId), - sql`${agentTaskSessions.sessionParamsJson}->>'sessionId' = ${identity.normalizedSessionId}`, - ), - ) - .returning({ id: agentTaskSessions.id }); - return { owner: blocker, cleared: cleared.length }; - }); - if ("reason" in decision) return skip(decision.reason); - - const bytes = await allocatedBytes(current); - result.candidates++; - result.candidateBytes += bytes; - if (options.dryRun) return; - result.clearedTaskSessions += decision.cleared; - if (options.isSessionBusy(scopeDigest)) return skip("session_active"); const id = randomUUID(); const retiring = path.join(backupRoot, `${RETIRING_PREFIX}${id}`); const trash = path.join(backupRoot, `${EXPIRED_PREFIX}${id}`); - await fs.rename(current, retiring); - let liveRun: boolean; + let retired = false; + let decision: { reason: string } | { cleared: number; bytes: number }; try { - liveRun = await hasLiveRun(db, decision.owner, identity.normalizedSessionId); - } catch (err) { - await fs.rename(retiring, current); - throw err; - } - if (liveRun) { - await fs.rename(retiring, current); - return skip("live_run"); + decision = await db.transaction(async (tx) => { + const blocker = await sessionRetentionBlocker(tx, { + scopeDigest, identity, closedBefore: options.closedBefore, + }); + if ("reason" in blocker) return blocker; + const bytes = await allocatedBytes(current); + if (options.dryRun) return { cleared: 0, bytes }; + if (options.isSessionBusy(scopeDigest)) return { reason: "session_active" }; + await notifyExternalConversationSettlement(tx, { companyId: blocker.companyId }); + await fs.rename(current, retiring); + retired = true; + if (await hasLiveRun(tx as unknown as Db, blocker, identity.normalizedSessionId)) { + await fs.rename(retiring, current); + retired = false; + return { reason: "live_run" }; + } + const cleared = await tx.delete(agentTaskSessions).where(and( + eq(agentTaskSessions.companyId, blocker.companyId), + eq(agentTaskSessions.agentId, blocker.agentId), + sql`${agentTaskSessions.sessionParamsJson}->>'sessionId' = ${identity.normalizedSessionId}`, + )).returning({ id: agentTaskSessions.id }); + return { cleared: cleared.length, bytes }; + }); + } catch (error) { + if (retired) await fs.rename(retiring, current); + throw error; } + if ("reason" in decision) return skip(decision.reason); + result.candidates++; + result.candidateBytes += decision.bytes; + if (options.dryRun) return; + result.clearedTaskSessions += decision.cleared; await fs.rename(retiring, trash); await removeBackupTree(trash); result.expired++; - result.reclaimedBytes += bytes; + result.reclaimedBytes += decision.bytes; } async function removeExpiredLeftovers(parent: string, context: EvidenceContext, result: NativeHarnessBackupExpirationResult) { @@ -745,5 +739,55 @@ export function nativeHarnessBackupExpirationService( }); } - return { tick, noteEvidence, nextEvidenceAt, tickCheckpoints, registerCheckpointExpiration }; + async function inspectBackup(scopeDigest: string) { + const base = path.resolve(defaults.stateBase ?? nativeHarnessStateBase()); + const root = path.join(base, scopeDigest); + if (!SCOPE_PATTERN.test(scopeDigest) || !(await isRealDirectory(root))) return null; + const backupRoot = path.join(root, "failover-backups"); + if (!(await isRealDirectory(backupRoot))) return null; + const names = await fs.readdir(backupRoot); + const current = path.join(backupRoot, "current"); + const stat = await fs.lstat(current).catch(missingPath); + const leftovers = names.some(name => [RETIRING_PREFIX, EXPIRED_PREFIX].some(prefix => + name.startsWith(prefix) && UUID_PATTERN.test(name.slice(prefix.length)))); + if ((!stat?.isDirectory() || stat.isSymbolicLink()) && !leftovers) return null; + if (defaults.dryRun && !stat?.isDirectory()) return null; + const identity = await readSessionIdentity(root); + if (!identity) return null; + const [owner] = await db.select({ companyId: heartbeatRuns.companyId, agentId: heartbeatRuns.agentId }) + .from(heartbeatRuns).where(eq(heartbeatRuns.id, identity.runId)); + if (!owner) return null; + if (leftovers && !defaults.dryRun) return { id: scopeDigest, ...owner, dueAt: 0 }; + if (await fs.lstat(path.join(root, "cleanup-activation.json")).then(() => true, () => false)) return null; + const blocker = await db.transaction(tx => sessionRetentionBlocker(tx, { + scopeDigest, identity, closedBefore: new Date(8_000_000_000_000_000), + })); + if ("reason" in blocker) return null; + const retentionMs = Math.max(1, defaults.closedForDays ?? NATIVE_HARNESS_BACKUP_EXPIRATION_DEFAULT_DAYS) * DAY_MS; + return { id: scopeDigest, ...owner, dueAt: Math.max(stat!.mtimeMs, blocker.closedAt) + retentionMs + 1 }; + } + + async function backupScopes(after?: string, limit = 100) { + const base = path.resolve(defaults.stateBase ?? nativeHarnessStateBase()); + const names = (await fs.readdir(base).catch((error: NodeJS.ErrnoException) => { + if (error.code === "ENOENT") return [] as string[]; + throw error; + })).filter(name => SCOPE_PATTERN.test(name) && (!after || name > after)).sort(); + return { names: names.slice(0, limit), complete: names.length <= limit }; + } + + async function expireBackup(scopeDigest: string) { + const result = emptyResult(); + const root = path.join(path.resolve(defaults.stateBase ?? nativeHarnessStateBase()), scopeDigest); + const dryRun = defaults.dryRun ?? false; + await expireSession(root, scopeDigest, { + dryRun, + isSessionBusy: defaults.isSessionBusy ?? nativeSessionScopeDigestIsBusy, + closedBefore: new Date(Date.now() - Math.max(1, defaults.closedForDays ?? NATIVE_HARNESS_BACKUP_EXPIRATION_DEFAULT_DAYS) * DAY_MS), + }, result); + return result; + } + + return { tick, noteEvidence, nextEvidenceAt, tickCheckpoints, registerCheckpointExpiration, + inspectBackup, backupScopes, expireBackup }; } diff --git a/server/src/services/native-runtime/native-session-executor.ts b/server/src/services/native-runtime/native-session-executor.ts index 4f5c5ae99de..b47ddc40fee 100644 --- a/server/src/services/native-runtime/native-session-executor.ts +++ b/server/src/services/native-runtime/native-session-executor.ts @@ -1,3 +1,4 @@ +import { noteNativeHarnessBackupChange } from "../native-harness-backup-expiration-notifications.js"; import { createNativeUsageDelivery } from "./native-usage-delivery.js"; import { createNativeTurnAccounting, nativeUsageCostUsd, nativeUsageBiller, normalizeNativeUsage, usesNativeUsageDelivery, isNativeUsageAccountingEvent } from "./native-usage-accounting.js"; export { nativeUsageCostUsd, nativeUsageBiller, normalizeNativeUsage } from "./native-usage-accounting.js"; @@ -13310,6 +13311,8 @@ async function createRunnerdBackendWithinSessionClaim( throw error; } removeNativeHarnessBackup(previousRoot); + noteNativeHarnessBackupChange(input.db, { companyId: input.execution.binding.companyId, + scopeDigest: createHash("sha256").update(sessionScopeId).digest("hex") }); } finally { removeNativeHarnessBackup(pendingRoot); } diff --git a/server/src/services/retained-native-session-cleanup-notifications.ts b/server/src/services/retained-native-session-cleanup-notifications.ts index 1ff39143e0e..d5aaff7c31a 100644 --- a/server/src/services/retained-native-session-cleanup-notifications.ts +++ b/server/src/services/retained-native-session-cleanup-notifications.ts @@ -1,3 +1,4 @@ +import { notifyNativeHarnessBackupExpiration } from "./native-harness-backup-expiration-notifications.js"; import { notifySkillVersionRetention } from "./skill-version-retention-notifications.js"; import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js"; @@ -9,6 +10,7 @@ export type RetainedNativeSessionCleanupScope = { }; export async function notifyRetainedNativeSessionCleanup(tx: object, scope: RetainedNativeSessionCleanupScope) { + await notifyNativeHarnessBackupExpiration(tx, { companyId: scope.companyId }); await notifySkillVersionRetention(tx, scope.companyId); await notifyDeliveryWork(tx, DELIVERY_QUEUES.retainedNativeSessionCleanup, JSON.stringify({ companyId: scope.companyId, runId: scope.runId, issueId: scope.issueId, agentId: scope.agentId })); } From 79d1e3a404aa929cb5182d9e37cb23f2fa2ac52a Mon Sep 17 00:00:00 2001 From: Dotta Date: Fri, 30 Oct 2026 16:07:12 -0500 Subject: [PATCH 03/74] test(server): qualify native backup publication and retention recovery Co-Authored-By: Paperclip --- .../native-harness-backup-expiration.test.ts | 69 ++++++++++++++++++- .../native-harness-backup-expiration.ts | 2 +- .../native-harness-backup-publication.test.ts | 44 ++++++++++++ .../native-harness-backup-publication.ts | 28 ++++++++ .../native-runtime/native-session-executor.ts | 43 +++--------- 5 files changed, 149 insertions(+), 37 deletions(-) create mode 100644 server/src/services/native-runtime/native-harness-backup-publication.test.ts create mode 100644 server/src/services/native-runtime/native-harness-backup-publication.ts diff --git a/server/src/__tests__/native-harness-backup-expiration.test.ts b/server/src/__tests__/native-harness-backup-expiration.test.ts index be38a9ae0d9..1ac2539545e 100644 --- a/server/src/__tests__/native-harness-backup-expiration.test.ts +++ b/server/src/__tests__/native-harness-backup-expiration.test.ts @@ -1,5 +1,6 @@ +import { publishNativeHarnessBackup } from "../services/native-runtime/native-harness-backup-publication.js"; import { registerNativeHarnessBackupExpirationWork } from "../services/native-runtime/native-harness-backup-expiration-work.js"; -import { noteNativeHarnessBackupChange, notifyNativeHarnessBackupExpiration } from "../services/native-harness-backup-expiration-notifications.js"; +import { noteNativeHarnessBackupChange } from "../services/native-harness-backup-expiration-notifications.js"; import { writeEnvironmentCleanup } from "../services/environment-cleanup-notifications.js"; import { issueService } from "../services/issues.js"; import { applyPendingMigrations } from "@paperclipai/db"; @@ -583,6 +584,10 @@ describeEmbeddedPostgres("native harness backup expiration", () => { if (history) { const session = await seedSession(); await fs.rm(path.join(session.root, "failover-backups"), { recursive: true }); + await db.insert(heartbeatRuns).values(Array.from({ length: 500 }, () => ({ + companyId: session.companyId, agentId: session.agentId, status: "succeeded", issueId: session.issueId, + nativeIssueId: session.issueId, nativeSessionId: randomUUID(), finishedAt: now, + }))); for (let index = 0; index < 250; index++) await fs.mkdir(path.join(stateBase, createHash("sha256").update(String(index)).digest("hex"))); } @@ -609,6 +614,68 @@ describeEmbeddedPostgres("native harness backup expiration", () => { expect(backupErrors).toEqual([]); await quiet(); }); + it("ordinary backup publication after quiet schedules its actual file age and leaves a durable cleanup result", async () => { + const session = await seedSession(); + const backupRoot = path.dirname(session.current); + const pendingRoot = path.join(backupRoot, "pending-publication"); + await fs.rename(session.current, pendingRoot); + await startBackups(); await settled(); await quiet(); + const publishedAt = Date.now(); + await fs.utimes(pendingRoot, new Date(publishedAt), new Date(publishedAt)); + await publishNativeHarnessBackup({ db, companyId: session.companyId, scopeDigest: session.scope, backupRoot, pendingRoot }); + const stat = await fs.stat(session.current); + const due = stat.mtime.getTime() + 7 * DAY_MS + 1; + await settled(due); await vi.advanceTimersByTimeAsync(due - Date.now()); await settled(); + expect(await exists(session.current)).toBe(false); expect(await taskSessionCount(session.agentId)).toBe(0); + expect(await fs.readdir(backupRoot)).toEqual([]); await quiet(); + }); + + it("a committed earlier close timestamp replaces the later deadline without early cleanup", async () => { + const session = await seedSession({ closedDaysAgo: 2 }); + await startBackups(); await settled(now.getTime() + 5 * DAY_MS + 1); + const earlier = new Date(now.getTime() - 6 * DAY_MS); + await issueService(db).update(session.issueId, { completedAt: earlier }); + const due = earlier.getTime() + 7 * DAY_MS + 1; + await settled(due); await quiet(60_000); + await vi.advanceTimersByTimeAsync(due - Date.now()); await settled(); + expect(await exists(session.current)).toBe(false); await quiet(); + }); + + it("holds nested ordinary close activation until the outer transaction commits", async () => { + const session = await seedSession({ issueStatus: "todo" }); + await startBackups(); await settled(); + let release!: () => void, written!: () => void; + const gate = new Promise(resolve => { release = resolve; }); + const staged = new Promise(resolve => { written = resolve; }); + const transaction = db.transaction(async outer => { + await outer.transaction(nested => issueService(db).update(session.issueId, { status: "done" }, nested)); + written(); await gate; + }); + await staged; + await vi.advanceTimersByTimeAsync(60_000); + expect(await exists(session.current)).toBe(true); + expect(await taskSessionCount(session.agentId)).toBe(1); + release(); await transaction; + const [closed] = await db.select().from(issues).where(eq(issues.id, session.issueId)); + const due = new Date(closed!.completedAt!).getTime() + 7 * DAY_MS + 1; + await settled(due); await vi.advanceTimersByTimeAsync(due - Date.now()); await settled(); + expect(await exists(session.current)).toBe(false); await quiet(); + }); + + it("preserves failed cleanup retry across drain, then finishes its physical tombstone on resume", async () => { + const session = await seedSession(); const remove = fs.rm; + const fault = vi.spyOn(fs, "rm").mockImplementation(async (...args) => { + if (String(args[0]).includes(".expired-")) throw new Error("controlled drain retry failure"); + return remove(...args); + }); + await startBackups(); + expect(backupErrors).toHaveLength(1); admitted = false; fault.mockRestore(); + await quiet(60_000); + expect((await fs.readdir(path.dirname(session.current))).some(name => name.startsWith(".expired-"))).toBe(true); + admitted = true; await settled(); + expect(await fs.readdir(path.dirname(session.current))).toEqual([]); await quiet(); + }); + it("reopen cancels a saved deadline and a later ordinary close restores retention", async () => { const session = await seedSession({ closedDaysAgo: 2 }); await startBackups(); await settled(now.getTime() + 5 * DAY_MS + 1); diff --git a/server/src/services/native-runtime/native-harness-backup-expiration.ts b/server/src/services/native-runtime/native-harness-backup-expiration.ts index 2d45b12135f..09ddea9e5d2 100644 --- a/server/src/services/native-runtime/native-harness-backup-expiration.ts +++ b/server/src/services/native-runtime/native-harness-backup-expiration.ts @@ -764,7 +764,7 @@ export function nativeHarnessBackupExpirationService( })); if ("reason" in blocker) return null; const retentionMs = Math.max(1, defaults.closedForDays ?? NATIVE_HARNESS_BACKUP_EXPIRATION_DEFAULT_DAYS) * DAY_MS; - return { id: scopeDigest, ...owner, dueAt: Math.max(stat!.mtimeMs, blocker.closedAt) + retentionMs + 1 }; + return { id: scopeDigest, ...owner, dueAt: Math.max(stat!.mtime.getTime(), blocker.closedAt) + retentionMs + 1 }; } async function backupScopes(after?: string, limit = 100) { diff --git a/server/src/services/native-runtime/native-harness-backup-publication.test.ts b/server/src/services/native-runtime/native-harness-backup-publication.test.ts new file mode 100644 index 00000000000..dd9d940d9df --- /dev/null +++ b/server/src/services/native-runtime/native-harness-backup-publication.test.ts @@ -0,0 +1,44 @@ +import { promises as fs } from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import { onNativeHarnessBackupChange } from "../native-harness-backup-expiration-notifications.js"; +import { publishNativeHarnessBackup } from "./native-harness-backup-publication.js"; + +describe("native harness backup publication", () => { + let backupRoot = ""; + let unsubscribe: (() => void) | undefined; + afterEach(async () => { + unsubscribe?.(); unsubscribe = undefined; + if (backupRoot) await fs.rm(backupRoot, { recursive: true, force: true }); + }); + async function fixture() { + backupRoot = await fs.mkdtemp(path.join(os.tmpdir(), "native-backup-publication-")); + const current = path.join(backupRoot, "current"), pendingRoot = path.join(backupRoot, "pending"); + await fs.mkdir(current); await fs.mkdir(pendingRoot); + await fs.writeFile(path.join(current, "manifest.json"), "saved"); + await fs.writeFile(path.join(pendingRoot, "manifest.json"), "replacement"); + return { db: {}, companyId: "company", scopeDigest: "scope", backupRoot, pendingRoot, current }; + } + it("restores the existing backup when its lease stamp fails without publishing a false change", async () => { + const input = await fixture(); const changes: unknown[] = []; + unsubscribe = onNativeHarnessBackupChange(input.db, change => changes.push(change)); + await expect(publishNativeHarnessBackup({ ...input, stamp: async current => { + expect(await fs.readFile(path.join(current, "manifest.json"), "utf8")).toBe("replacement"); + throw new Error("stamp transaction rejected"); + } })).rejects.toThrow("stamp transaction rejected"); + expect(await fs.readFile(path.join(input.current, "manifest.json"), "utf8")).toBe("saved"); + expect(await fs.readdir(backupRoot)).toEqual(["current"]); expect(changes).toEqual([]); + }); + it("publishes the completed replacement once and removes its predecessor", async () => { + const input = await fixture(); const changes: unknown[] = []; + unsubscribe = onNativeHarnessBackupChange(input.db, change => changes.push(change)); + await publishNativeHarnessBackup({ ...input, stamp: async current => { + expect(changes).toEqual([]); + expect(await fs.readFile(path.join(current, "manifest.json"), "utf8")).toBe("replacement"); + } }); + expect(await fs.readFile(path.join(input.current, "manifest.json"), "utf8")).toBe("replacement"); + expect(await fs.readdir(backupRoot)).toEqual(["current"]); + expect(changes).toEqual([{ companyId: input.companyId, scopeDigest: input.scopeDigest }]); + }); +}); diff --git a/server/src/services/native-runtime/native-harness-backup-publication.ts b/server/src/services/native-runtime/native-harness-backup-publication.ts new file mode 100644 index 00000000000..820303584b0 --- /dev/null +++ b/server/src/services/native-runtime/native-harness-backup-publication.ts @@ -0,0 +1,28 @@ +import { existsSync, renameSync } from "node:fs"; +import { resolve } from "node:path"; +import { noteNativeHarnessBackupChange } from "../native-harness-backup-expiration-notifications.js"; +import { removeNativeHarnessBackup } from "./native-harness-backup-cleanup.js"; + +export async function publishNativeHarnessBackup(input: { + db: object; companyId: string; scopeDigest: string; backupRoot: string; pendingRoot: string; + stamp?: (currentRoot: string) => Promise; +}) { + const currentRoot = resolve(input.backupRoot, "current"); + const previousRoot = resolve(input.backupRoot, "previous"); + removeNativeHarnessBackup(previousRoot); + let movedCurrent = false; + if (existsSync(currentRoot)) { + renameSync(currentRoot, previousRoot); + movedCurrent = true; + } + try { + renameSync(input.pendingRoot, currentRoot); + await input.stamp?.(currentRoot); + } catch (error) { + if (existsSync(currentRoot)) removeNativeHarnessBackup(currentRoot); + if (movedCurrent && existsSync(previousRoot) && !existsSync(currentRoot)) renameSync(previousRoot, currentRoot); + throw error; + } + removeNativeHarnessBackup(previousRoot); + noteNativeHarnessBackupChange(input.db, { companyId: input.companyId, scopeDigest: input.scopeDigest }); +} diff --git a/server/src/services/native-runtime/native-session-executor.ts b/server/src/services/native-runtime/native-session-executor.ts index b47ddc40fee..894dc631cee 100644 --- a/server/src/services/native-runtime/native-session-executor.ts +++ b/server/src/services/native-runtime/native-session-executor.ts @@ -1,4 +1,4 @@ -import { noteNativeHarnessBackupChange } from "../native-harness-backup-expiration-notifications.js"; +import { publishNativeHarnessBackup } from "./native-harness-backup-publication.js"; import { createNativeUsageDelivery } from "./native-usage-delivery.js"; import { createNativeTurnAccounting, nativeUsageCostUsd, nativeUsageBiller, normalizeNativeUsage, usesNativeUsageDelivery, isNativeUsageAccountingEvent } from "./native-usage-accounting.js"; export { nativeUsageCostUsd, nativeUsageBiller, normalizeNativeUsage } from "./native-usage-accounting.js"; @@ -13277,42 +13277,15 @@ async function createRunnerdBackendWithinSessionClaim( }); renameSync(temporaryManifest, manifestPath); - const currentRoot = resolve(backupRoot, "current"); - const previousRoot = resolve(backupRoot, "previous"); - removeNativeHarnessBackup(previousRoot); - let movedCurrent = false; - if (existsSync(currentRoot)) { - renameSync(currentRoot, previousRoot); - movedCurrent = true; - } - try { - renameSync(pendingRoot, currentRoot); - if ( - remoteTarget?.transport === "sandbox" && - remoteTarget.leaseId - ) { + await publishNativeHarnessBackup({ + db: input.db, companyId: input.execution.binding.companyId, + scopeDigest: createHash("sha256").update(sessionScopeId).digest("hex"), backupRoot, pendingRoot, + stamp: remoteTarget?.transport === "sandbox" && remoteTarget.leaseId ? async currentRoot => { await recordHarnessBackupStampForCurrentLease({ - root: currentRoot, - manifest, - bytes: backupSpanAttributes.bytesTransferred, + root: currentRoot, manifest, bytes: backupSpanAttributes.bytesTransferred, }); - } - } catch (error) { - if (existsSync(currentRoot)) { - removeNativeHarnessBackup(currentRoot); - } - if ( - movedCurrent && - existsSync(previousRoot) && - !existsSync(currentRoot) - ) { - renameSync(previousRoot, currentRoot); - } - throw error; - } - removeNativeHarnessBackup(previousRoot); - noteNativeHarnessBackupChange(input.db, { companyId: input.execution.binding.companyId, - scopeDigest: createHash("sha256").update(sessionScopeId).digest("hex") }); + } : undefined, + }); } finally { removeNativeHarnessBackup(pendingRoot); } From acaa86e88c0a93d39e98e685efe1158919aa25dd Mon Sep 17 00:00:00 2001 From: Dotta Date: Fri, 30 Oct 2026 16:10:20 -0500 Subject: [PATCH 04/74] fix: release settled deadline work holds during drain Preserve dirty generations with an in-memory resume deadline while allowing settled transaction holds to end during drain. Keep unresolved database intent fenced by the coordinator. Co-Authored-By: Paperclip --- server/src/__tests__/deadline-work.test.ts | 85 +++++++++++++++++++++- server/src/services/deadline-work.ts | 13 ++-- 2 files changed, 92 insertions(+), 6 deletions(-) diff --git a/server/src/__tests__/deadline-work.test.ts b/server/src/__tests__/deadline-work.test.ts index 7b852977345..7657a7e9d99 100644 --- a/server/src/__tests__/deadline-work.test.ts +++ b/server/src/__tests__/deadline-work.test.ts @@ -1,4 +1,5 @@ -import { installDatabaseWorkSignals } from "../../../packages/db/src/work-signals.js"; +import { idleWorkSnapshot } from "../services/task-admission.js"; +import { installDatabaseWorkSignals, signalDatabaseWork, databaseWorkPending } from "../../../packages/db/src/work-signals.js"; import { afterEach, expect, it, vi } from "vitest"; import { createDeliveryWorkCoordinator } from "../services/delivery-work-coordinator.js"; import { registerDeadlineWork } from "../services/deadline-work.js"; @@ -39,3 +40,85 @@ it.each([ expect(run).toHaveBeenCalledTimes(2); expect(onError).not.toHaveBeenCalled(); }); + +it("releases a settled first-write hold during drain and retains a probe-free resume wake", async () => { + vi.useFakeTimers(); + let draining = false; + const owner = installDatabaseWorkSignals({ + execute: vi.fn(async () => [{ status: "committed" }]), + transaction: async (callback: (tx: object) => Promise) => callback({ execute: async () => [{ xid: "1" }] }), + }); + const run = vi.fn(async () => {}), next = vi.fn(async () => null); + coordinator = createDeliveryWorkCoordinator({ owner, canRun: () => !draining, canReconcile: () => true, onError: vi.fn() }); + const worker = registerDeadlineWork(coordinator, DELIVERY_QUEUES.agentTimers, { run, next, canRun: () => !draining, retryMs: 30_000 }); + await worker.ready; + draining = true; + const before = idleWorkSnapshot().active; + await owner.transaction(async tx => { await signalDatabaseWork(tx, DELIVERY_QUEUES.agentTimers); }); + await vi.advanceTimersByTimeAsync(0); + expect(idleWorkSnapshot().active).toBe(before); + expect(coordinator.nextWakeAt()).not.toBeNull(); + await vi.advanceTimersByTimeAsync(3 * 30_000); + expect(run).toHaveBeenCalledOnce(); expect(next).toHaveBeenCalledOnce(); expect(owner.execute).not.toHaveBeenCalled(); + draining = false; + await vi.advanceTimersByTimeAsync(30_000); + expect(run).toHaveBeenCalledTimes(2); expect(coordinator.nextWakeAt()).toBeNull(); +}); + +it("retains failed work through drain and preserves a future domain deadline", async () => { + vi.useFakeTimers(); + let draining = false; + const owner = installDatabaseWorkSignals({ execute: vi.fn(async () => []) }); + const run = vi.fn(async () => { if (run.mock.calls.length === 1) { draining = true; throw new Error("retry debt"); } }); + const deadline = Date.now() + 600_000; + const next = vi.fn(async () => deadline); + const onError = vi.fn(); + coordinator = createDeliveryWorkCoordinator({ owner, canRun: () => !draining, canReconcile: () => true, onError }); + const worker = registerDeadlineWork(coordinator, DELIVERY_QUEUES.agentTimers, { run, next, canRun: () => !draining, retryMs: 30_000 }); + await worker.ready; + await vi.advanceTimersByTimeAsync(90_000); + expect(run).toHaveBeenCalledOnce(); expect(next).not.toHaveBeenCalled(); expect(owner.execute).not.toHaveBeenCalled(); + draining = false; await vi.advanceTimersByTimeAsync(30_000); + expect(run).toHaveBeenCalledTimes(2); expect(coordinator.nextWakeAt()).toBe(deadline); + draining = true; await vi.advanceTimersByTimeAsync(30_000); + expect(coordinator.nextWakeAt()).toBe(deadline); expect(onError).toHaveBeenCalledOnce(); +}); + +it("does no domain probes in warm standby and reconstructs after admission opens", async () => { + vi.useFakeTimers(); + let standby = true; + const owner = installDatabaseWorkSignals({ execute: vi.fn(async () => []) }); + const run = vi.fn(async () => {}), next = vi.fn(async () => null); + coordinator = createDeliveryWorkCoordinator({ owner, canRun: () => !standby, canReconcile: () => !standby, onError: vi.fn() }); + const worker = registerDeadlineWork(coordinator, DELIVERY_QUEUES.agentTimers, { run, next, canRun: () => !standby, retryMs: 30_000 }); + await worker.ready; await vi.advanceTimersByTimeAsync(90_000); + expect(run).not.toHaveBeenCalled(); expect(next).not.toHaveBeenCalled(); expect(owner.execute).not.toHaveBeenCalled(); + standby = false; await vi.advanceTimersByTimeAsync(30_000); + expect(run).toHaveBeenCalledOnce(); expect(coordinator.nextWakeAt()).toBeNull(); +}); + +it("does not release an ambiguous database intent while dispatch is drained", async () => { + vi.useFakeTimers(); + let draining = false, committed = false; + const owner = installDatabaseWorkSignals({ + execute: vi.fn(async () => [{ status: committed ? "committed" : "in progress" }]), + transaction: async (callback: (tx: object) => Promise) => { + await callback({ execute: async () => [{ xid: "2" }] }); + throw new Error("lost acknowledgement"); + }, + }); + coordinator = createDeliveryWorkCoordinator({ owner, canRun: () => !draining, canReconcile: () => true, onError: vi.fn() }); + const run = vi.fn(async () => {}); + const worker = registerDeadlineWork(coordinator, DELIVERY_QUEUES.agentTimers, { run, next: async () => null, canRun: () => !draining, retryMs: 30_000 }); + await worker.ready; draining = true; + const before = idleWorkSnapshot().active; + await expect(owner.transaction(async tx => { await signalDatabaseWork(tx, DELIVERY_QUEUES.agentTimers); })).rejects.toThrow("lost acknowledgement"); + await vi.advanceTimersByTimeAsync(0); + expect(idleWorkSnapshot().active).toBe(before + 1); + expect(databaseWorkPending(owner, DELIVERY_QUEUES.agentTimers)).toBe(true); + committed = true; await vi.advanceTimersByTimeAsync(30_000); + expect(databaseWorkPending(owner, DELIVERY_QUEUES.agentTimers)).toBe(false); + expect(idleWorkSnapshot().active).toBe(before); + draining = false; await vi.advanceTimersByTimeAsync(30_000); + expect(run).toHaveBeenCalledTimes(2); expect(coordinator.nextWakeAt()).toBeNull(); +}); diff --git a/server/src/services/deadline-work.ts b/server/src/services/deadline-work.ts index 306f340d9fc..15ba76abdae 100644 --- a/server/src/services/deadline-work.ts +++ b/server/src/services/deadline-work.ts @@ -26,7 +26,7 @@ export function registerDeadlineWork( pausePassiveMaintenance: true, onSettled: hints => { requested++; task.onSettled?.(hints); }, run: async signal => { - if (!task.canRun()) { requested++; next = null; return; } + if (!task.canRun()) { requested++; return; } if (Date.now() < earliest) { next = earliest; return; } if (Date.now() < coalesceUntil && (next === null || next > Date.now())) { next = next === null ? coalesceUntil : Math.min(next, coalesceUntil); @@ -39,8 +39,6 @@ export function registerDeadlineWork( if (signal.aborted || !task.canRun()) { next = null; return; } const deadline = await task.next(); - - if (!task.canRun()) { next = null; return; } earliest = deadline === null || deadline > Date.now() ? (task.minimumSpacingMs ? Date.now() + task.minimumSpacingMs : 0) @@ -49,8 +47,13 @@ export function registerDeadlineWork( coalesceUntil = Date.now() + (task.notificationCoalesceMs ?? 0); reconciled = generation; }, - hasPending: async () => requested !== reconciled, - nextRunAt: () => next, + // A settled producer must release its admission hold during drain. Keep + // the unreconciled generation and an in-memory resume wake, without asking + // the domain to probe its database while dispatch is closed. + hasPending: async () => task.canRun() && requested !== reconciled, + nextRunAt: () => !task.canRun() && requested !== reconciled + ? Math.min(next ?? Infinity, Date.now() + task.retryMs) + : next, }); return { ...worker, wake: () => { requested++; worker.wake(); } }; } From e55bc2df65d2a73125d1ed4c02fff69bc67c078a Mon Sep 17 00:00:00 2001 From: Dotta Date: Fri, 30 Oct 2026 16:19:27 -0500 Subject: [PATCH 05/74] fix(server): schedule only outstanding merge confirmations Co-Authored-By: Paperclip --- ...026-10-30-merged-pr-confirmations-sleep.md | 83 ++++++ .../merged-pr-confirmation-sweep.test.ts | 11 +- .../merged-pr-confirmation-work.test.ts | 278 ++++++++++++++++++ .../server-startup-feedback-export.test.ts | 5 +- server/src/index.ts | 17 +- .../chat-confirmation-notifications.ts | 2 + .../services/delivery-work-notifications.ts | 1 + .../src/services/issue-thread-interactions.ts | 101 ++++--- server/src/services/issues.ts | 4 + .../merged-pr-confirmation-notifications.ts | 7 + .../services/merged-pr-confirmation-work.ts | 30 ++ 11 files changed, 474 insertions(+), 65 deletions(-) create mode 100644 doc/plans/2026-10-30-merged-pr-confirmations-sleep.md create mode 100644 server/src/__tests__/merged-pr-confirmation-work.test.ts create mode 100644 server/src/services/merged-pr-confirmation-notifications.ts create mode 100644 server/src/services/merged-pr-confirmation-work.ts diff --git a/doc/plans/2026-10-30-merged-pr-confirmations-sleep.md b/doc/plans/2026-10-30-merged-pr-confirmations-sleep.md new file mode 100644 index 00000000000..fb8b091f841 --- /dev/null +++ b/doc/plans/2026-10-30-merged-pr-confirmations-sleep.md @@ -0,0 +1,83 @@ +# Merged pull request confirmation scheduling + +The merged-PR confirmation duty previously ran from every heartbeat maintenance +callback, including when there were no merge requests. It loaded all pending +confirmation cards and then applied the merge-only parser in memory. + +The interaction service still owns merge-only eligibility, provider access, +automatic acceptance, current resolver authority, audit, and continuation. +The worker registers the existing scoped-deadline helper with a dedicated delivery +topic. Shared scheduling receives interaction/company identities and deadlines; +it receives no GitHub payloads. No database migration is required. + +## Durable truth and producers + +Pending `request_confirmation` interaction rows remain the durable work source. +The candidate query excludes terminal and hidden issues, linked tool actions, +and text without merge intent or a PR reference. The existing fail-closed parser +then excludes additional actions, governed actions, and document/attachment targets. +No accepted or cancelled history keeps a deadline alive. + +Ordinary creation registers the topic before inserting a confirmation in its +transaction. Existing confirmation-resolution notifications invalidate its issue +before acceptance, rejection, withdrawal, supersession, and administrative expiry. +Issue status and visibility changes invalidate that issue before their update. +The signal helper joins caller-owned outer transactions; nested savepoint release +is not a commit. Startup reconstructs existing candidates. Direct tool, secret, +and OAuth interaction inserts do not create eligible merge-only work. Company +portability does not import pending interaction rows. + +The supported writer model is the instance's ordinary service/API process. +Independent SQL writers must use the same transaction-aware notification contract +or explicitly invalidate/restart the owning worker. Process-local notifications +do not establish a cross-replica ingress guarantee. No permanent database listener +or empty recovery scan is added. + +## External observation and retries + +The existing normalized `applyPullRequestEvent` to +`sweepMergedPullRequestConfirmations` seam remains available. A normalized merge +hint checks matching references in its company; unrelated tenants and PRs do not +cause provider reads. Existing provider/binding authorization remains unchanged. + +A pending candidate can reference a PR whose external changes have no dependable +webhook coverage. Each such candidate retains a five-minute observation deadline. +Resolution, withdrawal, supersession, hiding, or terminal issue state removes the +obligation. Provider/read failures and unknown states retain the candidate, using +the existing scoped helper's 30-second exponential backoff capped at five minutes. +These retries do not scan completed history. Removing candidate reconciliation +requires a proven durable delivery/replay guarantee for every referenced PR, +including the first event after idle and supported non-Cloud installations. + +This is candidate-backed external observation. It does not eliminate all polling, +register a Cloud wake alarm, or prove that the host/database suspends. Existing +sleep-safety blockers remain in force. + +## Acceptance evidence + +Focused tests use real PostgreSQL, the actual service producers, transaction +signals, coordinator, scheduler, acceptance transaction, and persisted audit and +chat continuation outbox. GitHub state and the direct heartbeat wake callback are +controlled simulation boundaries. The tests do not claim live GitHub delivery or +real agent execution. + +Cases cover empty/configured idle and retained terminal history with no recurring +SQL/provider work across old cadence windows; the first ordinary creation; +outer commit/rollback; exact candidate and failure deadlines; withdrawal; +provider failure across drain; standby/resume; restart; tenant-scoped and duplicate +merge hints; latest/superseded cards during provider observation; hidden/terminal +issues; URL targets; no-continuation policy; and shutdown during an in-flight read. +Existing startup and normalized GitHub event tests exercise the changed consumers. + +## Existing limitations outside this migration + +Direct non-chat continuation still uses the existing postcommit `wakeup` call. +If that call fails after acceptance commits, the previous periodic sweep also +skipped the accepted row and did not reconstruct the missing wake. This migration +preserves that behavior; a general durable interaction-continuation contract is +a separate follow-up. Real chat bindings retain the existing transactional +`interaction_wakeup` outbox, verified through ordinary creation and acceptance. + +The separate review-authority issue tracked in public issue #17704 and PR #17711 +remains owned there. The worker isolates candidate failures and preserves current +review authorization; it does not grant a system actor new review authority. diff --git a/server/src/__tests__/merged-pr-confirmation-sweep.test.ts b/server/src/__tests__/merged-pr-confirmation-sweep.test.ts index 1e4d56436e7..d9cf9880357 100644 --- a/server/src/__tests__/merged-pr-confirmation-sweep.test.ts +++ b/server/src/__tests__/merged-pr-confirmation-sweep.test.ts @@ -1,3 +1,4 @@ +import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js"; import { randomUUID } from "node:crypto"; import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest"; import { inArray } from "drizzle-orm"; @@ -131,7 +132,8 @@ describe("merged pull-request confirmation extraction", () => { }); }); -const embeddedPostgresSupport = await getEmbeddedPostgresTestSupport(); +const external = resolveExternalTestDatabaseUrl("PAPERCLIP_MERGE_SWEEP_TEST_DATABASE_URL"); +const embeddedPostgresSupport = external ? { supported: true } : await getEmbeddedPostgresTestSupport(); const describeEmbeddedPostgres = embeddedPostgresSupport.supported ? describe : describe.skip; describeEmbeddedPostgres("merged pull-request confirmation sweep", () => { @@ -139,8 +141,8 @@ describeEmbeddedPostgres("merged pull-request confirmation sweep", () => { let tempDb: Awaited> | null = null; beforeAll(async () => { - tempDb = await startEmbeddedPostgresTestDatabase("paperclip-merged-pr-confirmations-"); - db = createDb(tempDb.connectionString); + if (!external) tempDb = await startEmbeddedPostgresTestDatabase("paperclip-merged-pr-confirmations-"); + db = createDb(external ?? tempDb!.connectionString, { maxConnections: 6 }); }, 20_000); afterEach(async () => { @@ -153,6 +155,7 @@ describeEmbeddedPostgres("merged pull-request confirmation sweep", () => { }); afterAll(async () => { + await db?.$client.end({ timeout: 1 }); await tempDb?.cleanup(); }); @@ -276,7 +279,7 @@ describeEmbeddedPostgres("merged pull-request confirmation sweep", () => { repo: "paperclip", number: 39, }])).resolves.toEqual({ - checked: 6, + checked: 3, candidates: 3, accepted: 2, woken: 2, diff --git a/server/src/__tests__/merged-pr-confirmation-work.test.ts b/server/src/__tests__/merged-pr-confirmation-work.test.ts new file mode 100644 index 00000000000..51195e6157b --- /dev/null +++ b/server/src/__tests__/merged-pr-confirmation-work.test.ts @@ -0,0 +1,278 @@ +import { randomUUID } from "node:crypto"; +import { setTimeout as delay } from "node:timers/promises"; +import { eq, sql } from "drizzle-orm"; +import { activityLog, agents, chatActions, chatConversations, chatEndpoints, chatPublications, companies, createDb, issues, issueThreadInteractions, toolApplications, toolConnections, type Db } from "@paperclipai/db"; +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; +import { createDeliveryWorkCoordinator } from "../services/delivery-work-coordinator.js"; +import { issueThreadInteractionService } from "../services/issue-thread-interactions.js"; +import { issueService } from "../services/issues.js"; +import { registerMergeConfirmationWork, MERGE_CONFIRMATION_RECONCILE_MS } from "../services/merged-pr-confirmation-work.js"; +import { idleWorkSnapshot } from "../services/task-admission.js"; +import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; +import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js"; + +const external = resolveExternalTestDatabaseUrl("PAPERCLIP_MERGE_WORK_TEST_DATABASE_URL"); +const support = external ? { supported: true } : await getEmbeddedPostgresTestSupport(); +(support.supported ? describe : describe.skip)("merge confirmation work with PostgreSQL and simulated GitHub", () => { + let temporary: Awaited> | undefined; + let db: ReturnType; + let companyId: string, issueId: string, agentId: string; + let coordinator: ReturnType | undefined; + let draining = false, standby = false; + const errors: unknown[] = []; + const provider = vi.fn(async (_companyId: string, _reference: { number: number }): Promise<"open" | "merged" | "unknown"> => "merged"); + const wakeup = vi.fn(async () => ({ id: "simulated-durable-wake" })); + beforeAll(async () => { + if (!external) temporary = await startEmbeddedPostgresTestDatabase("merged-pr-work-"); + db = createDb(external ?? temporary!.connectionString, { maxConnections: 6, idleTimeoutSeconds: 0, maxLifetimeSeconds: 0 }); + }, 30_000); + beforeEach(async () => { + errors.length = 0; draining = false; standby = false; + provider.mockReset().mockResolvedValue("merged"); wakeup.mockClear(); + companyId = randomUUID(); agentId = randomUUID(); issueId = randomUUID(); + await db.insert(companies).values({ id: companyId, name: "Merge worker", issuePrefix: `M${companyId.slice(0, 6)}` }); + await db.insert(agents).values({ id: agentId, companyId, name: "Coder", role: "engineer", status: "idle", adapterType: "codex_local" }); + await db.insert(issues).values({ id: issueId, companyId, title: "Merge task", status: "todo", assigneeAgentId: agentId }); + await Promise.all(Array.from({ length: 6 }, () => db.execute(sql`select pg_sleep(0.01)`))); + vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"], shouldAdvanceTime: false }); + }); + afterEach(async () => { + await coordinator?.stop(); coordinator = undefined; + vi.useRealTimers(); vi.restoreAllMocks(); + await db.delete(chatActions); await db.delete(chatPublications); await db.delete(chatConversations); await db.delete(chatEndpoints); + await db.delete(toolConnections); await db.delete(toolApplications); + await db.delete(activityLog); await db.delete(issueThreadInteractions); await db.delete(issues); + await db.delete(agents); await db.delete(companies); + }); + afterAll(async () => { await db?.$client.end({ timeout: 1 }); await temporary?.cleanup(); }); + const service = (database: Db = db) => issueThreadInteractionService(database, { resolvePullRequestState: provider, wakeup }); + const input = (number = 39) => ({ kind: "request_confirmation" as const, title: `Merge paperclipai/paperclip#${number}?`, + resolverPolicy: "human_only" as const, continuationPolicy: "wake_assignee_on_accept" as const, + payload: { version: 1 as const, prompt: "Merge the PR?" } }); + const create = (database: Db = db, number = 39) => service(database).create({ id: issueId, companyId }, input(number), { agentId }); + const stored = (id: string) => db.select().from(issueThreadInteractions).where(eq(issueThreadInteractions.id, id)).then(rows => rows[0]); + async function start() { + coordinator = createDeliveryWorkCoordinator({ owner: db, canRun: () => !draining && !standby, + canReconcile: () => !standby, onError: error => errors.push(error) }); + const worker = registerMergeConfirmationWork(coordinator, service(), () => !draining && !standby); + await worker.ready; + return worker; + } + async function waitFor(check: () => Promise | void) { + for (let attempt = 0; attempt < 100; attempt++) { + await vi.advanceTimersByTimeAsync(1000); + await delay(10); + try { await check(); return; } catch (error) { if (attempt === 99) throw error; } + } + } + const settle = () => waitFor(() => expect(idleWorkSnapshot().active).toBe(0)); + async function quiet() { + await waitFor(() => { expect(idleWorkSnapshot().active).toBe(0); expect(coordinator!.nextWakeAt()).toBeNull(); }); + const reads = vi.spyOn(db, "select"), transactions = vi.spyOn(db, "transaction"), executes = vi.spyOn(db, "execute"); + const calls = provider.mock.calls.length; + await vi.advanceTimersByTimeAsync(3 * MERGE_CONFIRMATION_RECONCILE_MS); + expect(reads).not.toHaveBeenCalled(); expect(transactions).not.toHaveBeenCalled(); expect(executes).not.toHaveBeenCalled(); + expect(provider).toHaveBeenCalledTimes(calls); + reads.mockRestore(); transactions.mockRestore(); executes.mockRestore(); + } + + it("settles empty and configured idle, then ordinary creation accepts, audits, wakes once and becomes quiet with retained history", async () => { + await start(); await quiet(); expect(provider).not.toHaveBeenCalled(); + const card = await create(); + await waitFor(async () => expect((await stored(card.id)).status).toBe("accepted")); + await quiet(); + expect(wakeup).toHaveBeenCalledTimes(1); + expect(wakeup).toHaveBeenCalledWith(agentId, expect.objectContaining({ idempotencyKey: `interaction:${card.id}:accepted`, allowRunCoalescing: false })); + expect(await db.select().from(activityLog)).toEqual(expect.arrayContaining([expect.objectContaining({ + action: "issue.thread_interaction_accepted", actorId: "system:pr-merged", details: expect.objectContaining({ interactionId: card.id }), + })])); + expect(errors).toEqual([]); + }); + + it("reconciles an open candidate at its deadline, never early, then stops after merge", async () => { + provider.mockResolvedValue("open"); await start(); const card = await create(); + await waitFor(() => expect(provider).toHaveBeenCalledTimes(1)); await settle(); + const dueAt = coordinator!.nextWakeAt()!; + expect(dueAt).toBeGreaterThan(Date.now()); + await vi.advanceTimersByTimeAsync(dueAt - Date.now() - 1); expect(provider).toHaveBeenCalledTimes(1); + provider.mockResolvedValue("merged"); await vi.advanceTimersByTimeAsync(1); + await waitFor(async () => expect((await stored(card.id)).status).toBe("accepted")); await quiet(); + expect(provider).toHaveBeenCalledTimes(2); expect(wakeup).toHaveBeenCalledTimes(1); + }); + + it("manual cancellation removes the candidate deadline without provider work", async () => { + provider.mockResolvedValue("open"); await start(); const card = await create(); + await waitFor(() => expect(provider).toHaveBeenCalledTimes(1)); await settle(); + await service().withdrawInteraction({ id: issueId, companyId }, card.id, {}, { agentId }); + await quiet(); expect(provider).toHaveBeenCalledTimes(1); expect(wakeup).not.toHaveBeenCalled(); + expect((await stored(card.id)).status).toBe("cancelled"); + }); + + it("does not accept before caller-owned outer commit and forgets a rolled-back create", async () => { + await start(); await quiet(); + let release!: () => void, created!: () => void; + const barrier = new Promise(resolve => { release = resolve; }); + const inserted = new Promise(resolve => { created = resolve; }); + let id = ""; + const transaction = db.transaction(async tx => { + id = (await create(tx as unknown as Db)).id; created(); await barrier; + }); + await inserted; + await vi.advanceTimersByTimeAsync(10_000); expect(provider).not.toHaveBeenCalled(); + expect(await stored(id)).toBeUndefined(); release(); await transaction; + await waitFor(async () => expect((await stored(id)).status).toBe("accepted")); await quiet(); + const prior = provider.mock.calls.length; + await expect(db.transaction(async tx => { await create(tx as unknown as Db, 41); throw new Error("rollback"); })).rejects.toThrow("rollback"); + await quiet(); expect(provider).toHaveBeenCalledTimes(prior); expect(errors).toEqual([]); + }); + + it("retains provider failures across drain, resumes without overlap, and clears retry after success", async () => { + provider.mockRejectedValue(new Error("GitHub unavailable")); await start(); const card = await create(); + await waitFor(() => expect(errors).toHaveLength(1)); await settle(); + expect((await stored(card.id)).status).toBe("pending"); + draining = true; + const reads = vi.spyOn(db, "select"), calls = provider.mock.calls.length; + await vi.advanceTimersByTimeAsync(120_000); + expect(reads).not.toHaveBeenCalled(); expect(provider).toHaveBeenCalledTimes(calls); reads.mockRestore(); + draining = false; provider.mockResolvedValue("merged"); + await waitFor(async () => expect((await stored(card.id)).status).toBe("accepted")); await quiet(); + expect(wakeup).toHaveBeenCalledTimes(1); + }); + + it("keeps first work committed during drain and startup during standby until resume", async () => { + standby = true; await start(); const card = await create(); + await vi.advanceTimersByTimeAsync(60_000); expect(provider).not.toHaveBeenCalled(); + standby = false; draining = true; + await vi.advanceTimersByTimeAsync(60_000); expect(provider).not.toHaveBeenCalled(); + draining = false; + await waitFor(async () => expect((await stored(card.id)).status).toBe("accepted")); await quiet(); + }); + + it("reconstructs pending cards on restart and ignores terminal or hidden tasks", async () => { + provider.mockResolvedValue("open"); const card = await create(); await start(); + await waitFor(() => expect(provider).toHaveBeenCalledTimes(1)); await settle(); + await coordinator!.stop(); coordinator = undefined; provider.mockResolvedValue("merged"); + await start(); await waitFor(async () => expect((await stored(card.id)).status).toBe("accepted")); await quiet(); + const hidden = await db.transaction(async tx => { + const card = await create(tx as unknown as Db, 41); + await issueService(tx as unknown as Db).update(issueId, { hiddenAt: new Date() }); + return card; + }); + await quiet(); expect((await stored(hidden.id)).status).toBe("pending"); + await issueService(db).update(issueId, { hiddenAt: null }); + await waitFor(async () => expect((await stored(hidden.id)).status).toBe("accepted")); await quiet(); + }); + + it("preserves a real chat binding's durable continuation outbox on automatic acceptance", async () => { + const applicationId = randomUUID(), connectionId = randomUUID(), endpointId = randomUUID(); + await db.insert(toolApplications).values({ id: applicationId, companyId, applicationKey: `chat:slack:${endpointId}`, name: "Slack", type: "chat", status: "active" }); + await db.insert(toolConnections).values({ id: connectionId, companyId, applicationId, name: "Slack", uid: endpointId, + connectionPurpose: "channel", transport: "chat_sdk", status: "active", enabled: true }); + await db.insert(chatEndpoints).values({ id: endpointId, companyId, connectionId, provider: "slack", publicId: endpointId, + assignedAgentId: agentId, status: "active" }); + const [conversation] = await db.insert(chatConversations).values({ companyId, endpointId, issueId, + externalConversationId: "merge-channel", externalThreadId: "merge-thread", externalLabel: "Merge thread", state: "active" }).returning(); + await start(); await quiet(); const card = await create(); + await waitFor(async () => expect((await stored(card.id)).status).toBe("accepted")); await quiet(); + expect(await db.select().from(chatActions).where(eq(chatActions.kind, "interaction_wakeup"))).toEqual([ + expect.objectContaining({ companyId, endpointId, conversationId: conversation.id, providerActionId: `interaction_wakeup:${card.id}`, + payload: expect.objectContaining({ interactionId: card.id, issueId, agentId, interactionStatus: "accepted" }) }), + ]); + expect(await db.select().from(chatPublications)).toEqual(expect.arrayContaining([ + expect.objectContaining({ idempotencyKey: `interaction:${card.id}:${endpointId}`, state: "cancelled" }), + ])); + expect(errors).toEqual([]); + }); + + it("does not lose a newer card created during provider observation or wake the superseded card", async () => { + let release!: () => void, observed!: () => void; + const barrier = new Promise(resolve => { release = resolve; }); + const observing = new Promise(resolve => { observed = resolve; }); + provider.mockImplementationOnce(async () => { observed(); await barrier; return "merged"; }); + await start(); const old = await create(); + await vi.advanceTimersByTimeAsync(1000); await observing; + const current = await create(db, 41); + expect((await stored(old.id)).status).toBe("expired"); + release(); + await waitFor(async () => expect((await stored(current.id)).status).toBe("accepted")); await quiet(); + expect(wakeup).toHaveBeenCalledTimes(1); + expect(wakeup).toHaveBeenCalledWith(agentId, expect.objectContaining({ idempotencyKey: `interaction:${current.id}:accepted` })); + expect(errors).toEqual([]); + }); + + it("keeps unknown provider results as retryable debt and does not act before retry", async () => { + provider.mockResolvedValue("unknown"); await start(); const card = await create(); + await waitFor(() => expect(errors).toHaveLength(1)); + const next = coordinator!.nextWakeAt()!; + await vi.advanceTimersByTimeAsync(next - Date.now() - 1); expect(provider).toHaveBeenCalledTimes(1); + provider.mockResolvedValue("merged"); await vi.advanceTimersByTimeAsync(1); + await waitFor(async () => expect((await stored(card.id)).status).toBe("accepted")); await quiet(); + expect(provider).toHaveBeenCalledTimes(2); + }); + + it("honors terminal issues, pure merge URL targets and continuationPolicy none", async () => { + await start(); await quiet(); + const target = await service().create({ id: issueId, companyId }, { ...input(), title: "Merge the linked PR?", continuationPolicy: "none", + payload: { version: 1, prompt: "Merge the PR?", target: { type: "custom", key: "github-pr-39", href: "https://github.com/paperclipai/paperclip/pull/39" } } }, { agentId }); + await waitFor(async () => expect((await stored(target.id)).status).toBe("accepted")); await quiet(); + expect(wakeup).not.toHaveBeenCalled(); + provider.mockClear(); + const terminal = await db.transaction(async tx => { + const card = await create(tx as unknown as Db, 41); + await issueService(tx as unknown as Db).update(issueId, { status: "cancelled" }); + return card; + }); + await quiet(); expect((await stored(terminal.id)).status).toBe("expired"); + expect(provider).not.toHaveBeenCalled(); expect(wakeup).not.toHaveBeenCalled(); + }); + + it("rechecks hidden state under the acceptance lock when visibility changes during a provider read", async () => { + let release!: () => void, observed!: () => void; + const barrier = new Promise(resolve => { release = resolve; }); + const observing = new Promise(resolve => { observed = resolve; }); + provider.mockImplementationOnce(async () => { observed(); await barrier; return "merged"; }); + await start(); const card = await create(); await vi.advanceTimersByTimeAsync(1000); await observing; + await issueService(db).update(issueId, { hiddenAt: new Date() }); release(); await quiet(); + expect((await stored(card.id)).status).toBe("pending"); expect(wakeup).not.toHaveBeenCalled(); + await issueService(db).update(issueId, { hiddenAt: null }); + await waitFor(async () => expect((await stored(card.id)).status).toBe("accepted")); await quiet(); + expect(wakeup).toHaveBeenCalledTimes(1); expect(errors).toEqual([]); + }); + + it("unrelated pending confirmations do not retain worker SQL or provider work", async () => { + await start(); await quiet(); + await service().create({ id: issueId, companyId }, { ...input(), title: "Approve the plan?", payload: { version: 1, prompt: "Approve the plan?" } }, { agentId }); + await quiet(); expect(provider).not.toHaveBeenCalled(); expect(wakeup).not.toHaveBeenCalled(); + expect(errors).toEqual([]); + }); + + it("awaits an in-flight observation at shutdown and reconstructs its unaccepted card on restart", async () => { + let release!: () => void, observed!: () => void; + const barrier = new Promise(resolve => { release = resolve; }); + const observing = new Promise(resolve => { observed = resolve; }); + provider.mockImplementationOnce(async () => { observed(); await barrier; return "merged"; }); + await start(); const card = await create(); await vi.advanceTimersByTimeAsync(1000); await observing; + let stopped = false; + const stopping = coordinator!.stop().then(() => { stopped = true; }); + await delay(10); expect(stopped).toBe(false); release(); await stopping; coordinator = undefined; + expect((await stored(card.id)).status).toBe("pending"); expect(wakeup).not.toHaveBeenCalled(); + const reads = vi.spyOn(db, "select"); + await vi.advanceTimersByTimeAsync(2 * MERGE_CONFIRMATION_RECONCILE_MS); expect(reads).not.toHaveBeenCalled(); reads.mockRestore(); + await start(); await waitFor(async () => expect((await stored(card.id)).status).toBe("accepted")); await quiet(); + expect(wakeup).toHaveBeenCalledTimes(1); expect(errors).toEqual([]); + }); + + it("limits normalized merge hints to the owning company and matching PR and tolerates duplicates", async () => { + provider.mockResolvedValue("open"); const own = await create(); + const foreignCompany = randomUUID(), foreignIssue = randomUUID(); + await db.insert(companies).values({ id: foreignCompany, name: "Other company", issuePrefix: "OTH" }); + await db.insert(issues).values({ id: foreignIssue, companyId: foreignCompany, title: "Other", status: "todo" }); + const foreign = await service().create({ id: foreignIssue, companyId: foreignCompany }, input(), { userId: "board" }); + const local = service(); + const hint = [{ companyId, owner: "PAPERCLIPAI", repo: "paperclip", number: 39 }]; + expect((await local.sweepMergedPullRequestConfirmations(hint)).accepted).toBe(1); + expect((await stored(own.id)).status).toBe("accepted"); expect((await stored(foreign.id)).status).toBe("pending"); + expect(provider).not.toHaveBeenCalled(); + expect((await local.sweepMergedPullRequestConfirmations(hint)).accepted).toBe(0); expect(wakeup).toHaveBeenCalledTimes(1); + }); +}); diff --git a/server/src/__tests__/server-startup-feedback-export.test.ts b/server/src/__tests__/server-startup-feedback-export.test.ts index b645d7437c8..9fc7234b2e6 100644 --- a/server/src/__tests__/server-startup-feedback-export.test.ts +++ b/server/src/__tests__/server-startup-feedback-export.test.ts @@ -180,6 +180,7 @@ const { const heartbeatServiceFactoryMock = vi.fn(() => heartbeatServiceMock); const issueThreadInteractionServiceMock = { sweepSupersededPendingRequestConfirmations: vi.fn(async () => ({ expired: 0 })), + mergedPullRequestConfirmationCandidates: vi.fn(async () => []), sweepMergedPullRequestConfirmations: vi.fn(async () => ({ checked: 0, candidates: 0, @@ -1334,7 +1335,9 @@ describe("startServer feedback export wiring", () => { expect(heartbeatServiceMock.tickAgentTimers).toHaveBeenCalledTimes(1); expect(externalObjectsServiceMock.refreshDueObjectsForActiveCompanies).toHaveBeenCalledTimes(1); - expect(issueThreadInteractionServiceMock.sweepMergedPullRequestConfirmations).toHaveBeenCalledTimes(1); + expect(issueThreadInteractionServiceMock.mergedPullRequestConfirmationCandidates).toHaveBeenCalledTimes(1); + expect(issueThreadInteractionServiceMock.sweepMergedPullRequestConfirmations).not.toHaveBeenCalled(); + expect(deliveryWorkRegisterMock.mock.calls.some(([queue]) => queue === DELIVERY_QUEUES.mergedPullRequestConfirmations)).toBe(true); expect(executionWorkspaceServiceMock.sweepTerminalWorkspaces).toHaveBeenCalledTimes(1); expect(routineServiceMock.tickScheduledTriggers).toHaveBeenCalledTimes(1); const routineWorkerIndex = deliveryWorkRegisterMock.mock.calls.findIndex(([queue]) => queue === DELIVERY_QUEUES.routineSchedules); diff --git a/server/src/index.ts b/server/src/index.ts index ae229f03518..c1804998ef5 100644 --- a/server/src/index.ts +++ b/server/src/index.ts @@ -1,3 +1,4 @@ +import { registerMergeConfirmationWork } from "./services/merged-pr-confirmation-work.js"; import { registerProviderTraceRetentionWork } from "./services/provider-trace-retention-work.js"; import { registerRunEvidenceMigrationWork } from "./services/run-evidence-migration-work.js"; import { registerSkillVersionRetentionWork } from "./services/skill-version-retention-work.js"; @@ -1600,19 +1601,8 @@ async function startServerWithDatabaseTeardown( : config.workspaceCompactionDelayHours * 60 * 60 * 1000, terminalWorkspaceInspectionBackoffMs: TERMINAL_WORKSPACE_RETRY_MS, }); - const scheduleMergedPullRequestConfirmationSweep = () => { - if (isIdleTaskDrainActive() || heartbeatSchedulerStopped) return; - trackHeartbeatSchedulerWork(mergedPullRequestConfirmations - .sweepMergedPullRequestConfirmations() - .then((result) => { - if (result.accepted > 0) { - logger.info(result, "accepted merge confirmations for merged pull requests"); - } - }) - .catch((err) => { - logger.error({ err }, "merged pull-request confirmation sweep failed"); - })); - }; + await registerMergeConfirmationWork(deliveryWork, mergedPullRequestConfirmations, + () => !isIdleTaskDrainActive() && !heartbeatSchedulerStopped && !isWarmStandby()).ready; const gitRestorePointWorker = registerGitRestorePointWork(deliveryWork, gitRestorePointRetentionService(db as any, { storage: storageService, retentionDays: config.gitRestorePointRetentionDays, @@ -2005,7 +1995,6 @@ async function startServerWithDatabaseTeardown( scheduleExternalObjectRefreshSweep(new Date()); if (isIdleTaskDrainActive() || heartbeatSchedulerStopped) return; - scheduleMergedPullRequestConfirmationSweep(); scheduleGitHubConnectionEventPoll(); scheduleGitHubConnectionContinuitySweep(); diff --git a/server/src/services/chat-confirmation-notifications.ts b/server/src/services/chat-confirmation-notifications.ts index e55d8c0d7ac..02cdfbee8c8 100644 --- a/server/src/services/chat-confirmation-notifications.ts +++ b/server/src/services/chat-confirmation-notifications.ts @@ -1,3 +1,4 @@ +import { notifyMergeConfirmationWork } from "./merged-pr-confirmation-notifications.js"; import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js"; export type ChatConfirmationScope = { companyId: string; issueId?: string }; @@ -10,6 +11,7 @@ export async function notifyChatConfirmationWork(tx: object, scope: ChatConfirma /** Interaction resolution can release a callback parked while its human input * was pending. Register on the owning transaction before its resolution write. */ export async function notifyChatConfirmationResolution(tx: object, interaction: ChatConfirmationScope & { kind: string }) { + if (interaction.kind === "request_confirmation") await notifyMergeConfirmationWork(tx, interaction); if (["request_confirmation", "request_checkbox_confirmation", "suggest_tasks", "connection_intent"].includes(interaction.kind)) { await notifyChatConfirmationWork(tx, interaction); } diff --git a/server/src/services/delivery-work-notifications.ts b/server/src/services/delivery-work-notifications.ts index 9195f1fcb12..308a721cbc5 100644 --- a/server/src/services/delivery-work-notifications.ts +++ b/server/src/services/delivery-work-notifications.ts @@ -97,6 +97,7 @@ export const DELIVERY_QUEUES = { browser: "browser-use", email: "email-channels", museMaintenance: "muse-maintenance", + mergedPullRequestConfirmations: "merged-pull-request-confirmations", mcpEvents: "mcp-events", dotEvents: "dot-events", grokDispatch: "grok-dispatch", diff --git a/server/src/services/issue-thread-interactions.ts b/server/src/services/issue-thread-interactions.ts index 59c14618342..b306f637e27 100644 --- a/server/src/services/issue-thread-interactions.ts +++ b/server/src/services/issue-thread-interactions.ts @@ -1,3 +1,4 @@ +import { notifyMergeConfirmationWork, type MergeConfirmationScope } from "./merged-pr-confirmation-notifications.js"; import { notifyNativeStatusReassessment } from "./native-status-reassessment-notifications.js"; import { notifyIssueReviewState } from "./issue-review-state-notifications.js"; import { notifyChatConfirmationResolution } from "./chat-confirmation-notifications.js"; @@ -501,6 +502,7 @@ type IssueResolutionContext = { companyId: string; status: string; executionRunId: string | null; + hiddenAt?: Date | null; workMode: string; assigneeAgentId: string | null; assigneeUserId: string | null; @@ -2074,6 +2076,32 @@ export function issueThreadInteractionService( ); return states; } + async function mergedPullRequestConfirmationCandidates(scope?: MergeConfirmationScope) { + const text = sql`concat_ws(' ', ${issueThreadInteractions.title}, ${issueThreadInteractions.summary}, + ${issueThreadInteractions.payload}->>'prompt', ${issueThreadInteractions.payload}->>'acceptLabel', + ${issueThreadInteractions.payload}->>'detailsMarkdown', ${issueThreadInteractions.payload}->'target'->>'key', + ${issueThreadInteractions.payload}->'target'->>'label', ${issueThreadInteractions.payload}->'target'->>'href')`; + const rows = await db.select({ interaction: issueThreadInteractions, issue: { + id: issues.id, companyId: issues.companyId, projectId: issues.projectId, goalId: issues.goalId, + status: issues.status, assigneeAgentId: issues.assigneeAgentId, + } }).from(issueThreadInteractions).innerJoin(issues, and( + eq(issueThreadInteractions.issueId, issues.id), eq(issueThreadInteractions.companyId, issues.companyId), + )).where(and( + eq(issueThreadInteractions.kind, "request_confirmation"), eq(issueThreadInteractions.status, "pending"), + isNull(issues.hiddenAt), sql`${issues.status} NOT IN ('done', 'cancelled')`, + scope ? eq(issueThreadInteractions.companyId, scope.companyId) : undefined, + scope?.issueId ? eq(issueThreadInteractions.issueId, scope.issueId) : undefined, + scope?.interactionId ? eq(issueThreadInteractions.id, scope.interactionId) : undefined, + sql`${text} ~* '(merge|merging)'`, + sql`${text} ~* '(github\\.com/[^ /]+/[^ /]+/pull/[1-9][0-9]*|[a-z0-9_.-]+/[a-z0-9_.-]+#[1-9][0-9]*)'`, + sql`NOT EXISTS (SELECT 1 FROM ${toolActionRequests} WHERE ${toolActionRequests.interactionId} = ${issueThreadInteractions.id})`, + )); + return rows.flatMap(row => { + const references = getMergeConfirmationPullRequestReferences(row.interaction); + return references.length ? [{ ...row, references }] : []; + }); + } + async function getIdempotentInteraction(args: { issueId: string; companyId: string; @@ -2271,6 +2299,7 @@ export function issueThreadInteractionService( companyId: issues.companyId, status: issues.status, executionRunId: issues.executionRunId, + hiddenAt: issues.hiddenAt, workMode: issues.workMode, assigneeAgentId: issues.assigneeAgentId, assigneeUserId: issues.assigneeUserId, @@ -2309,6 +2338,10 @@ export function issueThreadInteractionService( "Interaction has already been resolved", ); } + if (args.actor.systemId === "system:pr-merged" && + (issueContext.hiddenAt || getMergeConfirmationPullRequestReferences(lockedCurrent).length === 0)) { + throw conflict("Merge confirmation is no longer actionable"); + } await assertRequestConfirmationResolutionAllowedUnderLock( tx as unknown as Db, issueContext, @@ -2979,6 +3012,7 @@ export function issueThreadInteractionService( await emitInteractionResolvedTelemetry(db, interaction); return interaction; }, + mergedPullRequestConfirmationCandidates, sweepMergedPullRequestConfirmations: async ( mergedHints: Array<{ companyId: string; @@ -2986,51 +3020,17 @@ export function issueThreadInteractionService( repo: string; number: number; }> = [], + scope?: MergeConfirmationScope, + signal?: AbortSignal, ) => { - const rows = await db - .select({ - interaction: issueThreadInteractions, - issue: { - id: issues.id, - companyId: issues.companyId, - projectId: issues.projectId, - goalId: issues.goalId, - status: issues.status, - assigneeAgentId: issues.assigneeAgentId, - }, - }) - .from(issueThreadInteractions) - .innerJoin(issues, eq(issueThreadInteractions.issueId, issues.id)) - .where( - and( - eq(issueThreadInteractions.kind, "request_confirmation"), - eq(issueThreadInteractions.status, "pending"), - ), - ); - - const candidates = rows.flatMap((row) => { - const references = getMergeConfirmationPullRequestReferences( - row.interaction, - ); - return references.length > 0 ? [{ ...row, references }] : []; - }); - if (candidates.length === 0) { - return { checked: rows.length, candidates: 0, accepted: 0, woken: 0 }; - } - - const candidateIds = candidates.map(({ interaction }) => interaction.id); - const linkedToolActions = await db - .select({ interactionId: toolActionRequests.interactionId }) - .from(toolActionRequests) - .where(inArray(toolActionRequests.interactionId, candidateIds)); - const toolActionInteractionIds = new Set( - linkedToolActions - .map((row) => row.interactionId) - .filter((value): value is string => Boolean(value)), - ); - const eligible = candidates.filter( - ({ interaction }) => !toolActionInteractionIds.has(interaction.id), - ); + const candidates = await mergedPullRequestConfirmationCandidates(scope ?? + (mergedHints.length === 1 ? { companyId: mergedHints[0]!.companyId } : undefined)); + const eligible = mergedHints.length ? candidates.filter(candidate => mergedHints.some(hint => + hint.companyId === candidate.issue.companyId && candidate.references.some(reference => + hint.owner.toLowerCase() === reference.owner.toLowerCase() && + hint.repo.toLowerCase() === reference.repo.toLowerCase() && hint.number === reference.number), + )) : candidates; + if (eligible.length === 0) return { checked: 0, candidates: 0, accepted: 0, woken: 0 }; const checkedAt = now().getTime(); const cacheTtlMs = opts.pullRequestCacheTtlMs ?? 5 * 60 * 1000; @@ -3065,6 +3065,9 @@ export function issueThreadInteractionService( const refreshedStates = await resolvePullRequestStates([ ...uniqueReferences.values(), ]); + if (scope && [...refreshedStates.values()].some(state => state === "unknown")) { + throw new Error("Merge confirmation provider state is unavailable"); + } for (const [key, state] of refreshedStates) { setBoundedPullRequestCacheEntry(pullRequestStateCache, key, { state, @@ -3075,6 +3078,7 @@ export function issueThreadInteractionService( let accepted = 0; let woken = 0; for (const candidate of eligible) { + if (signal?.aborted) break; const allMerged = candidate.references.every((reference) => { const key = `${candidate.issue.companyId}:${reference.owner.toLowerCase()}/${reference.repo.toLowerCase()}#${reference.number}`; return pullRequestStateCache.get(key)?.state === "merged"; @@ -3111,12 +3115,16 @@ export function issueThreadInteractionService( if (resolved.interaction.status !== "accepted") continue; accepted += 1; - const wakeIssue = resolved.continuationIssue ?? candidate.issue; + const wakeIssue = resolved.continuationIssue ?? await db.select({ + id: issues.id, assigneeAgentId: issues.assigneeAgentId, status: issues.status, + }).from(issues).where(and(eq(issues.id, candidate.issue.id), + eq(issues.companyId, candidate.issue.companyId), isNull(issues.hiddenAt))).then(rows => rows[0]); const shouldWake = resolved.interaction.continuationPolicy === "wake_assignee" || resolved.interaction.continuationPolicy === "wake_assignee_on_accept"; if ( !opts.wakeup || + !wakeIssue || !shouldWake || !wakeIssue.assigneeAgentId || isTerminalIssueStatus(wakeIssue.status) @@ -3158,7 +3166,7 @@ export function issueThreadInteractionService( } return { - checked: rows.length, + checked: candidates.length, candidates: eligible.length, accepted, woken, @@ -3909,6 +3917,7 @@ export function issueThreadInteractionService( } else { await notifyNativeStatusReassessment(tx, { companyId: issue.companyId, issueId: issue.id }); } + if (data.kind === "request_confirmation") await notifyMergeConfirmationWork(tx, { companyId: issue.companyId, issueId: issue.id }); const [row] = await tx .insert(issueThreadInteractions) .values({ diff --git a/server/src/services/issues.ts b/server/src/services/issues.ts index 42c65bb5d02..4178dc1a35c 100644 --- a/server/src/services/issues.ts +++ b/server/src/services/issues.ts @@ -1,3 +1,4 @@ +import { notifyMergeConfirmationWork } from "./merged-pr-confirmation-notifications.js"; import { notifyExternalConversationSettlement } from "./external-conversation-settlement-notifications.js"; import { notifyNativeRecoveryNotices } from "./native-recovery-notice-notifications.js"; import { notifyNativeRunProjection } from "./native-run-projection-notifications.js"; @@ -11491,6 +11492,9 @@ export function issueService(db: Db) { || issueData.parentId !== undefined || issueData.executionWorkspaceId !== undefined) { await notifyTerminalWorkspaceWork(tx, `issue:${id}`); } + if (patch.status !== undefined || patch.hiddenAt !== undefined) { + await notifyMergeConfirmationWork(tx, { companyId: receiptExisting.companyId, issueId: id }); + } if (patch.status !== undefined || patch.assigneeAgentId !== undefined || patch.assigneeUserId !== undefined || patch.hiddenAt !== undefined || patch.conversationAgentId !== undefined) { await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers); diff --git a/server/src/services/merged-pr-confirmation-notifications.ts b/server/src/services/merged-pr-confirmation-notifications.ts new file mode 100644 index 00000000000..34df4ecffb0 --- /dev/null +++ b/server/src/services/merged-pr-confirmation-notifications.ts @@ -0,0 +1,7 @@ +import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js"; + +export type MergeConfirmationScope = { companyId: string; issueId?: string; interactionId?: string }; + +export function notifyMergeConfirmationWork(tx: object, scope: MergeConfirmationScope) { + return notifyDeliveryWork(tx, DELIVERY_QUEUES.mergedPullRequestConfirmations, JSON.stringify({ companyId: scope.companyId, issueId: scope.issueId, interactionId: scope.interactionId })); +} diff --git a/server/src/services/merged-pr-confirmation-work.ts b/server/src/services/merged-pr-confirmation-work.ts new file mode 100644 index 00000000000..bfccaeea79f --- /dev/null +++ b/server/src/services/merged-pr-confirmation-work.ts @@ -0,0 +1,30 @@ +import type { createDeliveryWorkCoordinator } from "./delivery-work-coordinator.js"; +import { DELIVERY_QUEUES } from "./delivery-work-notifications.js"; +import type { issueThreadInteractionService } from "./issue-thread-interactions.js"; +import type { MergeConfirmationScope } from "./merged-pr-confirmation-notifications.js"; +import { registerScopedDeadlineWork } from "./scoped-deadline-work.js"; + +export const MERGE_CONFIRMATION_RECONCILE_MS = 5 * 60_000; + +export function registerMergeConfirmationWork( + coordinator: ReturnType, + service: Pick, + "mergedPullRequestConfirmationCandidates" | "sweepMergedPullRequestConfirmations">, + canRun: () => boolean, +) { + return registerScopedDeadlineWork(coordinator, { + queue: DELIVERY_QUEUES.mergedPullRequestConfirmations, + decodeScope: hint => hint === "*" ? undefined : JSON.parse(hint) as MergeConfirmationScope, + matches: (entry, scope?: MergeConfirmationScope) => !scope || (entry.companyId === scope.companyId && + (!scope.issueId || entry.issueId === scope.issueId) && (!scope.interactionId || entry.id === scope.interactionId)), + load: async (scope?: MergeConfirmationScope) => (await service.mergedPullRequestConfirmationCandidates(scope)) + .map(({ interaction }) => ({ id: interaction.id, companyId: interaction.companyId, issueId: interaction.issueId, dueAt: Date.now() })), + check: async (entry, signal) => { + const scope = { companyId: entry.companyId, issueId: entry.issueId, interactionId: entry.id }; + await service.sweepMergedPullRequestConfirmations([], scope, signal); + if (signal?.aborted) return entry; + const remaining = await service.mergedPullRequestConfirmationCandidates(scope); + return remaining.length ? { ...entry, dueAt: Date.now() + MERGE_CONFIRMATION_RECONCILE_MS } : null; + }, + }, canRun); +} From 50cd007005c7971612d6543da30aee0b3bf94d2b Mon Sep 17 00:00:00 2001 From: Dotta Date: Fri, 30 Oct 2026 16:24:46 -0500 Subject: [PATCH 06/74] perf(external-objects): schedule scoped refresh obligations Co-Authored-By: Paperclip --- .../2026-10-30-external-objects-sleep.md | 92 +++++ .../external-object-refresh-work.test.ts | 334 ++++++++++++++++++ .../external-objects-service.test.ts | 9 +- .../github-connection-events.test.ts | 20 +- server/src/index.ts | 17 +- server/src/services/companies.ts | 3 + server/src/services/company-deletion.ts | 2 + .../services/delivery-work-notifications.ts | 1 + .../external-object-refresh-notifications.ts | 7 + server/src/services/external-objects.ts | 183 +++++++--- .../src/services/github-connection-events.ts | 44 ++- server/src/services/instance-settings.ts | 2 + server/src/services/issues.ts | 2 + server/src/services/plugin-registry.ts | 43 ++- 14 files changed, 659 insertions(+), 100 deletions(-) create mode 100644 doc/plans/2026-10-30-external-objects-sleep.md create mode 100644 server/src/__tests__/external-object-refresh-work.test.ts create mode 100644 server/src/services/external-object-refresh-notifications.ts diff --git a/doc/plans/2026-10-30-external-objects-sleep.md b/doc/plans/2026-10-30-external-objects-sleep.md new file mode 100644 index 00000000000..639bbaea343 --- /dev/null +++ b/doc/plans/2026-10-30-external-objects-sleep.md @@ -0,0 +1,92 @@ +# External object refresh deadlines + +External-object refresh uses the delivery coordinator. The server registers the +service in both heartbeat-scheduler configurations. It no longer calls the +company-wide aggregate sweep on each heartbeat tick. + +`external-objects.ts` owns automatic eligibility, resolver selection, refresh +claims, provider results and the next deadline. `index.ts` supplies registration +and admission gates. The existing scoped deadline helper owns wake coalescing, +page reconciliation, fair company selection, retry backoff and shutdown. + +## Durable obligations and quiet states + +An automatic refresh needs an active company, a retained mention in that company, +a non-null `next_refresh_at` and an available resolver. Injected resolvers, +including custom `url/link` resolvers, remain supported. A plugin resolver needs a +ready plugin with `external.objects.read`, a matching provider/object declaration, +and no company setting that disables it. Built-in resolvers remain available +independently of optional plugin support. + +Terminal objects are passive. The existing exception remains: GitHub +`not_found` snapshots without `data.probeVerified=true` need a repository-visibility +probe before their terminal verdict is definitive. A null deadline retains its +existing never-refresh meaning. Registration does not change stored timestamps. + +The worker reconstructs eligible deadlines in pages of at most 250 rows. A +company notification reads that company only. A due callback reads only its +object. Pages select scheduling columns and retain compact deadlines in memory. +Dispatch uses fair batches of at most 50 objects. The deadline is the later of +`next_refresh_at` and the current refresh lease expiry. Lease renewals and atomic +refresh tokens continue to exclude another claimant. + +Unsupported generic links, orphan objects, disabled plugins, empty configuration +and retained terminal history do not schedule fallback discovery. Manual refresh +retains its existing `no_resolver` result. Actual provider failures retain their +saved retry deadline. Unexpected failures retain scoped coordinator backoff; +restart reconstructs the saved object and any unexpired claim. + +The supported-resolver and retained-reference approach follows the related draft +[PR #17298](https://github.com/paperclipai/paperclip/pull/17298) by +[@kaluli123123](https://github.com/kaluli123123). This change adapts that eligibility +pattern to scoped scheduling, adds dynamic plugin/company permission predicates, +and fences automatic result persistence. It does not modify that draft PR. + +## Producer and transaction audit + +| Mutation | Owner and activation | +| --- | --- | +| Issue title/description, comment and document references | Existing sync methods register company intent before mention replacement and object upsert. Routes, runtime tools, adapters and plugin paths reuse these methods. | +| Comment/document disappearance | Sync registers reconstruction before deleting obsolete mentions. | +| Last mention removal and reattachment | Mention replacement invalidates company deadlines. Retained objects stay available for reattachment. | +| Issue deletion | Issue service registers its company before cascading mention deletion. | +| Company status/archive/deletion | Company services register company reconstruction before their status or deletion writes. | +| Feature enable/disable | Instance settings registers global reconstruction before the experimental-settings write. | +| Plugin manifest/status/uninstall | Plugin registry registers reconstruction before lifecycle writes. Installed plugins acquire support through the ordinary ready transition. | +| Plugin company configuration and enabled setting | Registry writes register only the affected company. There is no plugin-readiness poll. | +| Manual and automatic resolver result | Result persistence registers object intent in the same transaction. Automatic persistence rechecks reference and resolver eligibility. Lost support releases the worker's own claim without publishing the stale result. | +| Normalized GitHub ingress | The event service registers company intent with the snapshot write. It clears the old refresh token, so an older provider response cannot overwrite that event. | +| Import and restore | No independent object-table importer exists in the source baseline. Supported reference ingestion uses the ordinary sync boundary. Startup reconstructs restored durable objects. | + +Notifications are awaited before writes. Caller-owned and nested transactions +retain the outer commit boundary through the existing database-work protocol. +Rollback does not publish the write. Notifications received during a scan or an +active resolver remain dirty until a successful reconciliation. + +The supported live writer model is one shared `createDb` signal owner, including +its transactions and child pools. These process-local signals do not cover +independent SQL clients or other replicas. Such writers must invoke the supported +service boundary or arrange explicit reconstruction/restart. Existing normalized +GitHub ingress uses the supported boundary. No permanent database listener or +periodic global safety scan is added. + +## Acceptance evidence + +The focused suite uses real PostgreSQL with a controlled simulated provider. +Ordinary sync and registry/settings writes create work; the fixture does not +pre-create the tested resolved outcome. Assertions independently read saved +objects. Tests measure owner reads and provider calls across former tick windows. + +Coverage includes empty and terminal quiet, unsupported links, orphan removal and +reattachment, feature/company eligibility, plugin readiness and permission loss +during a held provider call, exact and earlier deadlines, durable provider retry, +rollback and outer commit after nested savepoint release, startup and paginated +reconstruction, inactive companies, never-refresh rows, first work during drain, +warm standby, failed refresh after restart, lease renewal, duplicate exclusion, +and shutdown while a provider call remains active. The GitHub ingress suite also +checks a real normalized event superseding a held resolver response. + +The conservative external-object host-sleep blocker remains. An in-process +deadline cannot wake stopped compute. This change proves worker quiet and +reactivation in the tested process; it does not establish Cloud alarm coverage, +host/database suspension or deployed savings. diff --git a/server/src/__tests__/external-object-refresh-work.test.ts b/server/src/__tests__/external-object-refresh-work.test.ts new file mode 100644 index 00000000000..8d1f5cdc00c --- /dev/null +++ b/server/src/__tests__/external-object-refresh-work.test.ts @@ -0,0 +1,334 @@ +import { randomUUID } from "node:crypto"; +import { setTimeout as delay } from "node:timers/promises"; +import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest"; +import { eq, sql } from "drizzle-orm"; +import { activityLog, companies, createDb, externalObjects, issues, plugins } from "@paperclipai/db"; +import { externalObjectService, type ExternalObjectResolveResult } from "../services/external-objects.js"; +import type { PaperclipPluginManifestV1 } from "@paperclipai/shared"; +import type { PluginWorkerManager } from "../services/plugin-worker-manager.js"; +import { pluginRegistryService } from "../services/plugin-registry.js"; +import { companyService } from "../services/companies.js"; +import { instanceSettingsService } from "../services/instance-settings.js"; +import { notifyExternalObjectRefresh } from "../services/external-object-refresh-notifications.js"; +import { createDeliveryWorkCoordinator } from "../services/delivery-work-coordinator.js"; +import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js"; +import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; + +const url = resolveExternalTestDatabaseUrl("PAPERCLIP_EXTERNAL_OBJECT_REFRESH_TEST_DATABASE_URL"); +const support = url ? { supported: true } : await getEmbeddedPostgresTestSupport(); +const terminal: ExternalObjectResolveResult = { ok: true, snapshot: { + statusKey: "done", statusCategory: "succeeded", statusTone: "success", isTerminal: true, +} }; +const active: ExternalObjectResolveResult = { ok: true, snapshot: { + statusKey: "open", statusCategory: "open", statusTone: "neutral", isTerminal: false, ttlSeconds: 60, +} }; + +(support.supported ? describe : describe.skip)("external object refresh work (real PostgreSQL, simulated provider)", () => { + let db: ReturnType; + let temp: Awaited> | undefined; + let coordinator: ReturnType | undefined; + let admitted = true; + let reconcile = true; + let enabled = true; + const provider = vi.fn(async (): Promise => terminal); + const errors = vi.fn(); + const companyIds: string[] = []; + beforeAll(async () => { + if (!url) temp = await startEmbeddedPostgresTestDatabase("external-refresh-work-"); + db = createDb(url ?? temp!.connectionString, { maxConnections: 6 }); + await db.execute(sql`truncate table ${companies} cascade`); + }); + afterEach(async () => { + await coordinator?.stop(); coordinator = undefined; + vi.useRealTimers(); vi.restoreAllMocks(); + for (const id of companyIds.splice(0)) { + await db.delete(activityLog).where(eq(activityLog.companyId, id)); + await db.delete(issues).where(eq(issues.companyId, id)); + await db.delete(externalObjects).where(eq(externalObjects.companyId, id)); + await db.delete(companies).where(eq(companies.id, id)); + } + await db.delete(plugins); + admitted = true; reconcile = true; enabled = true; errors.mockClear(); + provider.mockReset().mockResolvedValue(terminal); + }); + afterAll(async () => { await db?.$client.end({ timeout: 1 }); await temp?.cleanup(); }); + + function service() { + return externalObjectService(db, { github: false, enabled: () => enabled, + resolvers: [{ providerKey: "url", resolve: provider }] }); + } + async function start(svc = service()) { + coordinator = createDeliveryWorkCoordinator({ owner: db, canRun: () => admitted, + canReconcile: () => reconcile, onError: errors }); + const worker = svc.registerRefreshWork(coordinator, () => admitted); + await worker.ready; + return worker; + } + async function fixture(status = "active") { + const id = randomUUID(); companyIds.push(id); + await db.insert(companies).values({ id, name: "Refresh test", status, + issuePrefix: `E${id.replaceAll("-", "").slice(0, 8)}` }); + const [issue] = await db.insert(issues).values({ companyId: id, title: "Reference", + description: `https://example.com/${id}`, status: "todo" }).returning(); + return { companyId: id, issueId: issue!.id }; + } + async function row(companyId: string) { + return (await db.select().from(externalObjects).where(eq(externalObjects.companyId, companyId)))[0]!; + } + async function settle(predicate: () => Promise | boolean) { + for (let i = 0; i < 200; i++) { + await vi.advanceTimersByTimeAsync(0); await delay(5); + if (await predicate()) return; + } + throw new Error("refresh worker did not settle"); + } + async function quiet() { await settle(() => coordinator!.nextWakeAt() === null); } + function clock() { vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout", "setInterval", "clearInterval"], shouldAdvanceTime: true }); } + async function change(companyId: string, patch: Partial) { + await db.transaction(async tx => { + await notifyExternalObjectRefresh(tx, companyId); + await tx.update(externalObjects).set(patch).where(eq(externalObjects.companyId, companyId)); + }); + } + + it("captures the old configured-empty aggregate SQL cost", async () => { + const svc = service(); await fixture(); + const selects = vi.spyOn(db, "select"); + for (let i = 0; i < 3; i++) await svc.refreshDueObjectsForActiveCompanies(); + expect(selects).toHaveBeenCalledTimes(6); + expect(provider).not.toHaveBeenCalled(); + }); + + it("activates saved work when the ordinary feature and company settings enable it", async () => { + clock(); const settings = instanceSettingsService(db); + await settings.updateExperimental({ enableExternalObjects: true }); + const svc = externalObjectService(db, { github: false, + enabled: async () => (await settings.getExperimental()).enableExternalObjects, + resolvers: [{ providerKey: "url", resolve: provider }] }); + const f = await fixture("paused"); await svc.syncIssue(f.issueId); + await settings.updateExperimental({ enableExternalObjects: false }); + await start(svc); await quiet(); + await companyService(db).update(f.companyId, { status: "active" }); await quiet(); + expect(provider).not.toHaveBeenCalled(); + await settings.updateExperimental({ enableExternalObjects: true }); + await settle(async () => (await row(f.companyId)).isTerminal); await quiet(); + expect(provider).toHaveBeenCalledOnce(); + }); + + it("settles empty, configured-idle and retained terminal history without recurring SQL or provider calls", async () => { + clock(); + const svc = service(); await start(svc); await quiet(); + const selects = vi.spyOn(db, "select"); + await vi.advanceTimersByTimeAsync(15 * 60_000); + expect(selects).not.toHaveBeenCalled(); expect(provider).not.toHaveBeenCalled(); + const f = await fixture(); await svc.syncIssue(f.issueId); + await settle(async () => (await row(f.companyId))?.isTerminal === true); await quiet(); + expect((await row(f.companyId)).statusKey).toBe("done"); + selects.mockClear(); provider.mockClear(); + await vi.advanceTimersByTimeAsync(15 * 60_000); + expect(selects).not.toHaveBeenCalled(); expect(provider).not.toHaveBeenCalled(); + expect(errors).not.toHaveBeenCalled(); + }); + + it("registers exact future deadlines, preempts them on earlier writes and cancels terminal work", async () => { + clock(); provider.mockResolvedValue(active); + const svc = service(); await start(svc); + const f = await fixture(); await svc.syncIssue(f.issueId); + await settle(() => provider.mock.calls.length === 1); + await settle(async () => (await row(f.companyId)).lastResolvedAt !== null); + const saved = await row(f.companyId); + await settle(() => coordinator!.nextWakeAt() === saved.nextRefreshAt!.getTime()); + const earlier = new Date(Date.now() + 1000); await change(f.companyId, { nextRefreshAt: earlier }); + await settle(() => coordinator!.nextWakeAt() === earlier.getTime()); + await vi.advanceTimersByTimeAsync(999); expect(provider).toHaveBeenCalledTimes(1); + provider.mockResolvedValue(terminal); await vi.advanceTimersByTimeAsync(1); + await settle(async () => (await row(f.companyId)).isTerminal); await quiet(); + await vi.advanceTimersByTimeAsync(120_000); expect(provider).toHaveBeenCalledTimes(2); + }); + + it("keeps disabled and paused company work passive and reconstructs on eligibility mutation", async () => { + clock(); enabled = false; + const svc = service(); await start(svc); await quiet(); + const f = await fixture("paused"); + enabled = true; await svc.syncIssue(f.issueId); await quiet(); + await vi.advanceTimersByTimeAsync(600_000); expect(provider).not.toHaveBeenCalled(); + await db.transaction(async tx => { + await notifyExternalObjectRefresh(tx, f.companyId); + await tx.update(companies).set({ status: "active" }).where(eq(companies.id, f.companyId)); + }); + await settle(async () => (await row(f.companyId)).isTerminal); await quiet(); + expect(provider).toHaveBeenCalledOnce(); + }); + + it("waits for the outer commit and leaves rolled-back ingestion quiet", async () => { + clock(); const svc = service(); await start(svc); await quiet(); + const f = await fixture(); + await expect(db.transaction(async tx => { + await svc.syncIssue(f.issueId, tx); throw new Error("rollback"); + })).rejects.toThrow("rollback"); + await quiet(); expect(await row(f.companyId)).toBeUndefined(); expect(provider).not.toHaveBeenCalled(); + let release!: () => void; const held = new Promise(resolve => { release = resolve; }); + let nestedDone = false; + const commit = db.transaction(async tx => { + await tx.transaction(async nested => { await svc.syncIssue(f.issueId, nested); }); + nestedDone = true; await held; + }); + await settle(() => nestedDone); await vi.advanceTimersByTimeAsync(60_000); + expect(provider).not.toHaveBeenCalled(); release(); await commit; + await settle(async () => (await row(f.companyId))?.isTerminal); await quiet(); + expect(provider).toHaveBeenCalledOnce(); + }); + + it("retains first work during drain and recovers a provider failure after restart", async () => { + clock(); const svc = service(); await start(svc); await quiet(); + admitted = false; const f = await fixture(); await svc.syncIssue(f.issueId); + const selects = vi.spyOn(db, "select"); + await vi.advanceTimersByTimeAsync(90_000); expect(provider).not.toHaveBeenCalled(); + selects.mockClear(); await vi.advanceTimersByTimeAsync(90_000); expect(selects).not.toHaveBeenCalled(); + admitted = true; provider.mockRejectedValueOnce(new Error("simulated transport failure")); + await vi.advanceTimersByTimeAsync(30_000); await settle(() => errors.mock.calls.length === 1); + const claimed = await row(f.companyId); expect(claimed.refreshToken).not.toBeNull(); + await coordinator!.stop(); coordinator = undefined; await start(svc); + const due = claimed.refreshStartedAt!.getTime() + 300_000; + expect(coordinator!.nextWakeAt()).toBe(due); + await vi.advanceTimersByTimeAsync(due - Date.now() - 1); expect(provider).toHaveBeenCalledTimes(1); + await vi.advanceTimersByTimeAsync(1); + await settle(async () => (await row(f.companyId)).isTerminal); await quiet(); + expect(provider).toHaveBeenCalledTimes(2); + }); + + it("leaves unsupported URLs passive and cancels orphan work until ordinary reattachment", async () => { + clock(); const unsupported = externalObjectService(db, { github: false }); + await start(unsupported); const f = await fixture(); await unsupported.syncIssue(f.issueId); await quiet(); + const selects = vi.spyOn(db, "select"); + await vi.advanceTimersByTimeAsync(900_000); expect(selects).not.toHaveBeenCalled(); + expect((await row(f.companyId)).lastResolvedAt).toBeNull(); + await coordinator!.stop(); coordinator = undefined; + provider.mockResolvedValue(active); const svc = service(); await start(svc); + await settle(async () => (await row(f.companyId)).lastResolvedAt !== null); + await db.update(issues).set({ description: null }).where(eq(issues.id, f.issueId)); + await svc.syncIssue(f.issueId); await quiet(); + selects.mockClear(); provider.mockClear(); + await vi.advanceTimersByTimeAsync(900_000); + expect(selects).not.toHaveBeenCalled(); expect(provider).not.toHaveBeenCalled(); + provider.mockResolvedValue(terminal); + await db.update(issues).set({ description: `https://example.com/${f.companyId}` }).where(eq(issues.id, f.issueId)); + await svc.syncIssue(f.issueId); + await settle(async () => (await row(f.companyId)).isTerminal); await quiet(); + expect(provider).toHaveBeenCalledOnce(); + }); + + it("activates a newly ready plugin and fences a result after company permission is withdrawn", async () => { + clock(); const registry = pluginRegistryService(db); + const manifest: PaperclipPluginManifestV1 = { + id: `paperclip.refresh-${randomUUID()}`, apiVersion: 1, version: "1.0.0", displayName: "Refresh fixture", + description: "Simulated external object provider", author: "Paperclip", categories: ["connector"], + capabilities: ["external.objects.detect", "external.objects.read"], entrypoints: { worker: "worker.js" }, + objectReferences: [{ providerKey: "tracker", displayName: "Tracker", objectTypes: ["ticket"], urlPatterns: ["https://example.com/:id"] }], + }; + const plugin = await registry.install({ packageName: "refresh-fixture" }, manifest); + let finish!: (result: ExternalObjectResolveResult) => void; + const calls = vi.fn(async (_id: string, method: string) => { + if (method === "detectExternalObjects") return { detections: [] }; + if (calls.mock.calls.filter(call => call[1] === "resolveExternalObject").length === 1) + return new Promise(resolve => { finish = resolve; }); + return terminal; + }); + const svc = externalObjectService(db, { github: false, + pluginWorkerManager: { call: calls } as unknown as PluginWorkerManager, + detectors: [{ key: "fixture", detect: ({ urls }) => urls.map(canonical => ({ canonical, + detectorKey: "fixture", providerKey: "tracker", objectType: "ticket", externalId: canonical.canonicalIdentityHash })) }], + }); + await start(svc); const f = await fixture(); await svc.syncIssue(f.issueId); await quiet(); + const selects = vi.spyOn(db, "select"); + await vi.advanceTimersByTimeAsync(900_000); expect(selects).not.toHaveBeenCalled(); expect(calls).not.toHaveBeenCalled(); + await registry.updateStatus(plugin!.id, { status: "ready" }); + await settle(() => typeof finish === "function"); + await registry.upsertCompanySettings(plugin!.id, f.companyId, { enabled: false, settingsJson: {} }); + finish(terminal); + await settle(async () => (await row(f.companyId)).refreshToken === null); await quiet(); + expect((await row(f.companyId)).isTerminal).toBe(false); + selects.mockClear(); calls.mockClear(); + await vi.advanceTimersByTimeAsync(900_000); expect(selects).not.toHaveBeenCalled(); expect(calls).not.toHaveBeenCalled(); + calls.mockImplementation(async () => terminal); + await registry.upsertCompanySettings(plugin!.id, f.companyId, { enabled: true, settingsJson: {} }); + await settle(async () => (await row(f.companyId)).isTerminal); await quiet(); + expect(calls).toHaveBeenCalledOnce(); + }); + + it("keeps warm standby startup SQL-free and resumes a first committed producer", async () => { + clock(); admitted = false; reconcile = false; + const svc = service(); const selects = vi.spyOn(db, "select"); + await start(svc); await vi.advanceTimersByTimeAsync(90_000); + expect(selects).not.toHaveBeenCalled(); + const f = await fixture(); await svc.syncIssue(f.issueId); + selects.mockClear(); await vi.advanceTimersByTimeAsync(90_000); + expect(selects).not.toHaveBeenCalled(); expect(provider).not.toHaveBeenCalled(); + admitted = true; reconcile = true; + await vi.advanceTimersByTimeAsync(30_000); + await settle(async () => (await row(f.companyId))?.isTerminal); await quiet(); + expect(provider).toHaveBeenCalledOnce(); + }); + + it("renews an active provider lease so another service cannot duplicate its action", async () => { + clock(); const svc = service(); await start(svc); + let finish!: (result: ExternalObjectResolveResult) => void; + provider.mockImplementationOnce(() => new Promise(resolve => { finish = resolve; })); + const f = await fixture(); await svc.syncIssue(f.issueId); + await settle(() => provider.mock.calls.length === 1); + const claimed = await row(f.companyId); + await vi.advanceTimersByTimeAsync(60_000); + await settle(async () => (await row(f.companyId)).refreshStartedAt!.getTime() > claimed.refreshStartedAt!.getTime()); + const competing = await service().refreshObject(claimed.id, { companyId: f.companyId, + force: true, now: new Date(claimed.refreshStartedAt!.getTime() + 301_000) }); + expect(competing.reason).toBe("refresh_in_progress"); expect(provider).toHaveBeenCalledOnce(); + finish(terminal); await settle(async () => (await row(f.companyId)).isTerminal); await quiet(); + }); + + it("uses durable provider retry deadlines and waits for in-flight shutdown without overlap", async () => { + clock(); const svc = service(); await start(svc); + provider.mockResolvedValueOnce({ ok: false, liveness: "rate_limited", errorCode: "slow_down", retryAfterSeconds: 5 }); + const f = await fixture(); await svc.syncIssue(f.issueId); + await settle(async () => (await row(f.companyId))?.lastErrorCode === "slow_down"); + const saved = await row(f.companyId); + await settle(() => coordinator!.nextWakeAt() === saved.nextRefreshAt!.getTime()); + expect(saved.refreshToken).toBeNull(); + await vi.advanceTimersByTimeAsync(saved.nextRefreshAt!.getTime() - Date.now() - 1); + expect(provider).toHaveBeenCalledOnce(); + let finish!: (result: ExternalObjectResolveResult) => void; + provider.mockImplementationOnce(() => new Promise(resolve => { finish = resolve; })); + await vi.advanceTimersByTimeAsync(1); await settle(() => provider.mock.calls.length === 2); + await svc.syncIssue(f.issueId); await vi.advanceTimersByTimeAsync(0); + expect(provider).toHaveBeenCalledTimes(2); + let stopped = false; const stop = coordinator!.stop().then(() => { stopped = true; }); + await delay(10); expect(stopped).toBe(false); + finish(terminal); await stop; coordinator = undefined; + const selects = vi.spyOn(db, "select"); + await vi.advanceTimersByTimeAsync(600_000); + expect(selects).not.toHaveBeenCalled(); expect(provider).toHaveBeenCalledTimes(2); + }); + + it("reconstructs paginated multi-company work and never invokes inactive or never-refresh objects", async () => { + clock(); const svc = service(); const a = await fixture(); const b = await fixture(); const inactive = await fixture("paused"); + for (const companyId of [b.companyId, inactive.companyId]) { + await db.insert(externalObjects).values(Array.from({ length: 255 }, (_, index) => ({ + companyId, providerKey: "url", objectType: "link", externalId: String(index), + sanitizedCanonicalUrl: `https://example.com/${companyId}/${index}`, canonicalIdentityHash: `${companyId}-${index}`, + nextRefreshAt: null, + }))); + } + await db.update(issues).set({ description: Array.from({ length: 255 }, (_, index) => + `https://example.com/${a.companyId}/${index}`).join(" ") }).where(eq(issues.id, a.issueId)); + await svc.syncIssue(a.issueId); + const future = new Date(Date.now() + 60_000); + await db.update(externalObjects).set({ nextRefreshAt: future }).where(eq(externalObjects.companyId, a.companyId)); + await svc.syncIssue(b.issueId); + await start(svc); await settle(() => provider.mock.calls.length === 1); + await settle(() => coordinator!.nextWakeAt() === future.getTime()); + expect((await row(a.companyId)).nextRefreshAt).toEqual(future); + const companiesSeen = provider.mock.calls.map(call => (call as unknown as [{ companyId: string }])[0].companyId); + expect(companiesSeen).toEqual([b.companyId]); + await change(a.companyId, { isTerminal: true }); await quiet(); + await vi.advanceTimersByTimeAsync(600_000); expect(provider).toHaveBeenCalledOnce(); + }); +}); diff --git a/server/src/__tests__/external-objects-service.test.ts b/server/src/__tests__/external-objects-service.test.ts index e9022599779..e50e298b466 100644 --- a/server/src/__tests__/external-objects-service.test.ts +++ b/server/src/__tests__/external-objects-service.test.ts @@ -11,6 +11,7 @@ import { issues, plugins, } from "@paperclipai/db"; +import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js"; import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase, @@ -27,7 +28,8 @@ import type { PluginWorkerManager } from "../services/plugin-worker-manager.js"; import { createGitHubExternalObjectProvider } from "../services/github-external-object-provider.js"; const embeddedPostgresSupport = await getEmbeddedPostgresTestSupport(); -const describeEmbeddedPostgres = embeddedPostgresSupport.supported ? describe : describe.skip; +const externalDatabaseUrl = resolveExternalTestDatabaseUrl("PAPERCLIP_EXTERNAL_OBJECTS_TEST_DATABASE_URL"); +const describeEmbeddedPostgres = externalDatabaseUrl || embeddedPostgresSupport.supported ? describe : describe.skip; if (!embeddedPostgresSupport.supported) { console.warn( @@ -709,8 +711,8 @@ describeEmbeddedPostgres("externalObjectService", () => { let tempDb: Awaited> | null = null; beforeAll(async () => { - tempDb = await startEmbeddedPostgresTestDatabase("paperclip-external-objects-"); - db = createDb(tempDb.connectionString); + if (!externalDatabaseUrl) tempDb = await startEmbeddedPostgresTestDatabase("paperclip-external-objects-"); + db = createDb(externalDatabaseUrl ?? tempDb!.connectionString, { maxConnections: 6 }); }, 20_000); afterEach(async () => { @@ -725,6 +727,7 @@ describeEmbeddedPostgres("externalObjectService", () => { }); afterAll(async () => { + await db?.$client.end({ timeout: 1 }); await tempDb?.cleanup(); }); diff --git a/server/src/__tests__/github-connection-events.test.ts b/server/src/__tests__/github-connection-events.test.ts index 7c7b7025243..0417a434718 100644 --- a/server/src/__tests__/github-connection-events.test.ts +++ b/server/src/__tests__/github-connection-events.test.ts @@ -1,3 +1,4 @@ +import { externalObjectService } from "../services/external-objects.js"; import { startToolHealthWorker } from "./helpers/tool-health-worker.js"; import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js"; import { randomUUID } from "node:crypto"; @@ -35,7 +36,7 @@ describeEmbeddedPostgres("GitHub connection event delivery", () => { beforeAll(async () => { if (!externalTestDatabaseUrl) tempDb = await startEmbeddedPostgresTestDatabase("paperclip-github-events-"); if (externalTestDatabaseUrl) await applyPendingMigrations(externalTestDatabaseUrl); - db = createDb(externalTestDatabaseUrl ?? tempDb!.connectionString); + db = createDb(externalTestDatabaseUrl ?? tempDb!.connectionString, { maxConnections: 6 }); }, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS); afterEach(async () => { @@ -171,12 +172,27 @@ describeEmbeddedPostgres("GitHub connection event delivery", () => { expect(connector.leaseEvents).toHaveBeenCalledTimes(5); } + let releaseRefresh!: () => void; + let markStarted!: () => void; + const started = new Promise(resolve => { markStarted = resolve; }); + const held = new Promise(resolve => { releaseRefresh = resolve; }); + const resolver = externalObjectService(db, { github: false, resolvers: [{ + providerKey: "github", resolve: async () => { + markStarted(); await held; + return { ok: true, snapshot: { statusKey: "open", statusCategory: "open", statusTone: "info", isTerminal: false } }; + }, + }] }); + const oldRefresh = resolver.refreshObject(externalObjectId, { companyId, force: true }); + await started; + const warning = vi.spyOn(logger, "warn").mockImplementation(() => undefined); await expect(service.pollOnce({ forSetup: activeSetup })).resolves.toMatchObject({ leased: 2, processed: 1, duplicate: 0, failed: 1 }); expect(warning.mock.calls).toEqual([[{ deliveryId: "undecryptable-delivery", reason: "inner_event_authentication_failed", }, "GitHub connector event retained for retry"]]); warning.mockRestore(); + releaseRefresh(); + await expect(oldRefresh).resolves.toMatchObject({ refreshed: false, reason: "refresh_superseded" }); const [snapshot] = await db.select().from(externalObjects).where(eq(externalObjects.id, externalObjectId)); expect(snapshot).toMatchObject({ statusKey: "merged", @@ -184,6 +200,8 @@ describeEmbeddedPostgres("GitHub connection event delivery", () => { statusCategory: "succeeded", statusTone: "success", isTerminal: true, + refreshToken: null, + refreshStartedAt: null, remoteVersion: "2026-09-04T11:59:01.000Z", data: expect.objectContaining({ marker: "preserved", merged: true, headRef: "feature", baseSha: "b".repeat(40) }), }); diff --git a/server/src/index.ts b/server/src/index.ts index c127753889c..f6b59d277cf 100644 --- a/server/src/index.ts +++ b/server/src/index.ts @@ -1323,19 +1323,6 @@ async function startServerWithDatabaseTeardown( pluginWorkerManager, enabled: async () => (await instanceSettingsService(db).getExperimental()).enableExternalObjects === true, }); - const scheduleExternalObjectRefreshSweep = (now = new Date()) => { - if (isIdleTaskDrainActive() || heartbeatSchedulerStopped) return; - trackHeartbeatSchedulerWork(externalObjects - .refreshDueObjectsForActiveCompanies(50, now) - .then((result) => { - if (result.checked > 0 || result.refreshed > 0) { - logger.info({ ...result }, "external-object scheduler tick refreshed due objects"); - } - }) - .catch((err) => { - logger.error({ err }, "external-object scheduler tick failed"); - })); - }; // The retry backstop for orphan sandboxes. An acquire that rejects a // foreign-company insert tears the provisioned sandbox down. If that teardown @@ -1362,6 +1349,8 @@ async function startServerWithDatabaseTeardown( resolveNativeQuestion: (interaction) => deliverNativeQuestionResponse(db as any, interaction), }); const deliveryWork = app.locals.deliveryWork as ReturnType; + externalObjects.registerRefreshWork(deliveryWork, + () => !heartbeatSchedulerStopped && !isIdleTaskDrainActive() && !isWarmStandby()); if (config.nativeHarnessBackupExpirationEnabled) { nativeHarnessBackupExpiration.registerCheckpointExpiration(deliveryWork); } @@ -2007,7 +1996,6 @@ async function startServerWithDatabaseTeardown( } if (isIdleTaskDrainActive() || heartbeatSchedulerStopped) return; - scheduleExternalObjectRefreshSweep(new Date()); if (isIdleTaskDrainActive() || heartbeatSchedulerStopped) return; scheduleMergedPullRequestConfirmationSweep(); @@ -2133,7 +2121,6 @@ async function startServerWithDatabaseTeardown( // as the enabled path. await startEnvironmentCleanup(); startHeartbeatSchedulerInterval(() => { - scheduleExternalObjectRefreshSweep(new Date()); scheduleGitHubConnectionEventPoll(); scheduleGitHubConnectionContinuitySweep(); }); diff --git a/server/src/services/companies.ts b/server/src/services/companies.ts index 0f2ed547a79..f429b08badf 100644 --- a/server/src/services/companies.ts +++ b/server/src/services/companies.ts @@ -1,5 +1,6 @@ import { notifyChatInboundWakeNotices } from "./chat-inbound-wake-notice-notifications.js"; import { notifyCompanyRetentionWork } from "./retention-work-notifications.js"; +import { notifyExternalObjectRefresh } from "./external-object-refresh-notifications.js"; import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js"; import { isRunnerApiResponseAsset } from "./asset-content-access.js"; import { deleteCompany } from "./company-deletion.js"; @@ -334,6 +335,7 @@ export function companyService(db: Db, _budgetHooks: BudgetServiceHooks = {}) { if (companyPatch.status !== undefined && companyPatch.status !== existing.status) { await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers); await notifyDeliveryWork(tx, DELIVERY_QUEUES.skillSourceSync); + await notifyExternalObjectRefresh(tx, id); await notifyDeliveryWork(tx, DELIVERY_QUEUES.routineSchedules); await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors); await notifyCompanyRetentionWork(tx, id); @@ -490,6 +492,7 @@ export function companyService(db: Db, _budgetHooks: BudgetServiceHooks = {}) { if (!wasAlreadyArchived) { await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers); await notifyDeliveryWork(tx, DELIVERY_QUEUES.skillSourceSync); + await notifyExternalObjectRefresh(tx, id); await notifyDeliveryWork(tx, DELIVERY_QUEUES.routineSchedules); await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors); await notifyCompanyRetentionWork(tx, id); diff --git a/server/src/services/company-deletion.ts b/server/src/services/company-deletion.ts index 7ab88aadfa9..645e17f528b 100644 --- a/server/src/services/company-deletion.ts +++ b/server/src/services/company-deletion.ts @@ -1,3 +1,4 @@ +import { notifyExternalObjectRefresh } from "./external-object-refresh-notifications.js"; import { notifyUnsafeWorkspaceExportRecovery } from "./unsafe-workspace-export-notifications.js"; import { notifyCancelledWorkspacePreservation } from "./cancelled-workspace-preservation-notifications.js"; import { notifyInstructionCollection } from "./instruction-collection-notifications.js"; @@ -71,6 +72,7 @@ export async function deleteCompany(db: Db, id: string) { const [existing] = await tx.select({ id: companies.id }).from(companies) .where(eq(companies.id, id)).for("no key update"); if (!existing) return null; + await notifyExternalObjectRefresh(tx, id); await notifyChatCompanyConfiguration(tx, id); await chatCompanyDeletion.deleteCompanyData(tx, id); await notifyDeliveryWork(tx, DELIVERY_QUEUES.skillSourceSync); diff --git a/server/src/services/delivery-work-notifications.ts b/server/src/services/delivery-work-notifications.ts index 9195f1fcb12..e9555d8ad53 100644 --- a/server/src/services/delivery-work-notifications.ts +++ b/server/src/services/delivery-work-notifications.ts @@ -6,6 +6,7 @@ export const DELIVERY_QUEUES = { providerTraceRetention: "provider-trace-retention", nativeRunnerEvidenceRetention: "native-runner-evidence-retention", skillVersionRetention: "skill-version-retention", + externalObjectRefresh: "external-object-refresh", databaseBackups: "database-backups", importSpoolCleanup: "import-spool-cleanup", externalConversationSettlement: "external-conversation-settlement", diff --git a/server/src/services/external-object-refresh-notifications.ts b/server/src/services/external-object-refresh-notifications.ts new file mode 100644 index 00000000000..95833421a8a --- /dev/null +++ b/server/src/services/external-object-refresh-notifications.ts @@ -0,0 +1,7 @@ +import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js"; + +export async function notifyExternalObjectRefresh(tx: object, companyId?: string, objectId?: string) { + await notifyDeliveryWork(tx, DELIVERY_QUEUES.externalObjectRefresh, + companyId ? JSON.stringify({ companyId, ...(objectId ? { objectId } : {}) }) : "*"); +} + diff --git a/server/src/services/external-objects.ts b/server/src/services/external-objects.ts index 56a8fb91834..c49df7ff076 100644 --- a/server/src/services/external-objects.ts +++ b/server/src/services/external-objects.ts @@ -1,7 +1,7 @@ import { randomUUID } from "node:crypto"; -import { and, asc, eq, inArray, isNull, lte, not, or, sql } from "drizzle-orm"; +import { and, asc, eq, exists, gt, inArray, isNotNull, isNull, lte, not, or, sql } from "drizzle-orm"; import type { Db } from "@paperclipai/db"; -import { companies, documents, externalObjectMentions, externalObjects, issueComments, issueDocuments, issues, plugins } from "@paperclipai/db"; +import { companies, documents, externalObjectMentions, externalObjects, issueComments, issueDocuments, issues, plugins, pluginCompanySettings } from "@paperclipai/db"; import { formatExternalObjectMentionSourceLabel, type ExternalObjectCanonicalUrl, @@ -18,6 +18,10 @@ import { notFound } from "../errors.js"; import { logger } from "../middleware/logger.js"; import { logActivity, type LogActivityInput } from "./activity-log.js"; import { createGitHubExternalObjectProvider, type GitHubExternalObjectProviderOptions } from "./github-external-object-provider.js"; +import { notifyExternalObjectRefresh } from "./external-object-refresh-notifications.js"; +import { DELIVERY_QUEUES } from "./delivery-work-notifications.js"; +import type { createDeliveryWorkCoordinator } from "./delivery-work-coordinator.js"; +import { registerScopedDeadlineWork } from "./scoped-deadline-work.js"; import { publishLiveEvent } from "./live-events.js"; import type { PluginWorkerManager } from "./plugin-worker-manager.js"; @@ -95,6 +99,18 @@ const DEFAULT_RETRY_AFTER_SECONDS = 300; const DEFAULT_REFRESH_LEASE_SECONDS = 300; const REFRESH_LEASE_RENEW_INTERVAL_MS = 60_000; +type RefreshScope = { companyId: string; objectId?: string }; +type RefreshDeadline = { id: string; companyId: string; dueAt: number }; + + +function refreshableObjectWhere() { + return or(eq(externalObjects.isTerminal, false), and( + eq(externalObjects.providerKey, "github"), + eq(externalObjects.statusKey, "not_found"), + not(sql`coalesce(${externalObjects.data}->>'probeVerified', 'false') = 'true'`), + ))!; +} + function sourceWhere(input: ExternalObjectSourceContext) { const conditions = [ eq(externalObjectMentions.companyId, input.companyId), @@ -245,7 +261,7 @@ function objectSnapshot(object: ExternalObjectRecord): PluginExternalObjectRecor }; } -async function readyObjectReferencePlugins(db: Db) { +async function readyObjectReferencePlugins(db: Db, companyId?: string) { return db .select({ id: plugins.id, @@ -253,7 +269,12 @@ async function readyObjectReferencePlugins(db: Db) { manifestJson: plugins.manifestJson, }) .from(plugins) - .where(eq(plugins.status, "ready")) + .where(and(eq(plugins.status, "ready"), companyId ? not(exists(db.select({ id: pluginCompanySettings.id }) + .from(pluginCompanySettings).where(and( + eq(pluginCompanySettings.pluginId, plugins.id), + eq(pluginCompanySettings.companyId, companyId), + eq(pluginCompanySettings.enabled, false), + )))) : undefined)) .orderBy(asc(plugins.installOrder)) .then((rows) => rows.filter((row) => (row.manifestJson.objectReferences?.length ?? 0) > 0), @@ -267,7 +288,7 @@ function createPluginProviderDetector( return { key: "plugin-object-reference-providers", async detect(input) { - const providers = await readyObjectReferencePlugins(db); + const providers = await readyObjectReferencePlugins(db, input.companyId); const detections: ExternalObjectDetection[] = []; for (const provider of providers) { @@ -326,7 +347,7 @@ async function resolveViaPluginProvider( object: ExternalObjectRecord, ): Promise { if (!pluginWorkerManager) return null; - const providers = await readyObjectReferencePlugins(db); + const providers = await readyObjectReferencePlugins(db, object.companyId); for (const provider of providers) { const manifest = provider.manifestJson; if (!manifest.capabilities.includes("external.objects.read")) continue; @@ -374,10 +395,38 @@ export function externalObjectService( ...(opts.detectors ?? []), ...(githubProvider ? [githubProvider.detector] : []), ]); - const resolverRegistry = createExternalObjectResolverRegistry([ + const resolvers = [ ...(opts.resolvers ?? []), ...(githubProvider?.resolvers ?? []), - ]); + ]; + const resolverRegistry = createExternalObjectResolverRegistry(resolvers); + + function automaticRefreshWhere(database: Db = db) { + const supported = resolvers.map(resolver => and( + eq(externalObjects.providerKey, resolver.providerKey), + resolver.objectType ? eq(externalObjects.objectType, resolver.objectType) : undefined, + )!); + if (opts.pluginWorkerManager) supported.push(exists(database.select({ id: plugins.id }).from(plugins).where(and( + eq(plugins.status, "ready"), + sql`${plugins.manifestJson}->'capabilities' ? 'external.objects.read'`, + sql`exists (select 1 from jsonb_array_elements(coalesce(${plugins.manifestJson}->'objectReferences', '[]'::jsonb)) as reference + where reference->>'providerKey' = ${externalObjects.providerKey} + and reference->'objectTypes' ? ${externalObjects.objectType})`, + not(exists(database.select({ id: pluginCompanySettings.id }).from(pluginCompanySettings).where(and( + eq(pluginCompanySettings.pluginId, plugins.id), + eq(pluginCompanySettings.companyId, externalObjects.companyId), + eq(pluginCompanySettings.enabled, false), + )))), + )))); + return and( + refreshableObjectWhere(), isNotNull(externalObjects.nextRefreshAt), + supported.length ? or(...supported) : sql`false`, + exists(database.select({ id: externalObjectMentions.id }).from(externalObjectMentions).where(and( + eq(externalObjectMentions.companyId, externalObjects.companyId), + eq(externalObjectMentions.objectId, externalObjects.id), + ))), + )!; + } async function isEnabled() { if (typeof opts.enabled === "function") return await opts.enabled(); @@ -422,6 +471,7 @@ export function externalObjectService( nextRefreshAt: now, updatedAt: now, }; + await notifyExternalObjectRefresh(dbOrTx, companyId); const inserted = await dbOrTx .insert(externalObjects) .values(values) @@ -451,6 +501,7 @@ export function externalObjectService( dbOrTx: any = db, ) { const urls = extractExternalObjectCanonicalUrls(input.text ?? ""); + await notifyExternalObjectRefresh(dbOrTx, input.companyId); await dbOrTx.delete(externalObjectMentions).where(sourceWhere(input)); if (urls.length === 0) return; @@ -534,6 +585,7 @@ export function externalObjectService( .where(eq(issueComments.id, commentId)) .then((rows: Array<{ id: string; companyId: string; issueId: string; body: string }>) => rows[0] ?? null); if (!comment) { + await notifyExternalObjectRefresh(tx); await tx .delete(externalObjectMentions) .where(and(eq(externalObjectMentions.sourceKind, "comment"), eq(externalObjectMentions.sourceRecordId, commentId))); @@ -568,6 +620,7 @@ export function externalObjectService( .where(eq(documents.id, documentId)) .then((rows: Array<{ documentId: string; companyId: string; issueId: string; key: string; body: string }>) => rows[0] ?? null); if (!document) { + await notifyExternalObjectRefresh(tx); await tx .delete(externalObjectMentions) .where(and(eq(externalObjectMentions.sourceKind, "document"), eq(externalObjectMentions.sourceRecordId, documentId))); @@ -780,6 +833,7 @@ export function externalObjectService( companyId: string; actor?: Pick; force?: boolean; + scheduled?: boolean; now?: Date; }; @@ -801,6 +855,18 @@ export function externalObjectService( return { object: toObjectPayload(latest, now), refreshed: false, reason: "refresh_superseded" as const }; } + async function persistRefresh(object: ExternalObjectRecord, token: string, + patch: Partial, scheduled = false) { + return db.transaction(async tx => { + await notifyExternalObjectRefresh(tx, object.companyId, object.id); + const rows = await tx.update(externalObjects).set(patch) + .where(and(refreshOwnerWhere(object, token), scheduled ? automaticRefreshWhere(tx as unknown as Db) : undefined)).returning(); + if (scheduled && rows.length === 0) await tx.update(externalObjects) + .set({ refreshStartedAt: null, refreshToken: null }).where(refreshOwnerWhere(object, token)); + return rows; + }); + } + async function resolveObjectRefresh( object: ExternalObjectRecord, input: RefreshObjectInput, @@ -810,17 +876,13 @@ export function externalObjectService( const pluginResult = await resolveViaPluginProvider(db, opts.pluginWorkerManager, object); const resolver = pluginResult ? null : resolverRegistry.find(object); if (!pluginResult && !resolver) { - const [updated] = await db - .update(externalObjects) - .set({ + const [updated] = await persistRefresh(object, refreshToken, { liveness: object.isTerminal ? object.liveness : visibleLiveness(object, now) === "fresh" ? "stale" : object.liveness, nextRefreshAt: addSeconds(now, DEFAULT_RETRY_AFTER_SECONDS), refreshStartedAt: null, refreshToken: null, updatedAt: now, - }) - .where(refreshOwnerWhere(object, refreshToken)) - .returning(); + }, input.scheduled); if (!updated) return refreshSupersededResult(object, now); return { object: toObjectPayload(updated ?? object, now), refreshed: false, reason: "no_resolver" as const }; } @@ -831,9 +893,7 @@ export function externalObjectService( // terminal. Clear a previous terminal verdict so the object keeps being // scheduled for automatic refresh and self-heals (e.g. a GitHub "not // found" that later resolves once repo access improves). - const [updated] = await db - .update(externalObjects) - .set({ + const [updated] = await persistRefresh(object, refreshToken, { liveness: result.liveness, isTerminal: false, lastErrorAt: now, @@ -843,9 +903,7 @@ export function externalObjectService( refreshStartedAt: null, refreshToken: null, updatedAt: now, - }) - .where(refreshOwnerWhere(object, refreshToken)) - .returning(); + }, input.scheduled); if (!updated) return refreshSupersededResult(object, now); publishLiveEvent({ companyId: object.companyId, @@ -879,14 +937,10 @@ export function externalObjectService( refreshToken: null, updatedAt: now, }; - const [updated] = await db - .update(externalObjects) - .set({ + const [updated] = await persistRefresh(object, refreshToken, { ...patch, lastChangedAt: objectChanged(object, { ...object, ...patch }) ? now : object.lastChangedAt, - }) - .where(refreshOwnerWhere(object, refreshToken)) - .returning(); + }, input.scheduled); if (!updated) return refreshSupersededResult(object, now); const next = updated ?? object; if (objectChanged(object, next) && input.actor) { @@ -947,6 +1001,8 @@ export function externalObjectService( eq(externalObjects.id, object.id), eq(externalObjects.companyId, object.companyId), dueOrForced, + input.scheduled ? automaticRefreshWhere() : undefined, + input.scheduled ? sql`exists (select 1 from ${companies} where ${companies.id} = ${externalObjects.companyId} and ${companies.status} = 'active')` : undefined, )) .returning(); return claimed ?? null; @@ -1032,26 +1088,7 @@ export function externalObjectService( async function refreshDueObjectsUnchecked(companyId: string, limit = 50, now = new Date()) { const staleRefreshStartedBefore = new Date(now.getTime() - DEFAULT_REFRESH_LEASE_SECONDS * 1000); - // Terminal objects are usually skipped forever, but a GitHub "not found" - // verdict is only trustworthy while the repository is actually visible to - // the configured credentials. GitHub hides inaccessible (often private) - // repos behind a 404, so a not-found object can become resolvable later - // when credentials or repo visibility improve. Re-check objects that were - // frozen as "not found" before the repository probe existed so they - // self-heal instead of freezing. Once the provider has probed the repo and - // confirmed it is visible (data.probeVerified = "true"), the not-found - // verdict is definitive, so stop re-polling that object to avoid burning - // the company's GitHub API budget on deleted references forever. - const probeVerifiedNotFound = sql`coalesce(${externalObjects.data}->>'probeVerified', 'false') = 'true'`; - const refreshable = or( - eq(externalObjects.isTerminal, false), - and( - eq(externalObjects.isTerminal, true), - eq(externalObjects.providerKey, "github"), - eq(externalObjects.statusKey, "not_found"), - not(probeVerifiedNotFound), - ), - )!; + const refreshable = refreshableObjectWhere(); const due = await db .select({ id: externalObjects.id }) .from(externalObjects) @@ -1099,6 +1136,59 @@ export function externalObjectService( return { companies: activeCompanies.length, checked, refreshed }; } + function registerRefreshWork( + coordinator: ReturnType, + canRun: () => boolean, + ) { + const cursors = new Map(); + const scopeKey = (scope?: RefreshScope) => JSON.stringify(scope ?? null); + async function load(scope?: RefreshScope) { + if (!(await isEnabled())) { cursors.clear(); return []; } + const key = scopeKey(scope); + const cursor = cursors.get(key); + const rows = await db.select({ id: externalObjects.id, companyId: externalObjects.companyId, + nextRefreshAt: externalObjects.nextRefreshAt, refreshStartedAt: externalObjects.refreshStartedAt }).from(externalObjects) + .innerJoin(companies, eq(companies.id, externalObjects.companyId)) + .where(and( + eq(companies.status, "active"), automaticRefreshWhere(), + scope ? eq(externalObjects.companyId, scope.companyId) : undefined, + scope?.objectId ? eq(externalObjects.id, scope.objectId) : undefined, + cursor ? gt(externalObjects.id, cursor) : undefined, + )).orderBy(asc(externalObjects.id)).limit(250); + const entries = rows.map(deadline); + if (rows.length === 250) cursors.set(key, rows[rows.length - 1]!.id); + else cursors.delete(key); + return { rows: entries, complete: rows.length < 250 }; + } + function deadline(object: Pick): RefreshDeadline { + return { id: object.id, companyId: object.companyId, + dueAt: Math.max(object.nextRefreshAt!.getTime(), object.refreshStartedAt + ? object.refreshStartedAt.getTime() + DEFAULT_REFRESH_LEASE_SECONDS * 1000 : 0) }; + } + return registerScopedDeadlineWork(coordinator, { + queue: DELIVERY_QUEUES.externalObjectRefresh, + batchSize: 50, + minimumPassIntervalMs: 0, + decodeScope: hint => hint === "*" ? undefined : JSON.parse(hint) as RefreshScope, + matches: (entry, scope) => !scope || (entry.companyId === scope.companyId + && (!scope.objectId || entry.id === scope.objectId)), + load, + check: async (entry, signal) => { + if (signal?.aborted || !canRun()) return entry; + const current = await load({ companyId: entry.companyId, objectId: entry.id }); + const scheduled = Array.isArray(current) ? current[0] : current.rows[0]; + if (!scheduled || scheduled.dueAt > Date.now()) return scheduled ?? null; + await refreshObject(entry.id, { + companyId: entry.companyId, + scheduled: true, + actor: { actorType: "system", actorId: "external-object-resolver", agentId: null, runId: null }, + }); + const updated = await load({ companyId: entry.companyId, objectId: entry.id }); + return (Array.isArray(updated) ? updated[0] : updated.rows[0]) ?? null; + }, + }, canRun); + } + return { syncIssue, syncComment, @@ -1114,5 +1204,6 @@ export function externalObjectService( refreshIssueObjects, refreshDueObjects, refreshDueObjectsForActiveCompanies, + registerRefreshWork, }; } diff --git a/server/src/services/github-connection-events.ts b/server/src/services/github-connection-events.ts index 47bdac122a2..76755ce30c3 100644 --- a/server/src/services/github-connection-events.ts +++ b/server/src/services/github-connection-events.ts @@ -1,3 +1,4 @@ +import { notifyExternalObjectRefresh } from "./external-object-refresh-notifications.js"; import { notifyToolConnectionHealth } from "./tool-connection-health-notifications.js"; import { dispatchGitHubBotCloudEvent } from "./chat-github-cloud-ingress.js"; import { chatEndpoints } from "@paperclipai/db"; @@ -244,25 +245,30 @@ export function githubConnectionEventService( const snapshot = githubSnapshotUpdate(event.payload); if (!snapshot) return; const appliedAt = now(); - await db.update(externalObjects).set({ - statusKey: snapshot.statusKey, - statusLabel: snapshot.statusLabel, - statusCategory: snapshot.statusCategory, - statusTone: snapshot.statusTone, - statusIconKey: snapshot.statusIconKey, - isTerminal: snapshot.merged || snapshot.state === "closed", - data: sql`${externalObjects.data} || ${JSON.stringify(snapshot.data)}::jsonb`, - remoteVersion: snapshot.remoteVersion, - lastResolvedAt: appliedAt, - lastChangedAt: appliedAt, - nextRefreshAt: appliedAt, - updatedAt: appliedAt, - }).where(and( - eq(externalObjects.companyId, companyId), - eq(externalObjects.providerKey, "github"), - eq(externalObjects.objectType, "pull_request"), - sql`lower(${externalObjects.externalId}) = lower(${snapshot.externalId})`, - )); + await db.transaction(async tx => { + await notifyExternalObjectRefresh(tx, companyId); + await tx.update(externalObjects).set({ + statusKey: snapshot.statusKey, + statusLabel: snapshot.statusLabel, + statusCategory: snapshot.statusCategory, + statusTone: snapshot.statusTone, + statusIconKey: snapshot.statusIconKey, + isTerminal: snapshot.merged || snapshot.state === "closed", + data: sql`${externalObjects.data} || ${JSON.stringify(snapshot.data)}::jsonb`, + remoteVersion: snapshot.remoteVersion, + lastResolvedAt: appliedAt, + lastChangedAt: appliedAt, + nextRefreshAt: appliedAt, + refreshStartedAt: null, + refreshToken: null, + updatedAt: appliedAt, + }).where(and( + eq(externalObjects.companyId, companyId), + eq(externalObjects.providerKey, "github"), + eq(externalObjects.objectType, "pull_request"), + sql`lower(${externalObjects.externalId}) = lower(${snapshot.externalId})`, + )); + }); if (snapshot.merged && event.action === "closed") { await issueThreadInteractionService(db, { wakeup: options.wakeup }) .sweepMergedPullRequestConfirmations([{ diff --git a/server/src/services/instance-settings.ts b/server/src/services/instance-settings.ts index 0208653a86f..a2f5eb5015c 100644 --- a/server/src/services/instance-settings.ts +++ b/server/src/services/instance-settings.ts @@ -13,6 +13,7 @@ import { notifyInstructionCommit } from "./instruction-commit-notifications.js"; import { notifyScheduledRetryCancellation } from "./scheduled-retry-cancellation-notifications.js"; import { notifyVoiceChange } from "./voice/voice-recovery-notifications.js"; import { notifyToolConnectionHealth } from "./tool-connection-health-notifications.js"; +import { notifyExternalObjectRefresh } from "./external-object-refresh-notifications.js"; import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js"; import type { Db } from "@paperclipai/db"; import { companies, instanceSettings } from "@paperclipai/db"; @@ -628,6 +629,7 @@ export function instanceSettingsService(db: Db, options: InstanceSettingsService await notifyDeliveryWork(tx, DELIVERY_QUEUES.mcpEvents, "*"); await notifyDeliveryWork(tx, DELIVERY_QUEUES.dotEvents, "*"); await notifyDeliveryWork(tx, DELIVERY_QUEUES.grokDispatch); + if (patch.enableExternalObjects !== undefined) await notifyExternalObjectRefresh(tx); if (patch.enableWorktreeRunExecution !== undefined) await notifyScheduledRetryCancellation(tx); if (patch.enableWorktreeRunExecution !== undefined) { await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers); diff --git a/server/src/services/issues.ts b/server/src/services/issues.ts index 42c65bb5d02..53ac68718a6 100644 --- a/server/src/services/issues.ts +++ b/server/src/services/issues.ts @@ -1,3 +1,4 @@ +import { notifyExternalObjectRefresh } from "./external-object-refresh-notifications.js"; import { notifyExternalConversationSettlement } from "./external-conversation-settlement-notifications.js"; import { notifyNativeRecoveryNotices } from "./native-recovery-notice-notifications.js"; import { notifyNativeRunProjection } from "./native-run-projection-notifications.js"; @@ -11953,6 +11954,7 @@ export function issueService(db: Db) { const [owner] = await tx.select({ companyId: issues.companyId }).from(issues).where(eq(issues.id, id)); if (owner) { await notifyExternalConversationSettlement(tx, { companyId: owner.companyId }); + await notifyExternalObjectRefresh(tx, owner.companyId); await notifyApprovalWork(tx, owner.companyId); } await notifyDecisionTargetIssueRemoval(tx, id); diff --git a/server/src/services/plugin-registry.ts b/server/src/services/plugin-registry.ts index 239c3c50490..62d9b7cde70 100644 --- a/server/src/services/plugin-registry.ts +++ b/server/src/services/plugin-registry.ts @@ -1,3 +1,4 @@ +import { notifyExternalObjectRefresh } from "./external-object-refresh-notifications.js"; import { isDeepStrictEqual } from "node:util"; import { writePluginJobChange } from "./plugin-job-work-notifications.js"; import { asc, eq, isNull, ne, sql, and } from "drizzle-orm"; @@ -239,12 +240,15 @@ export function pluginRegistryService(db: Db) { setClause.categories = data.manifest.categories; } - return db - .update(plugins) - .set(setClause) - .where(eq(plugins.id, id)) - .returning() - .then((rows) => rows[0] ?? null); + return db.transaction(async tx => { + await notifyExternalObjectRefresh(tx); + return tx + .update(plugins) + .set(setClause) + .where(eq(plugins.id, id)) + .returning() + .then((rows) => rows[0] ?? null); + }); }, // ----- Status --------------------------------------------------------- @@ -254,16 +258,19 @@ export function pluginRegistryService(db: Db) { const plugin = await getById(id); if (!plugin) throw notFound("Plugin not found"); - return db - .update(plugins) - .set({ - status: input.status, - lastError: input.lastError ?? null, - updatedAt: new Date(), - }) - .where(eq(plugins.id, id)) - .returning() - .then((rows) => rows[0] ?? null); + return db.transaction(async tx => { + await notifyExternalObjectRefresh(tx); + return tx + .update(plugins) + .set({ + status: input.status, + lastError: input.lastError ?? null, + updatedAt: new Date(), + }) + .where(eq(plugins.id, id)) + .returning() + .then((rows) => rows[0] ?? null); + }); }, // ----- Uninstall / Remove -------------------------------------------- @@ -280,6 +287,7 @@ export function pluginRegistryService(db: Db) { if (!plugin) throw notFound("Plugin not found"); return writePluginJobChange(db, async db => { + await notifyExternalObjectRefresh(db); if (removeData) { // Hard delete – plugin_config cascades via FK onDelete return db @@ -343,6 +351,7 @@ export function pluginRegistryService(db: Db) { return db.transaction(async tx => { const db = tx as unknown as Db; await notifyDeliveryWork(db, DELIVERY_QUEUES.pluginLifecycle); + await notifyExternalObjectRefresh(db, companyId); const plugin = await getById(pluginId, db); if (!plugin) throw notFound("Plugin not found"); @@ -385,6 +394,7 @@ export function pluginRegistryService(db: Db) { return db.transaction(async tx => { const db = tx as unknown as Db; await notifyDeliveryWork(db, DELIVERY_QUEUES.pluginLifecycle); + await notifyExternalObjectRefresh(db, companyId); const plugin = await getById(pluginId, db); if (!plugin) throw notFound("Plugin not found"); @@ -467,6 +477,7 @@ export function pluginRegistryService(db: Db) { return db.transaction(async tx => { const db = tx as unknown as Db; await notifyDeliveryWork(db, DELIVERY_QUEUES.pluginLifecycle); + await notifyExternalObjectRefresh(db, companyId); const plugin = await getById(pluginId, db); if (!plugin) throw notFound("Plugin not found"); From 17564db697a1b6d3c1a417a0f0a719b38885b5f3 Mon Sep 17 00:00:00 2001 From: Dotta Date: Fri, 30 Oct 2026 16:25:54 -0500 Subject: [PATCH 07/74] fix(server): isolate backup discovery and recover orphan cleanup Co-Authored-By: Paperclip --- .../2026-10-30-backup-expiration-sleep.md | 8 ++- .../native-harness-backup-expiration.test.ts | 59 +++++++++++++++++++ .../server-startup-feedback-export.test.ts | 10 +++- .../native-harness-backup-expiration-work.ts | 37 ++++++++++-- .../native-harness-backup-expiration.ts | 19 +++--- 5 files changed, 115 insertions(+), 18 deletions(-) diff --git a/doc/plans/2026-10-30-backup-expiration-sleep.md b/doc/plans/2026-10-30-backup-expiration-sleep.md index 47d41bbf581..91cf1e9ec0a 100644 --- a/doc/plans/2026-10-30-backup-expiration-sleep.md +++ b/doc/plans/2026-10-30-backup-expiration-sleep.md @@ -14,7 +14,9 @@ must invalidate the source when it changes. A busy process or failed I/O leaves real unresolved cleanup and uses the shared retry backoff. Dry-run reports an eligible source once and waits for source invalidation or restart. -Startup pages existing disk scopes. Roots without backup or cleanup tombstones +Startup pages existing disk scopes and indexes company ownership, including +protected backups. Company notifications inspect only that index; local backup +publication adds newly created roots. Roots without backup or cleanup tombstones perform no backup SQL. After discovery and all obligations settle, there is no recurring SQL or filesystem scan. Startup reconstructs future work; inspecting a source never changes its business timestamps. Runtime task close/reopen and @@ -32,7 +34,9 @@ no permanent database listener and does not add a polling table or migration. The existing issue share locks now remain held through the retirement rename. This fences task reopen while the backup becomes unavailable. A failed database transaction restores `.retiring-*`; a process interruption is recovered from -that existing file journal at startup. After database settlement, `.expired-*` +that existing file journal at startup. Journal reconciliation runs before +identity or owner lookup, so an agent purge cannot strand expired physical +cleanup or prevent restoration of a retiring backup. After database settlement, `.expired-*` records outstanding physical removal so a partial delete retries idempotently. Ownership, session identity, live run, lease and local execution safety checks remain in the backup owner. No host-sleep blocker or expiry-on-use rule changes. diff --git a/server/src/__tests__/native-harness-backup-expiration.test.ts b/server/src/__tests__/native-harness-backup-expiration.test.ts index 1ac2539545e..afadf5d1d63 100644 --- a/server/src/__tests__/native-harness-backup-expiration.test.ts +++ b/server/src/__tests__/native-harness-backup-expiration.test.ts @@ -1,3 +1,4 @@ +import { agentService } from "../services/agents.js"; import { publishNativeHarnessBackup } from "../services/native-runtime/native-harness-backup-publication.js"; import { registerNativeHarnessBackupExpirationWork } from "../services/native-runtime/native-harness-backup-expiration-work.js"; import { noteNativeHarnessBackupChange } from "../services/native-harness-backup-expiration-notifications.js"; @@ -711,6 +712,64 @@ describeEmbeddedPostgres("native harness backup expiration", () => { await quiet(); }); + it("company-only notifications inspect only the indexed company's blocked backup sources", async () => { + const own = await seedSession({ issueStatus: "todo" }); + const other = await seedSession({ issueStatus: "todo" }); + await startBackups({ maxEntries: 1 }); await settled(); await quiet(); + const sql: Array<{ query: string; parameters: unknown[] }> = []; + const previous = db.$client.options.debug; + db.$client.options.debug = (_connection, query, parameters) => { sql.push({ query, parameters }); }; + const directories = vi.spyOn(fs, "readdir"); + try { + const closed = await issueService(db).update(own.issueId, { status: "done" }); + await settled(new Date(closed!.completedAt!).getTime() + 7 * DAY_MS + 1); + const unrelated = [other.companyId, other.agentId, other.runId, other.sessionId, other.issueId]; + expect(sql.filter(query => query.parameters.some(parameter => + unrelated.some(id => String(parameter).includes(id))))).toEqual([]); + expect(directories.mock.calls.some(call => call[0] === stateBase + || String(call[0]).startsWith(other.root))).toBe(false); + expect(await exists(other.current)).toBe(true); + } finally { db.$client.options.debug = previous; directories.mockRestore(); } + await quiet(); + }); + + it.each([false, true])("finishes cleanup journals after ordinary agent deletion (restart %s)", async restart => { + const session = await seedSession(); const remove = fs.rm; + const fault = vi.spyOn(fs, "rm").mockImplementation(async (...args) => { + if (String(args[0]).includes(".expired-")) throw new Error("controlled orphan cleanup failure"); + return remove(...args); + }); + await startBackups(); + expect(backupErrors).toHaveLength(1); + const retryAt = coordinator!.nextWakeAt()!; + const backupRoot = path.dirname(session.current); + expect((await fs.readdir(backupRoot)).some(name => name.startsWith(".expired-"))).toBe(true); + if (restart) { await backupWorker!.stop(); await coordinator!.stop(); } + await db.update(agents).set({ status: "terminated", lifecycleState: "terminated" }).where(eq(agents.id, session.agentId)); + await agentService(db).remove(session.agentId); + expect(await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, session.runId))).toEqual([]); + fault.mockRestore(); + if (restart) await startBackups(); else await vi.advanceTimersByTimeAsync(retryAt - Date.now()); + await settled(); + expect(await fs.readdir(backupRoot)).toEqual([]); expect(await taskSessionCount(session.agentId)).toBe(0); + await quiet(); + }); + + it("restores an orphan retiring journal without expiring the recovered backup or unrelated names", async () => { + const session = await seedSession(); const backupRoot = path.dirname(session.current); + const retiring = path.join(backupRoot, `.retiring-${randomUUID()}`); + await fs.rename(session.current, retiring); + await fs.mkdir(path.join(backupRoot, ".expired-unrecognized")); + await db.update(agents).set({ status: "terminated", lifecycleState: "terminated" }).where(eq(agents.id, session.agentId)); + await agentService(db).remove(session.agentId); + await fs.rm(path.join(session.root, "control-plane"), { recursive: true }); + await startBackups(); await settled(); + expect(await exists(session.current)).toBe(true); expect(await exists(retiring)).toBe(false); + expect(await exists(path.join(session.current, "manifest.json"))).toBe(true); + expect(await exists(path.join(backupRoot, ".expired-unrecognized"))).toBe(true); + await quiet(); + }); + it("retains live lease references and only removes the owning company's backup after lease release", async () => { const first = await seedSession({ leaseStatus: "retained" }); const other = await seedSession({ leaseStatus: "retained" }); diff --git a/server/src/__tests__/server-startup-feedback-export.test.ts b/server/src/__tests__/server-startup-feedback-export.test.ts index b645d7437c8..801c4c64ea6 100644 --- a/server/src/__tests__/server-startup-feedback-export.test.ts +++ b/server/src/__tests__/server-startup-feedback-export.test.ts @@ -566,6 +566,8 @@ vi.mock("../services/native-runtime/native-harness-backup-expiration.js", () => tick: vi.fn(async () => ({ checked: 0, candidates: 0, evidenceCandidates: 0, failed: 0, truncated: false })), noteEvidence: vi.fn(), nextEvidenceAt: vi.fn(() => null), tickCheckpoints: vi.fn(), registerCheckpointExpiration: vi.fn(), + backupScopes: vi.fn(async () => ({ names: [], complete: true })), + reconcileBackup: vi.fn(async () => null), expireBackup: vi.fn(), })), })); @@ -676,9 +678,12 @@ describe("startServer feedback export wiring", () => { loadConfigMock.mockReturnValue(buildTestConfig({ nativeHarnessBackupExpirationEnabled: enabled })); await startServer(); await settleStartupHousekeeping(); expect(deliveryWorkRegisterMock.mock.calls.filter(([queue]) => queue === DELIVERY_QUEUES.nativeRunnerEvidenceRetention)).toHaveLength(enabled ? 1 : 0); - expect(vi.mocked(nativeHarnessBackupExpirationService).mock.calls.map(([, options]) => options?.duty)).toEqual(enabled ? ["backups", "evidence"] : ["backups"]); + expect(vi.mocked(nativeHarnessBackupExpirationService).mock.calls.map(([, options]) => options?.duty)).toEqual(enabled ? ["backups", "backups", "evidence"] : ["backups"]); const backups = vi.mocked(nativeHarnessBackupExpirationService).mock.results[0]!.value; expect(backups.registerCheckpointExpiration).toHaveBeenCalledTimes(enabled ? 1 : 0); + expect(deliveryWorkRegisterMock.mock.calls.filter(([queue]) => queue === DELIVERY_QUEUES.nativeHarnessBackupExpiration)).toHaveLength(enabled ? 1 : 0); + expect(backups.tick).not.toHaveBeenCalled(); + expect(deliveryWorkErrors.filter(error => error.queue === DELIVERY_QUEUES.nativeHarnessBackupExpiration)).toEqual([]); }); it("tracks held evidence reconstruction after HTTP readiness", async () => { @@ -686,6 +691,9 @@ describe("startServer feedback export wiring", () => { const pending = Promise.withResolvers<{ truncated: boolean }>(); vi.mocked(nativeHarnessBackupExpirationService).mockReturnValueOnce({ tick: vi.fn(), noteEvidence: vi.fn(), nextEvidenceAt: () => null, tickCheckpoints: vi.fn(), registerCheckpointExpiration: vi.fn() } as never); + vi.mocked(nativeHarnessBackupExpirationService).mockReturnValueOnce({ + backupScopes: vi.fn(async () => ({ names: [], complete: true })), reconcileBackup: vi.fn(), expireBackup: vi.fn(), + } as never); vi.mocked(nativeHarnessBackupExpirationService).mockReturnValueOnce({ tick: vi.fn(() => pending.promise), noteEvidence: vi.fn(), nextEvidenceAt: () => null, tickCheckpoints: vi.fn(), registerCheckpointExpiration: vi.fn() } as never); try { diff --git a/server/src/services/native-runtime/native-harness-backup-expiration-work.ts b/server/src/services/native-runtime/native-harness-backup-expiration-work.ts index 93c4f08e340..f0dafe195e0 100644 --- a/server/src/services/native-runtime/native-harness-backup-expiration-work.ts +++ b/server/src/services/native-runtime/native-harness-backup-expiration-work.ts @@ -15,6 +15,26 @@ export function registerNativeHarnessBackupExpirationWork( ) { const service = nativeHarnessBackupExpirationService(db, { ...options, duty: "backups" }); const cursors = new Map(); + const roots = new Map>(); + const owners = new Map(); + function remember(id: string, companyId?: string) { + const previous = owners.get(id); + if (previous && previous !== companyId) { + const companyRoots = roots.get(previous)!; + companyRoots.delete(id); + if (!companyRoots.size) roots.delete(previous); + owners.delete(id); + } + if (!companyId) return; + let companyRoots = roots.get(companyId); + if (!companyRoots) { companyRoots = new Set(); roots.set(companyId, companyRoots); } + companyRoots.add(id); owners.set(id, companyId); + } + async function reconcile(id: string): Promise { + const source = await service.reconcileBackup(id); + remember(id, source?.companyId); + return source && source.dueAt !== null ? { ...source, dueAt: source.dueAt } : null; + } const worker = registerScopedDeadlineWork(coordinator, { queue: DELIVERY_QUEUES.nativeHarnessBackupExpiration, batchSize: options.maxRemovals ?? 25, decodeScope: (hint: string): NativeHarnessBackupExpirationScope | undefined => hint === "*" ? undefined : JSON.parse(hint), @@ -22,12 +42,16 @@ export function registerNativeHarnessBackupExpirationWork( && (!scope.scopeDigest || entry.id === scope.scopeDigest), load: async (scope, signal) => { const key = scope ? JSON.stringify(scope) : "*"; + const limit = options.maxEntries ?? 100; + const indexed = scope ? [...roots.get(scope.companyId) ?? []] + .filter(id => !cursors.get(key) || id > cursors.get(key)!).sort() : null; const page = scope?.scopeDigest ? { names: [scope.scopeDigest], complete: true } - : await service.backupScopes(cursors.get(key), options.maxEntries ?? 100); + : indexed ? { names: indexed.slice(0, limit), complete: indexed.length <= limit } + : await service.backupScopes(cursors.get(key), limit); const rows: Entry[] = []; for (const name of page.names) { if (signal?.aborted || !canRun()) throw new Error("Backup expiration discovery paused"); - const entry = await service.inspectBackup(name); + const entry = await reconcile(name); if (entry && (!scope || entry.companyId === scope.companyId)) rows.push(entry); } if (page.complete) cursors.delete(key); else cursors.set(key, page.names.at(-1)!); @@ -35,19 +59,22 @@ export function registerNativeHarnessBackupExpirationWork( }, check: async (entry, signal) => { if (signal?.aborted || !canRun()) throw new Error("Backup expiration paused"); - const current = await service.inspectBackup(entry.id); + const current = await reconcile(entry.id); if (!current || current.companyId !== entry.companyId) return null; if (current.dueAt > Date.now()) return current; const result = await service.expireBackup(entry.id); if (result.candidates || result.failed) logger.info({ ...result, dryRun: options.dryRun ?? false }, "native harness backup expiration"); if (options.dryRun) return null; - const remaining = await service.inspectBackup(entry.id); + const remaining = await reconcile(entry.id); if (!remaining) return null; if (result.skipped.session_active) throw new Error("Backup session is active"); return remaining; }, }, canRun); - const unsubscribe = onNativeHarnessBackupChange(db, scope => worker.invalidate(JSON.stringify(scope))); + const unsubscribe = onNativeHarnessBackupChange(db, scope => { + if (scope.scopeDigest) remember(scope.scopeDigest, scope.companyId); + worker.invalidate(JSON.stringify(scope)); + }); const release = onRetainedNativeSessionCleanupRelease(db, scope => worker.invalidate(JSON.stringify({ companyId: scope.companyId }))); return { ...worker, async stop() { unsubscribe(); release(); await worker.stop(); } }; } diff --git a/server/src/services/native-runtime/native-harness-backup-expiration.ts b/server/src/services/native-runtime/native-harness-backup-expiration.ts index 09ddea9e5d2..ad9ac18cfb2 100644 --- a/server/src/services/native-runtime/native-harness-backup-expiration.ts +++ b/server/src/services/native-runtime/native-harness-backup-expiration.ts @@ -739,30 +739,29 @@ export function nativeHarnessBackupExpirationService( }); } - async function inspectBackup(scopeDigest: string) { + async function reconcileBackup(scopeDigest: string) { const base = path.resolve(defaults.stateBase ?? nativeHarnessStateBase()); const root = path.join(base, scopeDigest); if (!SCOPE_PATTERN.test(scopeDigest) || !(await isRealDirectory(root))) return null; const backupRoot = path.join(root, "failover-backups"); if (!(await isRealDirectory(backupRoot))) return null; - const names = await fs.readdir(backupRoot); + if (!defaults.dryRun) await settleLeftovers(backupRoot); const current = path.join(backupRoot, "current"); const stat = await fs.lstat(current).catch(missingPath); - const leftovers = names.some(name => [RETIRING_PREFIX, EXPIRED_PREFIX].some(prefix => - name.startsWith(prefix) && UUID_PATTERN.test(name.slice(prefix.length)))); - if ((!stat?.isDirectory() || stat.isSymbolicLink()) && !leftovers) return null; - if (defaults.dryRun && !stat?.isDirectory()) return null; + if (!stat?.isDirectory() || stat.isSymbolicLink()) return null; const identity = await readSessionIdentity(root); if (!identity) return null; const [owner] = await db.select({ companyId: heartbeatRuns.companyId, agentId: heartbeatRuns.agentId }) .from(heartbeatRuns).where(eq(heartbeatRuns.id, identity.runId)); if (!owner) return null; - if (leftovers && !defaults.dryRun) return { id: scopeDigest, ...owner, dueAt: 0 }; - if (await fs.lstat(path.join(root, "cleanup-activation.json")).then(() => true, () => false)) return null; + const source = { id: scopeDigest, ...owner }; + if (await fs.lstat(path.join(root, "cleanup-activation.json")).then(() => true, () => false)) { + return { ...source, dueAt: null }; + } const blocker = await db.transaction(tx => sessionRetentionBlocker(tx, { scopeDigest, identity, closedBefore: new Date(8_000_000_000_000_000), })); - if ("reason" in blocker) return null; + if ("reason" in blocker) return { ...source, dueAt: null }; const retentionMs = Math.max(1, defaults.closedForDays ?? NATIVE_HARNESS_BACKUP_EXPIRATION_DEFAULT_DAYS) * DAY_MS; return { id: scopeDigest, ...owner, dueAt: Math.max(stat!.mtime.getTime(), blocker.closedAt) + retentionMs + 1 }; } @@ -789,5 +788,5 @@ export function nativeHarnessBackupExpirationService( } return { tick, noteEvidence, nextEvidenceAt, tickCheckpoints, registerCheckpointExpiration, - inspectBackup, backupScopes, expireBackup }; + reconcileBackup, backupScopes, expireBackup }; } From 25a8a4392c807d4b5eb19cbf713b5eda99c017d3 Mon Sep 17 00:00:00 2001 From: Dotta Date: Fri, 30 Oct 2026 16:35:43 -0500 Subject: [PATCH 08/74] fix(external-objects): cover budget resume and scoped deletions Co-Authored-By: Paperclip --- .../2026-10-30-external-objects-sleep.md | 4 +- .../external-object-refresh-work.test.ts | 63 ++++++++++++++++++- .../external-objects-service.test.ts | 4 +- server/src/services/budgets.ts | 3 + server/src/services/external-objects.ts | 18 +++--- 5 files changed, 79 insertions(+), 13 deletions(-) diff --git a/doc/plans/2026-10-30-external-objects-sleep.md b/doc/plans/2026-10-30-external-objects-sleep.md index 639bbaea343..ab566999917 100644 --- a/doc/plans/2026-10-30-external-objects-sleep.md +++ b/doc/plans/2026-10-30-external-objects-sleep.md @@ -47,10 +47,10 @@ and fences automatic result persistence. It does not modify that draft PR. | Mutation | Owner and activation | | --- | --- | | Issue title/description, comment and document references | Existing sync methods register company intent before mention replacement and object upsert. Routes, runtime tools, adapters and plugin paths reuse these methods. | -| Comment/document disappearance | Sync registers reconstruction before deleting obsolete mentions. | +| Comment/document disappearance | Sync derives the retained mention owners and registers only those companies before deleting obsolete mentions. | | Last mention removal and reattachment | Mention replacement invalidates company deadlines. Retained objects stay available for reattachment. | | Issue deletion | Issue service registers its company before cascading mention deletion. | -| Company status/archive/deletion | Company services register company reconstruction before their status or deletion writes. | +| Company status/archive/deletion, including budget pause/resume | Company and budget services register company reconstruction before status or deletion writes. | | Feature enable/disable | Instance settings registers global reconstruction before the experimental-settings write. | | Plugin manifest/status/uninstall | Plugin registry registers reconstruction before lifecycle writes. Installed plugins acquire support through the ordinary ready transition. | | Plugin company configuration and enabled setting | Registry writes register only the affected company. There is no plugin-readiness poll. | diff --git a/server/src/__tests__/external-object-refresh-work.test.ts b/server/src/__tests__/external-object-refresh-work.test.ts index 8d1f5cdc00c..f429c31396b 100644 --- a/server/src/__tests__/external-object-refresh-work.test.ts +++ b/server/src/__tests__/external-object-refresh-work.test.ts @@ -2,10 +2,12 @@ import { randomUUID } from "node:crypto"; import { setTimeout as delay } from "node:timers/promises"; import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest"; import { eq, sql } from "drizzle-orm"; -import { activityLog, companies, createDb, externalObjects, issues, plugins } from "@paperclipai/db"; +import { activityLog, approvals, applyPendingMigrations, budgetIncidents, budgetPolicies, companies, costEvents, createDb, externalObjects, issues, issueComments, plugins } from "@paperclipai/db"; import { externalObjectService, type ExternalObjectResolveResult } from "../services/external-objects.js"; import type { PaperclipPluginManifestV1 } from "@paperclipai/shared"; import type { PluginWorkerManager } from "../services/plugin-worker-manager.js"; +import { documentService } from "../services/documents.js"; +import { budgetService } from "../services/budgets.js"; import { pluginRegistryService } from "../services/plugin-registry.js"; import { companyService } from "../services/companies.js"; import { instanceSettingsService } from "../services/instance-settings.js"; @@ -34,14 +36,19 @@ const active: ExternalObjectResolveResult = { ok: true, snapshot: { const errors = vi.fn(); const companyIds: string[] = []; beforeAll(async () => { + if (url) await applyPendingMigrations(url); if (!url) temp = await startEmbeddedPostgresTestDatabase("external-refresh-work-"); db = createDb(url ?? temp!.connectionString, { maxConnections: 6 }); await db.execute(sql`truncate table ${companies} cascade`); - }); + }, 90_000); afterEach(async () => { await coordinator?.stop(); coordinator = undefined; vi.useRealTimers(); vi.restoreAllMocks(); for (const id of companyIds.splice(0)) { + await db.delete(budgetIncidents).where(eq(budgetIncidents.companyId, id)); + await db.delete(budgetPolicies).where(eq(budgetPolicies.companyId, id)); + await db.delete(approvals).where(eq(approvals.companyId, id)); + await db.delete(costEvents).where(eq(costEvents.companyId, id)); await db.delete(activityLog).where(eq(activityLog.companyId, id)); await db.delete(issues).where(eq(issues.companyId, id)); await db.delete(externalObjects).where(eq(externalObjects.companyId, id)); @@ -256,6 +263,58 @@ const active: ExternalObjectResolveResult = { ok: true, snapshot: { expect(calls).toHaveBeenCalledOnce(); }); + it("reactivates refresh after ordinary budget pause and budget raise", async () => { + clock(); const svc = service(); provider.mockResolvedValue(active); await start(svc); + const f = await fixture(); await svc.syncIssue(f.issueId); + await settle(async () => (await row(f.companyId))?.lastResolvedAt !== null); + await db.insert(costEvents).values({ companyId: f.companyId, usageKind: "decision", provider: "fixture", + biller: "fixture", billingType: "metered_api", model: "simulated", costCents: 200, occurredAt: new Date() }); + const budgets = budgetService(db); + await budgets.upsertPolicy(f.companyId, { scopeType: "company", scopeId: f.companyId, + amount: 100, hardStopEnabled: true }, null); + expect((await db.select().from(companies).where(eq(companies.id, f.companyId)))[0]!.status).toBe("paused"); + await quiet(); + const reads = vi.spyOn(db, "select"); provider.mockClear(); + await vi.advanceTimersByTimeAsync(900_000); expect(reads).not.toHaveBeenCalled(); expect(provider).not.toHaveBeenCalled(); + provider.mockResolvedValue(terminal); + await budgets.upsertPolicy(f.companyId, { scopeType: "company", scopeId: f.companyId, amount: 300 }, null); + await settle(async () => (await row(f.companyId)).isTerminal); await quiet(); + expect(provider).toHaveBeenCalledOnce(); + }); + + it.each(["comment", "document"] as const)("keeps hard-deleted %s activation scoped to its retained mention owner", async kind => { + clock(); const svc = service(); provider.mockResolvedValue(active); await start(svc); + const a = await fixture(); const b = await fixture(); await svc.syncIssue(b.issueId); + let id: string; + if (kind === "comment") { + const [comment] = await db.insert(issueComments).values({ companyId: a.companyId, issueId: a.issueId, + body: `https://example.com/${a.companyId}` }).returning(); + id = comment!.id; await svc.syncComment(id); + } else { + const result = await documentService(db).upsertIssueDocument({ issueId: a.issueId, + key: "notes", format: "markdown", body: `https://example.com/${a.companyId}` }); + id = result.document.id; await svc.syncDocument(id); + } + await settle(async () => (await row(a.companyId))?.lastResolvedAt !== null && (await row(b.companyId))?.lastResolvedAt !== null); + await settle(() => coordinator!.nextWakeAt() !== null); + const reads: Array<{ query: string; params: unknown[] }> = []; + const debug = db.$client.options.debug; + db.$client.options.debug = (_connection, query, params) => { + if (/^select/i.test(query) && query.includes('from "external_objects"') && query.includes('inner join "companies"')) + reads.push({ query, params: [...params] }); + }; + try { + if (kind === "comment") { await db.delete(issueComments).where(eq(issueComments.id, id)); await svc.syncComment(id); } + else { await documentService(db).deleteIssueDocument(a.issueId, "notes"); await svc.syncDocument(id); } + await settle(() => reads.length > 0); + for (const read of reads) { + expect(read.query).toContain('"external_objects"."company_id" ='); + expect(read.params).toContain(a.companyId); expect(read.params).not.toContain(b.companyId); + } + } finally { db.$client.options.debug = debug; } + expect(provider).toHaveBeenCalledTimes(2); + }); + it("keeps warm standby startup SQL-free and resumes a first committed producer", async () => { clock(); admitted = false; reconcile = false; const svc = service(); const selects = vi.spyOn(db, "select"); diff --git a/server/src/__tests__/external-objects-service.test.ts b/server/src/__tests__/external-objects-service.test.ts index e50e298b466..bc268d0ba6a 100644 --- a/server/src/__tests__/external-objects-service.test.ts +++ b/server/src/__tests__/external-objects-service.test.ts @@ -3,6 +3,7 @@ import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest import { eq } from "drizzle-orm"; import { companies, + applyPendingMigrations, createDb, activityLog, externalObjectMentions, @@ -711,9 +712,10 @@ describeEmbeddedPostgres("externalObjectService", () => { let tempDb: Awaited> | null = null; beforeAll(async () => { + if (externalDatabaseUrl) await applyPendingMigrations(externalDatabaseUrl); if (!externalDatabaseUrl) tempDb = await startEmbeddedPostgresTestDatabase("paperclip-external-objects-"); db = createDb(externalDatabaseUrl ?? tempDb!.connectionString, { maxConnections: 6 }); - }, 20_000); + }, 90_000); afterEach(async () => { await db.delete(activityLog); diff --git a/server/src/services/budgets.ts b/server/src/services/budgets.ts index 6777124ee90..88c3255da4e 100644 --- a/server/src/services/budgets.ts +++ b/server/src/services/budgets.ts @@ -1,3 +1,4 @@ +import { notifyExternalObjectRefresh } from "./external-object-refresh-notifications.js"; import { notifyApprovalWork } from "./approval-work-notifications.js"; import { notifyCompanyRetentionWork } from "./retention-work-notifications.js"; import { addCents, compareCents, normalizeCents } from "@paperclipai/shared"; @@ -301,6 +302,7 @@ export function budgetServiceInTransaction(db: Db, publications: ActivityPublica const scope = await resolveScopeRecord(db, "company", policy.scopeId); if (scope.paused) return; await notifyCompanyRetentionWork(db, policy.companyId); + await notifyExternalObjectRefresh(db, policy.companyId); await notifyDeliveryWork(db, DELIVERY_QUEUES.agentTimers); await notifyDeliveryWork(db, DELIVERY_QUEUES.routineSchedules); await notifyDeliveryWork(db, DELIVERY_QUEUES.issueMonitors); @@ -361,6 +363,7 @@ export function budgetServiceInTransaction(db: Db, publications: ActivityPublica const scope = await resolveScopeRecord(db, "company", policy.scopeId); if (scope.pauseReason !== "budget") return; await notifyCompanyRetentionWork(db, policy.companyId); + await notifyExternalObjectRefresh(db, policy.companyId); await notifyDeliveryWork(db, DELIVERY_QUEUES.agentTimers); await notifyDeliveryWork(db, DELIVERY_QUEUES.routineSchedules); await notifyDeliveryWork(db, DELIVERY_QUEUES.issueMonitors); diff --git a/server/src/services/external-objects.ts b/server/src/services/external-objects.ts index c49df7ff076..d524b473e1a 100644 --- a/server/src/services/external-objects.ts +++ b/server/src/services/external-objects.ts @@ -571,6 +571,14 @@ export function externalObjectService( return dbOrTx === db ? db.transaction(runSync) : runSync(dbOrTx); } + async function deleteMissingSourceMentions(kind: "comment" | "document", recordId: string, tx: any) { + const source = and(eq(externalObjectMentions.sourceKind, kind), eq(externalObjectMentions.sourceRecordId, recordId)); + const owners: Array<{ companyId: string }> = await tx.selectDistinct({ companyId: externalObjectMentions.companyId }) + .from(externalObjectMentions).where(source); + for (const owner of owners) await notifyExternalObjectRefresh(tx, owner.companyId); + await tx.delete(externalObjectMentions).where(source); + } + async function syncComment(commentId: string, dbOrTx: any = db) { const runSync = async (tx: any) => { if (!(await isEnabled())) return; @@ -585,10 +593,7 @@ export function externalObjectService( .where(eq(issueComments.id, commentId)) .then((rows: Array<{ id: string; companyId: string; issueId: string; body: string }>) => rows[0] ?? null); if (!comment) { - await notifyExternalObjectRefresh(tx); - await tx - .delete(externalObjectMentions) - .where(and(eq(externalObjectMentions.sourceKind, "comment"), eq(externalObjectMentions.sourceRecordId, commentId))); + await deleteMissingSourceMentions("comment", commentId, tx); return; } await replaceSourceMentions({ @@ -620,10 +625,7 @@ export function externalObjectService( .where(eq(documents.id, documentId)) .then((rows: Array<{ documentId: string; companyId: string; issueId: string; key: string; body: string }>) => rows[0] ?? null); if (!document) { - await notifyExternalObjectRefresh(tx); - await tx - .delete(externalObjectMentions) - .where(and(eq(externalObjectMentions.sourceKind, "document"), eq(externalObjectMentions.sourceRecordId, documentId))); + await deleteMissingSourceMentions("document", documentId, tx); return; } await replaceSourceMentions({ From 0f3017c96de9cc2c9adf907ae2d7c9145ba4aac0 Mon Sep 17 00:00:00 2001 From: Dotta Date: Fri, 30 Oct 2026 16:43:45 -0500 Subject: [PATCH 09/74] perf(server): archive attention from source deadlines Co-Authored-By: Paperclip --- .../2026-10-30-attention-retention-sleep.md | 48 +++ .../attention-retention-work.test.ts | 339 ++++++++++++++++++ .../src/__tests__/attention-service.test.ts | 2 +- server/src/index.ts | 25 +- .../__tests__/lifecycle.test.ts | 2 +- .../agent-lifecycle/adapters/records.ts | 13 +- server/src/services/accounting-integrity.ts | 3 + server/src/services/agent-configuration.ts | 2 + server/src/services/agent-deletion.ts | 2 + server/src/services/agent-hiring.ts | 2 + server/src/services/agent-lifecycle.ts | 2 + server/src/services/agent-project-access.ts | 2 + server/src/services/agents.ts | 6 +- server/src/services/attention-retention.ts | 204 +++++++++++ .../services/attention-source-eligibility.ts | 108 ++++++ server/src/services/attention.ts | 110 +----- server/src/services/budgets.ts | 6 +- server/src/services/company-portability.ts | 6 +- server/src/services/decision-retention.ts | 43 ++- .../services/decision-work-notifications.ts | 5 +- .../services/delivery-work-notifications.ts | 1 + .../execution-control-notifications.ts | 3 + .../services/grok-bot/workspace-inspection.ts | 2 + server/src/services/heartbeat-run-events.ts | 5 +- server/src/services/heartbeat/agent-status.ts | 3 + server/src/services/heartbeat/retries.ts | 2 + server/src/services/issue-recovery-actions.ts | 2 + .../issue-review-state-notifications.ts | 2 + server/src/services/issue-title.ts | 2 + server/src/services/issue-visibility.ts | 8 +- server/src/services/issues.ts | 9 + .../native-finalization-notifications.ts | 2 + .../native-provider-capacity-retry.ts | 2 + .../native-review-participant.ts | 2 + .../native-workspace-reconstruction.ts | 2 + .../paperclip-runner-tool-authority.ts | 2 + .../superseded-external-review-requests.ts | 2 + ...ative-status-reassessment-notifications.ts | 4 +- server/src/services/plugin-managed-agents.ts | 2 + .../services/retention-work-notifications.ts | 9 +- .../services/slack-conversation-lifecycle.ts | 2 + .../src/services/slack-conversation-state.ts | 2 + 42 files changed, 845 insertions(+), 155 deletions(-) create mode 100644 doc/plans/2026-10-30-attention-retention-sleep.md create mode 100644 server/src/__tests__/attention-retention-work.test.ts create mode 100644 server/src/services/attention-retention.ts create mode 100644 server/src/services/attention-source-eligibility.ts diff --git a/doc/plans/2026-10-30-attention-retention-sleep.md b/doc/plans/2026-10-30-attention-retention-sleep.md new file mode 100644 index 00000000000..c0e4e168bd0 --- /dev/null +++ b/doc/plans/2026-10-30-attention-retention-sleep.md @@ -0,0 +1,48 @@ +# Archive attention sources without repeated feed scans + +Attention retention now reconstructs source deadlines at startup and after committed source changes. It no longer rebuilds every active company's full feed on each heartbeat tick. The archive age remains 90 days. Keep and manual archive state remain authoritative, and automatic archive still commits its notification outbox atomically. + +This document records source ownership, producer coverage, and the evidence needed to review the replacement. It makes no claim about host suspension or external wake alarms. + +## Existing source rows own eligibility and deadlines + +`attention-retention.ts` selects at most 100 unfinished source candidates per reconstruction page. It excludes inactive companies and kept or archived retention states. The scoped coordinator retains eligible deadlines, retries concrete failures, and cancels entries that leave attention. It does not schedule another scan after settlement unless a producer changes a relevant source. + +| Source | Eligible durable state | Activity owner | +| --- | --- | --- | +| Decision | Open decision with a visible execution issue | Decision updated time | +| Interaction | Pending board-visible interaction; newest ordinary confirmation or independent quality gate | Interaction updated time | +| Recovery action | Active or escalated action owned by a user or board, with a visible source issue | Recovery updated time | +| Blocker attention | Board unblock descriptor or non-live terminal blocker selected by the issue owner | Blocked transition or terminal issue updated time | +| Review | Human participant, user assignee, pending approval, or stalled review selected by the issue owner | Issue updated time | +| Failed run | Failed or timed-out run with an exhaustion event, eligible agent and issue, and no newer matching run | Finished time, falling back to updated or created time | +| Budget alert | Open hard incident, or soft incident meeting the existing 85 percent rounded threshold | Incident updated time | +| Agent error | Agent in error status | Agent updated time | + +The feed and worker share confirmation collapse, newer-run probes, issue visibility, review eligibility, and terminal-blocker predicates. Issue graph and agent invokability logic remain with their existing owners. Blocker entries retain both the root used to reconstruct the graph and the terminal subject that owns retention state. PostgreSQL deadlines round submillisecond timestamps upward. The worker locks the source and terminal subject before archival. + +`productivity_review` generates no feed items and owes no retention work. Queue retention-day overrides control shelf presentation, which is computed on reads. They do not change the fixed automatic archive age. Approval retention, join-request retention, and archive notification delivery keep their existing workers. + +## Existing mutation boundaries reactivate work + +Committed hints identify a company, source kind, and source ID where those values are known. Retention's own writes use exact source hints; blocker retention changes invalidate the company/source because multiple blocked roots can project one terminal subject. Reconstruction pages and due batches remain bounded. + +Producer coverage includes decision creation and changes; interaction creation, resolution, expiry and supersession; issue creation, updates, blocker relations, checkout, adoption and release; agent hire, configuration, lifecycle, error clearing and deletion; budget incidents and company pause/resume; exhausted-run events and retry-run creation; native status, review, finalization and reconstruction; external conversation transitions; and imported blocker relations. Existing service transactions retain outer commit ownership. Savepoint release does not publish work before outer commit. + +The writer contract follows the existing delivery workers: serving processes use the transaction-aware service boundaries, and each serving process registers its workers. Independent raw SQL writers must supply equivalent ingress or restart reconstruction. Restores reconstruct work at startup. The change adds no permanent listener, independent timer, table, or Cloud wake mechanism. Existing sleep-safety gates remain in force. + +## Real PostgreSQL evidence proves quiet and reactivation + +The new acceptance suite exercises all eight sources, including terminal blocker identity and date, newest confirmations, independent quality gates, unhealthy reporting chains, conversation visibility, maintained reviews and pending approvals, newer-run suppression, ordinary task and interaction producers, ordinary exhaustion-event ingress, and ordinary error clearing. + +It also checks keep/revive, moved and cancelled deadlines, company activation and isolation, retained terminal history, restart, nested outer commit and rollback, first writes during drain, retry across drain, atomic archive/outbox rollback, and notifications during reconstruction. A 225-item backlog uses three startup pages and source-specific self-invalidations. SQL is counted at the PostgreSQL driver boundary. + +The old aggregate path issued 12 statements per tick in each of three configured-idle samples. The replacement issued zero statements during 300 seconds after settlement. One transaction-status read after an injected rollback is classified as uncertainty reconciliation; the following 300-second drain window must remain quiet. + +The focused suite supports `PAPERCLIP_ATTENTION_RETENTION_TEST_DATABASE_URL`, validates its dedicated `_test` database name, applies pending migrations, and prewarms a six-connection pool before advancing fake time. Providers are not involved in retention selection; the existing archive outbox delivery tests verify the separate controlled notification boundary. Fake time proves scheduling and SQL quiet, not database or host suspension. + +## Review and operational limits remain explicit + +Local embedded PostgreSQL could not start on the test host because system semaphores were exhausted. The accepted runs use isolated PostgreSQL 17 databases. A mixed consumer run also exposed fixture contamination from a lifecycle suite that retains native finalization rows; isolated reruns are required and preserve the failed run in the review evidence. Broad Linux CI remains the required full-suite gate. + +Review the source parity fixtures and producer hints together. The intended outcome is unchanged archive state and notifications with no recurring retention SQL when the sources remain unchanged. No merge or deployment is part of this change. diff --git a/server/src/__tests__/attention-retention-work.test.ts b/server/src/__tests__/attention-retention-work.test.ts new file mode 100644 index 00000000000..22a966d5fef --- /dev/null +++ b/server/src/__tests__/attention-retention-work.test.ts @@ -0,0 +1,339 @@ +import { randomUUID } from "node:crypto"; +import { and, eq, sql } from "drizzle-orm"; +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; +import { authUsers, companyMemberships, agents, approvals, budgetIncidents, budgetPolicies, companies, createDb, applyPendingMigrations, decisionArchiveNotificationOutbox, + decisionRetention, decisions, heartbeatRunEvents, heartbeatRuns, issueApprovals, issueRecoveryActions, + issueRelations, issueThreadInteractions, issues, type Db } from "@paperclipai/db"; +import { attentionRetentionService, registerAttentionRetentionWork } from "../services/attention-retention.js"; +import { attentionService } from "../services/attention.js"; +import { companyService } from "../services/companies.js"; +import { createDeliveryWorkCoordinator } from "../services/delivery-work-coordinator.js"; +import { decisionRetentionService, DEFAULT_DECISION_ARCHIVE_DAYS } from "../services/decision-retention.js"; +import { appendHeartbeatRunEvent } from "../services/heartbeat-run-events.js"; +import { issueThreadInteractionService } from "../services/issue-thread-interactions.js"; +import { createAgentLifecycle } from "../services/agent-lifecycle.js"; +import { issueService } from "../services/issues.js"; +import { notifyAttentionRetentionWork } from "../services/retention-work-notifications.js"; +import { idleWorkSnapshot } from "../services/task-admission.js"; +import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; + +import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js"; + +const externalUrl = resolveExternalTestDatabaseUrl("PAPERCLIP_ATTENTION_RETENTION_TEST_DATABASE_URL"); +const support = externalUrl ? { supported: true } : await getEmbeddedPostgresTestSupport(); +const delay = DEFAULT_DECISION_ARCHIVE_DAYS * 86_400_000; +(support.supported ? describe : describe.skip)("remaining attention retention", () => { + let temporary: Awaited> | undefined; + let db: ReturnType; + let companyId: string, agentId: string, draining = false; + let coordinator: ReturnType | undefined; + const errors: unknown[] = []; + const statements: string[] = []; + beforeAll(async () => { + if (externalUrl) await applyPendingMigrations(externalUrl); + else temporary = await startEmbeddedPostgresTestDatabase("attention-retention-"); + db = createDb(externalUrl ?? temporary!.connectionString, { maxConnections: 6, idleTimeoutSeconds: 0, maxLifetimeSeconds: 0 }); + db.$client.options.onnotice = () => {}; + db.$client.options.debug = (_connection, query) => statements.push(query); + }, 30_000); + beforeEach(async () => { + companyId = randomUUID(); agentId = randomUUID(); draining = false; errors.length = 0; + await db.insert(companies).values({ id: companyId, name: "Retention", issuePrefix: `A${companyId.slice(0, 6)}` }); + await db.insert(authUsers).values({ id: "board", name: "Board", email: "board@retention.test", createdAt: new Date(), updatedAt: new Date() }).onConflictDoNothing(); + await db.insert(companyMemberships).values({ companyId, principalType: "user", principalId: "board", membershipRole: "owner", status: "active" }); + await db.insert(agents).values({ id: agentId, companyId, name: "Origin", role: "engineer", status: "active" }); + await Promise.all(Array.from({ length: 6 }, () => db.execute(sql`select pg_sleep(0.01)`))); + vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout", "setInterval", "clearInterval"], shouldAdvanceTime: true }); + vi.setSystemTime(new Date("2026-11-01T12:00:00Z")); + }); + afterEach(async () => { + await coordinator?.stop(); coordinator = undefined; + vi.useRealTimers(); vi.restoreAllMocks(); + await db.transaction(async tx => { + await tx.execute(sql`set local client_min_messages = warning`); + await tx.execute(sql`drop trigger if exists attention_retention_fail on decision_retention`); + await tx.execute(sql`drop function if exists attention_retention_fail()`); + await tx.execute(sql`truncate companies cascade`); + }); + expect(errors).toEqual([]); + }); + afterAll(async () => { await db?.$client.end({ timeout: 1 }); await temporary?.cleanup(); }); + const old = () => new Date(Date.now() - delay - 600_000); + async function insertIssue(status = "todo", values: Partial = {}, database: Db = db) { + const [row] = await database.insert(issues).values({ companyId, title: randomUUID(), status, + createdByAgentId: agentId, createdAt: old(), updatedAt: old(), ...values }).returning(); + return row; + } + async function change(write: (tx: Db) => PromiseLike, scope = { companyId }) { + return db.transaction(async tx => { await notifyAttentionRetentionWork(tx, scope); return write(tx as unknown as Db); }); + } + const state = (kind: string, id: string) => db.select().from(decisionRetention).where(and( + eq(decisionRetention.companyId, companyId), eq(decisionRetention.sourceKind, kind), eq(decisionRetention.sourceId, id))) + .then(rows => rows[0] ?? null); + async function settle() { + await vi.waitFor(async () => { await vi.advanceTimersByTimeAsync(1000); expect(idleWorkSnapshot().active).toBe(0); }, { timeout: 15_000 }); + } + async function quiet() { + await settle(); expect(coordinator!.nextWakeAt()).toBeNull(); + const select = vi.spyOn(db, "select"), execute = vi.spyOn(db, "execute"), transaction = vi.spyOn(db, "transaction"); + statements.length = 0; + await vi.advanceTimersByTimeAsync(300_000); + expect(statements).toEqual([]); + expect(select).not.toHaveBeenCalled(); expect(execute).not.toHaveBeenCalled(); expect(transaction).not.toHaveBeenCalled(); + select.mockRestore(); execute.mockRestore(); transaction.mockRestore(); + } + async function start(service = attentionRetentionService(db)) { + coordinator = createDeliveryWorkCoordinator({ owner: db, canRun: () => !draining, canReconcile: () => true, onError: error => errors.push(error) }); + await registerAttentionRetentionWork(coordinator, service, () => !draining).ready; + return service; + } + async function archived(kind: string, id: string) { + await vi.waitFor(async () => { await vi.advanceTimersByTimeAsync(1000); expect((await state(kind, id))?.archivedAt).toBeInstanceOf(Date); }, { timeout: 15_000 }); + } + async function loadAll(service = attentionRetentionService(db)) { + const entries = []; + for (;;) { const page = await service.load({ companyId }); entries.push(...page.rows ?? []); if (page.complete) return entries; } + } + const mutation = (sourceKind: "review", sourceId: string) => ({ companyId, sourceKind, sourceId, + authActor: { type: "board" as const, source: "local_implicit" as const, userId: "board", companyIds: [companyId], isInstanceAdmin: true }, + actor: { actorType: "user" as const, actorId: "board", userId: "board", agentId: null, runId: null, agentApiKeyId: null, responsibleUserId: "board" } }); + + it("records the aggregate baseline SQL across old ticks and eliminates it after settlement", async () => { + await insertIssue("done"); + const ticks: number[] = []; + for (let tick = 0; tick < 3; tick++) { + statements.length = 0; + const feed = await attentionService(db).list(companyId, { includeDismissed: true, excludeSources: ["approval", "join_request"], all: true, allowUnscopedAll: true }); + await decisionRetentionService(db).autoArchive({ companyId, items: feed.items }); + ticks.push(statements.length); + await vi.advanceTimersByTimeAsync(30_000); + } + expect(ticks.every(count => count > 0)).toBe(true); + process.stdout.write(JSON.stringify({ baselineStatementsPer30sTick: ticks, treatmentIdle300s: 0 }) + "\n"); + await start(); await quiet(); + }); + + it("has no recurring SQL when empty or when terminal source history remains", async () => { + await insertIssue("done"); await insertIssue("cancelled"); + await start(); await quiet(); + }); + + it("wakes from the ordinary task producer, waits until due, archives with an outbox, and settles", async () => { + await start(); await quiet(); + const row = await issueService(db).create(companyId, { title: "Review a delivered result", status: "in_review", assigneeUserId: "board", createdByAgentId: agentId }); + await settle(); + const [saved] = await db.select({ at: sql`ceil(extract(epoch from ${issues.updatedAt}) * 1000)`.mapWith(Number) }).from(issues).where(eq(issues.id, row.id)); + const due = saved.at + delay; + expect(coordinator!.nextWakeAt()).toBe(due); expect(await state("review", row.id)).toBeNull(); + const select = vi.spyOn(db, "select"), execute = vi.spyOn(db, "execute"); + await vi.advanceTimersByTimeAsync(due - Date.now() - 1000); + expect(select).not.toHaveBeenCalled(); expect(execute).not.toHaveBeenCalled(); select.mockRestore(); execute.mockRestore(); + await vi.advanceTimersByTimeAsync(2000); await archived("review", row.id); await quiet(); + const outbox = await db.select().from(decisionArchiveNotificationOutbox); + expect(outbox).toHaveLength(1); expect(outbox[0]).toMatchObject({ companyId, sourceKind: "review", sourceId: row.id, originAgentId: agentId, originIssueId: row.id }); + expect((await db.select().from(issues).where(eq(issues.id, row.id)))[0].status).toBe("in_review"); + await coordinator!.stop(); coordinator = undefined; await start(); await quiet(); + }); + + it("matches the existing feed across all eight remaining source kinds and exclusion variants", async () => { + const origin = await insertIssue(); + const review = await insertIssue("in_review", { assigneeUserId: "board" }); + const blocker = await insertIssue("blocked", { unblockDescriptor: { owner: "board", action: "Choose a dependency" }, blockedTransitionAt: old() }); + await insertIssue("in_review", { hiddenAt: old(), assigneeUserId: "board" }); + const hidden = await insertIssue("todo", { hiddenAt: old() }); + const [run] = await db.insert(heartbeatRuns).values({ companyId, agentId, status: "failed", contextSnapshot: { issueId: origin.id }, createdAt: old(), updatedAt: old(), finishedAt: old() }).returning(); + await db.insert(heartbeatRunEvents).values({ companyId, agentId, runId: run.id, seq: 1, eventType: "lifecycle", message: "Bounded retry exhausted: fixture" }); + await db.insert(decisions).values({ companyId, originAgentId: agentId, originIssueId: origin.id, originRunId: run.id, + title: "Decision", body: "Choose", options: [], signedSpec: "fixture", targetSnapshots: {}, expiresAt: new Date(Date.now() + 86_400_000), updatedAt: old() }); + await db.insert(issueThreadInteractions).values([ + { companyId, issueId: origin.id, kind: "ask_user_questions", payload: { questions: [] }, updatedAt: old() }, + { companyId, issueId: origin.id, kind: "ask_user_questions", payload: { questions: [] }, addresseeAgentId: agentId, updatedAt: old() }, + { companyId, issueId: origin.id, kind: "ask_user_questions", payload: { questions: [] }, addresseeUserId: "board", updatedAt: old() }, + { companyId, issueId: hidden.id, kind: "ask_user_questions", payload: { questions: [] }, updatedAt: old() }, + ]); + await db.insert(issueRecoveryActions).values({ companyId, sourceIssueId: origin.id, kind: "execution", ownerType: "board", cause: "fixture", fingerprint: "fixture", nextAction: "Choose", updatedAt: old() }); + const [policy] = await db.insert(budgetPolicies).values({ companyId, scopeType: "company", scopeId: companyId, windowKind: "calendar_month_utc", amount: 100 }).returning(); + await db.insert(budgetIncidents).values({ companyId, policyId: policy.id, scopeType: "company", scopeId: companyId, + metric: "billed_cents", windowKind: "calendar_month_utc", windowStart: old(), windowEnd: new Date(), thresholdType: "hard", amountLimit: 100, amountObserved: 100, updatedAt: old() }); + await db.insert(agents).values({ companyId, name: "Error", role: "engineer", status: "error", updatedAt: old() }); + const feed = await attentionService(db).list(companyId, { all: true, allowUnscopedAll: true, includeDismissed: true, excludeSources: ["approval", "join_request"] }); + const expected = [...new Set(feed.items.map(item => `${item.sourceKind}:${item.subject.id}`))].sort(); + const actual = [...new Set((await loadAll()).map(item => `${item.sourceKind}:${item.sourceId}`))].sort(); + expect(new Set(feed.items.map(item => item.sourceKind)).size).toBe(8); expect(actual).toEqual(expected); + expect(actual).toContain(`review:${review.id}`); expect(actual).toContain(`blocker_attention:${blocker.id}`); + await start(); for (const item of feed.items) await archived(item.sourceKind, item.subject.id); await quiet(); + }); + + it("keeps unseen sources, cancels their deadline, and reactivates when unkept or revived", async () => { + const row = await insertIssue("in_review", { assigneeUserId: "board" }); + await decisionRetentionService(db).setKeep({ ...mutation("review", row.id), keep: true }); + await start(); await quiet(); expect((await state("review", row.id))?.archivedAt).toBeNull(); + await decisionRetentionService(db).setKeep({ ...mutation("review", row.id), keep: false }); + await archived("review", row.id); await quiet(); + await decisionRetentionService(db).revive(mutation("review", row.id)); await archived("review", row.id); await quiet(); + }); + + it("moves deadlines on source activity and cancels when the source leaves attention", async () => { + const row = await insertIssue("in_review", { assigneeUserId: "board", updatedAt: new Date() }); + await start(); await settle(); const first = coordinator!.nextWakeAt()!; + await change(tx => tx.update(issues).set({ updatedAt: new Date(Date.now() - delay + 600_000) }).where(eq(issues.id, row.id))); + await settle(); expect(coordinator!.nextWakeAt()).toBeLessThan(first); + await issueService(db).update(row.id, { status: "todo" }); await quiet(); expect(await state("review", row.id)).toBeNull(); + }); + + it("preserves outer transaction ownership and first writes during drain", async () => { + await start(); await quiet(); draining = true; + let id: string | undefined; + await db.transaction(async outer => { + await outer.transaction(async nested => { + await notifyAttentionRetentionWork(nested, { companyId }); + id = (await insertIssue("in_review", { assigneeUserId: "board" }, nested as unknown as Db)).id; + }); + await vi.advanceTimersByTimeAsync(60_000); expect(await state("review", id!)).toBeNull(); + }); + await vi.advanceTimersByTimeAsync(120_000); expect(await state("review", id!)).toBeNull(); + draining = false; await vi.advanceTimersByTimeAsync(30_000); await archived("review", id!); await quiet(); + await expect(db.transaction(async outer => { + await outer.transaction(async nested => { await notifyAttentionRetentionWork(nested, { companyId }); await insertIssue("in_review", { assigneeUserId: "board" }, nested as unknown as Db); }); + throw new Error("rollback"); + })).rejects.toThrow("rollback"); + await quiet(); expect(await db.select().from(decisionRetention)).toHaveLength(1); + }); + + it("isolates companies, catches up at activation, and bounds retained-history pages", async () => { + const other = randomUUID(); await db.insert(companies).values({ id: other, name: "Paused", issuePrefix: "RP", status: "paused" }); + const row = await insertIssue("in_review", { companyId: other, assigneeUserId: "board" }); + await db.insert(issues).values(Array.from({ length: 1000 }, () => ({ companyId, title: randomUUID(), status: "done", updatedAt: old() }))); + const service = attentionRetentionService(db); const page = await service.load({ companyId }); + expect(page).toEqual({ rows: [], complete: true }); + await start(service); await quiet(); expect(await db.select().from(decisionRetention)).toHaveLength(0); + await companyService(db).update(other, { status: "active" }); + await vi.waitFor(async () => { await vi.advanceTimersByTimeAsync(1000); expect(await db.select().from(decisionRetention)).toHaveLength(1); }); + expect((await db.select().from(decisionRetention))[0]).toMatchObject({ companyId: other, sourceId: row.id }); await quiet(); + }); + it("preserves terminal blocker source identity and its own activity deadline", async () => { + const root = await insertIssue("blocked", { unblockDescriptor: { owner: "board", action: "Pick an owner" }, blockedTransitionAt: old() }); + const terminal = await insertIssue("backlog", { assigneeAgentId: agentId, updatedAt: new Date(old().getTime() + 60_000) }); + await db.insert(issueRelations).values({ companyId, issueId: terminal.id, relatedIssueId: root.id, type: "blocks" }); + const feed = await attentionService(db).list(companyId, { all: true, allowUnscopedAll: true }); + const entries = await loadAll(); + const expected = feed.items.filter(item => item.sourceKind === "blocker_attention") + .map(item => `${item.subject.id}:${new Date(item.activityAt).toISOString()}`).sort(); + expect(entries.filter(item => item.sourceKind === "blocker_attention") + .map(item => `${item.sourceId}:${new Date(item.activityAt).toISOString()}`).sort()).toEqual(expected); + expect(expected).toContain(`${terminal.id}:${terminal.updatedAt.toISOString()}`); + await start(); await archived("blocker_attention", terminal.id); await archived("blocker_attention", root.id); await quiet(); + expect((await state("blocker_attention", terminal.id))?.sourceActivityAt).toEqual(terminal.updatedAt); + expect((await state("blocker_attention", root.id))?.sourceActivityAt).toEqual(root.blockedTransitionAt); + }); + + it("matches newest confirmations, independent gates, org invokability, and conversation visibility", async () => { + const origin = await insertIssue(); + const conversation = await insertIssue("in_review", { conversationAgentId: agentId, conversationUserId: "board", conversationState: "waiting", assigneeAgentId: agentId }); + const hidden = await insertIssue("todo", { hiddenAt: old() }); + const [paused, manager, unhealthy] = await db.insert(agents).values([ + { companyId, name: "Paused", role: "engineer", status: "paused" }, + { companyId, name: "Terminated", role: "manager", status: "terminated" }, + { companyId, name: "Unhealthy", role: "engineer", status: "active" }, + ]).returning(); + await db.update(agents).set({ reportsTo: manager.id }).where(eq(agents.id, unhealthy.id)); + const common = { companyId, issueId: origin.id, kind: "request_confirmation", payload: { prompt: "Approve?" }, createdAt: old(), updatedAt: old() }; + const [older, newest, independent] = await db.insert(issueThreadInteractions).values([ + common, { ...common, createdAt: new Date(old().getTime() + 1000) }, + { ...common, payload: { prompt: "Independent", qualityReview: { artifactKey: "copy" } } }, + { ...common, createdAt: new Date(), addresseeUserId: "board" }, + { ...common, addresseeAgentId: agentId }, + { ...common, issueId: hidden.id }, + { ...common, issueId: conversation.id, kind: "ask_user_questions", payload: { questions: [] } }, + { ...common, issueId: conversation.id, addresseeAgentId: paused.id }, + { ...common, issueId: conversation.id, addresseeAgentId: unhealthy.id, kind: "ask_user_questions", payload: { questions: [] } }, + ] as typeof issueThreadInteractions.$inferInsert[]).returning(); + const feed = await attentionService(db).list(companyId, { all: true, allowUnscopedAll: true }); + const expected = [...new Set(feed.items.map(item => `${item.sourceKind}:${item.subject.id}`))].sort(); + expect([...new Set((await loadAll()).map(item => `${item.sourceKind}:${item.sourceId}`))].sort()).toEqual(expected); + expect(expected).not.toContain(`issue_thread_interaction:${older.id}`); + expect(expected).toContain(`issue_thread_interaction:${newest.id}`); expect(expected).toContain(`issue_thread_interaction:${independent.id}`); + await start(); await archived("issue_thread_interaction", newest.id); await archived("issue_thread_interaction", independent.id); await quiet(); + await change(tx => tx.update(issueThreadInteractions).set({ status: "cancelled" }).where(eq(issueThreadInteractions.id, newest.id))); + await archived("issue_thread_interaction", older.id); await quiet(); + }); + + it("excludes maintained reviews until a pending approval creates human attention", async () => { + const row = await insertIssue("in_review", { assigneeAgentId: agentId }); + const [interaction] = await db.insert(issueThreadInteractions).values({ companyId, issueId: row.id, + kind: "request_confirmation", addresseeAgentId: agentId, payload: { prompt: "Agent verdict" } }).returning(); + expect((await loadAll()).some(entry => entry.sourceKind === "review")).toBe(false); + const [approval] = await db.insert(approvals).values({ companyId, type: "hire_agent", payload: {}, status: "pending" }).returning(); + await db.insert(issueApprovals).values({ companyId, issueId: row.id, approvalId: approval.id }); + const feed = await attentionService(db).list(companyId, { all: true, allowUnscopedAll: true }); + expect(feed.items.some(item => item.sourceKind === "review" && item.subject.id === row.id)).toBe(true); + await start(); await archived("review", row.id); await quiet(); + expect(await state("issue_thread_interaction", interaction.id)).toBeNull(); + }); + + it("reactivates an exhausted failed run through the ordinary durable event producer", async () => { + const origin = await insertIssue(); + const [run, newer] = await db.insert(heartbeatRuns).values([ + { companyId, agentId, status: "failed", contextSnapshot: { issueId: origin.id }, createdAt: old(), finishedAt: old() }, + { companyId, agentId, status: "succeeded", contextSnapshot: { issueId: origin.id }, createdAt: new Date(old().getTime() + 1000), finishedAt: old() }, + ]).returning(); + await start(); await quiet(); + await appendHeartbeatRunEvent(db, { companyId, agentId, runId: run.id, eventType: "lifecycle", message: "Bounded retry exhausted after 3 attempts" }); + await quiet(); expect(await state("failed_run", run.id)).toBeNull(); + await change(tx => tx.delete(heartbeatRuns).where(eq(heartbeatRuns.id, newer.id))); + await archived("failed_run", run.id); await quiet(); + }); + + it("cancels an agent error deadline through ordinary clearError", async () => { + await db.update(agents).set({ status: "error", updatedAt: new Date() }).where(eq(agents.id, agentId)); + await start(); await settle(); expect(coordinator!.nextWakeAt()).not.toBeNull(); + await createAgentLifecycle(db).clearError(agentId); await quiet(); + expect(await state("agent_error_alert", agentId)).toBeNull(); + }); + + it("drains a paged backlog with source-scoped self notifications and leaves completed history quiet", async () => { + await db.insert(issues).values(Array.from({ length: 225 }, () => ({ companyId, title: randomUUID(), status: "in_review", assigneeUserId: "board", updatedAt: old() }))); + const service = attentionRetentionService(db); + const load = vi.spyOn(service, "load"); + await start(service); + await vi.waitFor(async () => { await vi.advanceTimersByTimeAsync(1000); expect(await db.select().from(decisionRetention)).toHaveLength(225); }, { timeout: 30_000 }); + await quiet(); + const scopes = load.mock.calls.map(([scope]) => scope); + expect(scopes.filter(scope => !scope)).toHaveLength(3); + expect(scopes.filter(scope => scope && !scope.sourceId)).toHaveLength(0); + expect(scopes.filter(scope => scope?.sourceId).length).toBeLessThanOrEqual(225); + expect(new Set((await db.select().from(decisionRetention)).map(row => row.sourceId)).size).toBe(225); + }); + + it("retries a failed atomic archive across drain and preserves notifications during a scan", async () => { + const row = await insertIssue("in_review", { assigneeUserId: "board" }); + await db.execute(sql`create function attention_retention_fail() returns trigger language plpgsql as $$ begin raise exception 'retention injected'; end $$`); + await db.execute(sql`create trigger attention_retention_fail before insert on decision_retention for each row execute function attention_retention_fail()`); + await start(); expect(errors).toHaveLength(1); errors.length = 0; + expect(await state("review", row.id)).toBeNull(); expect(await db.select().from(decisionArchiveNotificationOutbox)).toHaveLength(0); + draining = true; await db.execute(sql`drop trigger attention_retention_fail on decision_retention`); + statements.length = 0; await vi.advanceTimersByTimeAsync(120_000); + expect(statements.length).toBeLessThanOrEqual(1); + expect(statements.every(query => query.includes("pg_xact_status"))).toBe(true); + statements.length = 0; await vi.advanceTimersByTimeAsync(300_000); expect(statements).toEqual([]); + draining = false; await vi.advanceTimersByTimeAsync(30_000); await archived("review", row.id); await quiet(); + await coordinator!.stop(); coordinator = undefined; + let release!: () => void; let scanned!: () => void; + const gate = new Promise(resolve => { release = resolve; }); + const entered = new Promise(resolve => { scanned = resolve; }); + const service = attentionRetentionService(db), original = service.load; + vi.spyOn(service, "load").mockImplementationOnce(async (...args) => { const result = await original(...args); scanned(); await gate; return result; }); + const starting = start(service); await entered; + const next = await change(async tx => insertIssue("in_review", { assigneeUserId: "board" }, tx)); + release(); await starting; await archived("review", next.id); await quiet(); + }); + + it("registers and cancels a deadline through ordinary interaction creation and withdrawal", async () => { + const origin = await insertIssue(); await start(); await quiet(); + const service = issueThreadInteractionService(db); + const interaction = await service.create(origin, { kind: "request_confirmation", payload: { version: 1, prompt: "Approve the result?" } }, { userId: "board" }); + await settle(); expect(coordinator!.nextWakeAt()).not.toBeNull(); expect(await state("issue_thread_interaction", interaction.id)).toBeNull(); + await service.withdrawInteraction(origin, interaction.id, { reason: "No longer needed" }, { userId: "board" }); await quiet(); + }); + +}); diff --git a/server/src/__tests__/attention-service.test.ts b/server/src/__tests__/attention-service.test.ts index a024ed421d7..5af78c2cb8b 100644 --- a/server/src/__tests__/attention-service.test.ts +++ b/server/src/__tests__/attention-service.test.ts @@ -62,7 +62,7 @@ describeEmbeddedPostgres("attention service", () => { beforeAll(async () => { if (!externalUrl) tempDb = await startEmbeddedPostgresTestDatabase("paperclip-attention-service-"); - db = createDb(externalUrl ?? tempDb!.connectionString); + db = createDb(externalUrl ?? tempDb!.connectionString, { maxConnections: 6 }); }, 30_000); afterEach(async () => { diff --git a/server/src/index.ts b/server/src/index.ts index c127753889c..48bcade6531 100644 --- a/server/src/index.ts +++ b/server/src/index.ts @@ -1,3 +1,4 @@ +import { attentionRetentionService, registerAttentionRetentionWork } from "./services/attention-retention.js"; import { registerProviderTraceRetentionWork } from "./services/provider-trace-retention-work.js"; import { registerRunEvidenceMigrationWork } from "./services/run-evidence-migration-work.js"; import { registerSkillVersionRetentionWork } from "./services/skill-version-retention-work.js"; @@ -127,7 +128,6 @@ import { cloudActorHeaderSourceFromHeaders, resolveCloudTenantActor } from "./mi import { feedbackService, applyManagedEnvironments, - attentionService, backfillPrincipalAccessCompatibility, backfillLegacyToolOAuthTokens, bootstrapExecutionPolicyFromEnv, @@ -1926,23 +1926,6 @@ async function startServerWithDatabaseTeardown( if (config.heartbeatSchedulerEnabled) await heartbeat.registerStoppedInstructionCollectionWork(deliveryWork, () => !isIdleTaskDrainActive() && !heartbeatSchedulerStopped && !isWarmStandby()).ready; - const runRetentionSweep = async () => { - const activeCompanies = await db.select({ id: companies.id }).from(companies).where(eq(companies.status, "active")); - let archived = 0; - for (const company of activeCompanies) { - // Cursor pagination rebuilds the whole feed for every page; one - // unscoped all-items build keeps this sweep at a single feed build - // per company per tick. - const page = await attentionService(db as any).list(company.id, { - includeDismissed: true, - excludeSources: ["approval", "join_request"], - all: true, - allowUnscopedAll: true, - }); - archived += await retentionExecutor.autoArchive({ companyId: company.id, items: page.items }); - } - return { archived }; - }; // Startup housekeeping: provider and connection sweeps that every timer tick // repeats and that neither serving requests nor run recovery depends on. // Readiness does not wait for it. It is tracked as scheduler work, so idle @@ -1985,7 +1968,8 @@ async function startServerWithDatabaseTeardown( () => !isIdleTaskDrainActive() && !isWarmStandby() && !heartbeatSchedulerStopped).ready); await step("join_request_retention", () => registerJoinRequestRetentionWork(deliveryWork, joinRequestRetentionService(db as any), () => !isIdleTaskDrainActive() && !isWarmStandby() && !heartbeatSchedulerStopped).ready); - await step("retention_sweep", runRetentionSweep); + await step("attention_retention", () => registerAttentionRetentionWork(deliveryWork, attentionRetentionService(db as any), + () => !isIdleTaskDrainActive() && !isWarmStandby() && !heartbeatSchedulerStopped).ready); }; const createHeartbeatSchedulerTick = (scheduled: ReturnType) => () => { @@ -1995,9 +1979,6 @@ async function startServerWithDatabaseTeardown( trackHeartbeatSchedulerWork((async () => { if (isIdleTaskDrainActive() || heartbeatSchedulerStopped) return; scheduleRoutineStatusReconcile(); - trackHeartbeatSchedulerWork(runRetentionSweep().catch((err: unknown) => { - logger.error({ err }, "decision retention sweep failed"); - })); const sweptRuntimeStatuses = heartbeat.sweepExpiredRuntimeStatuses(); if (sweptRuntimeStatuses > 0) { logger.info( diff --git a/server/src/modules/agent-lifecycle/__tests__/lifecycle.test.ts b/server/src/modules/agent-lifecycle/__tests__/lifecycle.test.ts index 246cb9f707a..404e76d6b30 100644 --- a/server/src/modules/agent-lifecycle/__tests__/lifecycle.test.ts +++ b/server/src/modules/agent-lifecycle/__tests__/lifecycle.test.ts @@ -40,7 +40,7 @@ const support = external ? { supported: true } : await getEmbeddedPostgresTestSu let companyId: string; const workers: ReturnType[] = []; beforeAll(async () => { - if (external) { await applyPendingMigrations(external); db = createDb(external); } + if (external) { await applyPendingMigrations(external); db = createDb(external, { maxConnections: 6 }); } else { database = await startEmbeddedPostgresTestDatabase("paperclip-agent-lifecycle-"); db = createDb(database.connectionString); } }, 90_000); afterAll(async () => { diff --git a/server/src/modules/agent-lifecycle/adapters/records.ts b/server/src/modules/agent-lifecycle/adapters/records.ts index 1c59758dd91..db842941836 100644 --- a/server/src/modules/agent-lifecycle/adapters/records.ts +++ b/server/src/modules/agent-lifecycle/adapters/records.ts @@ -80,18 +80,19 @@ export function agentRecords(db: Db, effects: LifecycleEffects) { throw conflict("Only agents in error status can have their error cleared"); } - const updated = await db - .update(agents) - .set({ + const updated = await effects.transaction(db, existing.companyId, async tx => { + await effects.beforeStatusChange(tx, existing.companyId, id, "error", "idle"); + return tx.update(agents).set({ status: "idle", pauseReason: null, pausedAt: null, errorReason: null, updatedAt: new Date(), }) - .where(and(eq(agents.id, id), eq(agents.status, "error"), eq(agents.lifecycleState, "ready"))) - .returning() - .then((rows) => rows[0] ?? null); + .where(and(eq(agents.id, id), eq(agents.status, "error"), eq(agents.lifecycleState, "ready"))) + .returning() + .then((rows) => rows[0] ?? null); + }); if (!updated) { throw conflict("Only agents in error status can have their error cleared"); diff --git a/server/src/services/accounting-integrity.ts b/server/src/services/accounting-integrity.ts index e3f5ef1f649..02b6043ab2d 100644 --- a/server/src/services/accounting-integrity.ts +++ b/server/src/services/accounting-integrity.ts @@ -1,3 +1,4 @@ +import { notifyAttentionRetentionWork } from "./retention-work-notifications.js"; import { and, eq, sql } from "drizzle-orm"; import { agents, agentRuntimeState, companies, heartbeatRuns, type Db } from "@paperclipai/db"; import { normalizeCents, usdToCents, type AccountingFinding, type AccountingHealth, type AccountingInspection } from "@paperclipai/shared"; @@ -75,6 +76,8 @@ export function accountingIntegrityService(db: Db, hooks: BudgetServiceHooks = { if (!reason.trim()) throw conflict("A repair reason is required"); const before = await inspect(tx, companyId); if (before.fingerprint !== fingerprint) throw conflict("Accounting changed since inspection; inspect again before repairing"); + if (before.findings.some(f => f.repairable && f.kind === "agent_projection")) + await notifyAttentionRetentionWork(tx, { companyId, sourceKind: "agent_error_alert" }); for (const finding of before.findings.filter(f => f.repairable)) { if (finding.kind === "company_projection") await tx.update(companies).set({ spentMonthlyCents: sql`${finding.expected.cents}::numeric`, updatedAt: new Date() }).where(eq(companies.id, companyId)); if (finding.kind === "agent_projection") await tx.update(agents).set({ spentMonthlyCents: sql`${finding.expected.cents}::numeric`, updatedAt: new Date() }).where(and(eq(agents.id, finding.entityId), eq(agents.companyId, companyId))); diff --git a/server/src/services/agent-configuration.ts b/server/src/services/agent-configuration.ts index 4139dc6dfe0..2c2c18ac1c7 100644 --- a/server/src/services/agent-configuration.ts +++ b/server/src/services/agent-configuration.ts @@ -1,3 +1,4 @@ +import { notifyAttentionRetentionWork } from "./retention-work-notifications.js"; import { notifySkillVersionPinChange } from "./skill-version-retention-notifications.js"; import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js"; import { notifyExternalConversationSettlement } from "./external-conversation-settlement-notifications.js"; @@ -16,6 +17,7 @@ import { clearPrimaryAgent } from "./primary-agent.js"; import { budgetServiceInTransaction } from "./budgets.js"; export async function prepareAgentConfiguration(db: Db, existing: AgentConfigurationRecord, data: AgentConfigurationPatch, options?: UpdateAgentOptions): Promise { + await notifyAttentionRetentionWork(db, { companyId: existing.companyId }); const id = existing.id; const { ensureManager, assertNoCycle, assertCompanyShortnameAvailable, assertBuiltInAgentMetadataMutationAllowed } = agentRecordQueries(db); if (existing.status === "pending_approval" && !options?.allowPendingApprovalConfigUpdate) { diff --git a/server/src/services/agent-deletion.ts b/server/src/services/agent-deletion.ts index 3650c40f1a6..50564fd9c4a 100644 --- a/server/src/services/agent-deletion.ts +++ b/server/src/services/agent-deletion.ts @@ -1,3 +1,4 @@ +import { notifyAttentionRetentionWork } from "./retention-work-notifications.js"; import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js"; import { notifyExternalConversationSettlement } from "./external-conversation-settlement-notifications.js"; import { notifyUnsafeWorkspaceExportRecovery } from "./unsafe-workspace-export-notifications.js"; @@ -34,6 +35,7 @@ export async function deleteAgentDependencies(tx: Db, companyId: string, id: str if (classifierHold) throw conflict("Wait for active prompt injection checks or resolve their unknown charges in Costs before deleting this agent", { code: "agent_prompt_injection_accounting_pending", }); + await notifyAttentionRetentionWork(tx, { companyId }); await notifyExternalConversationSettlement(tx, { companyId }); await issueThreadInteractionService(tx).cancelPendingForDeletedAddressee(companyId, id); await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors); diff --git a/server/src/services/agent-hiring.ts b/server/src/services/agent-hiring.ts index a150937a212..0c238907bbc 100644 --- a/server/src/services/agent-hiring.ts +++ b/server/src/services/agent-hiring.ts @@ -1,3 +1,4 @@ +import { notifyAttentionRetentionWork } from "./retention-work-notifications.js"; import { notifySkillVersionRetention, notifySkillVersionPinChange } from "./skill-version-retention-notifications.js"; import { agentAppearanceSchema, randomAgentAppearance } from "@paperclipai/shared"; import { normalizePaperclipRunnerAdapterConfig } from "@paperclipai/adapter-utils/server-utils"; @@ -16,6 +17,7 @@ import { initializePrimaryAgent } from "./primary-agent.js"; import { resolveAgentRunnerTaskDriverForCompany, resolveNewAgentRunnerForCompany } from "./agent-runner-selection.js"; export async function prepareAgentHire(db: Db, companyId: string, input: CreateAgentData, options?: CreateAgentOptions): Promise { + await notifyAttentionRetentionWork(db, { companyId }); const { runner, ...data } = input; const selection = options?.runnerResolved ? { adapterType: data.adapterType ?? "process", adapterConfig: isPlainRecord(data.adapterConfig) ? data.adapterConfig : {} } diff --git a/server/src/services/agent-lifecycle.ts b/server/src/services/agent-lifecycle.ts index 0a6dcaf0fc9..cb8818c4a15 100644 --- a/server/src/services/agent-lifecycle.ts +++ b/server/src/services/agent-lifecycle.ts @@ -1,3 +1,4 @@ +import { notifyAttentionRetentionWork } from "./retention-work-notifications.js"; import { notifyNativeReviewResponse } from "./native-review-response-notifications.js"; import { isAgentStatusInvokable } from "@paperclipai/shared"; import { notifyNativeResponsePresentation } from "./native-response-presentation-notifications.js"; @@ -38,6 +39,7 @@ export function createAgentLifecycleEffects(_hooks: BudgetServiceHooks = {}): li }, notifyWork: tx => notifyDeliveryWork(tx, DELIVERY_QUEUES.agentLifecycle), async beforeStatusChange(tx, companyId, agentId, before, after) { + await notifyAttentionRetentionWork(tx, { companyId }); if (before !== after) await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers); if (!isAgentStatusInvokable(before) && isAgentStatusInvokable(after)) { await notifyNativeResponsePresentation(tx, { companyId, agentId }); diff --git a/server/src/services/agent-project-access.ts b/server/src/services/agent-project-access.ts index 4c6ee81659a..feb183924c1 100644 --- a/server/src/services/agent-project-access.ts +++ b/server/src/services/agent-project-access.ts @@ -1,3 +1,4 @@ +import { notifyAttentionRetentionWork } from "./retention-work-notifications.js"; import { and, eq, inArray } from "drizzle-orm"; import { agents, projects, type Db } from "@paperclipai/db"; import { trustAuthorizationPolicySchema, type AgentProjectAccess } from "@paperclipai/shared"; @@ -55,6 +56,7 @@ export async function addAgentProjectAccess(db: Db, actor: AuthorizationActor, c const added = requested.filter(id => !previous.includes(id)); const next = [...previous, ...added]; if (added.length) { + await notifyAttentionRetentionWork(tx, { companyId, sourceKind: "agent_error_alert" }); await tx.update(agents).set({ permissions: { ...agent.permissions, authorizationPolicy: { ...policy, trustBoundary: { ...boundary, mode: "low_trust_review", companyId, projectIds: next }, } }, updatedAt: new Date() }).where(and(eq(agents.id, agentId), eq(agents.companyId, companyId))); diff --git a/server/src/services/agents.ts b/server/src/services/agents.ts index c9dd824f495..8a5f7551ee2 100644 --- a/server/src/services/agents.ts +++ b/server/src/services/agents.ts @@ -1,3 +1,4 @@ +import { notifyAttentionRetentionWork } from "./retention-work-notifications.js"; import { type BudgetServiceHooks } from "./budgets.js"; import { and, desc, eq, inArray } from "drizzle-orm"; @@ -52,7 +53,9 @@ export function agentService(db: Db, budgetHooks: BudgetServiceHooks = {}) { }); } - const updated = await db + const updated = await db.transaction(async tx => { + await notifyAttentionRetentionWork(tx, { companyId: existing.companyId }); + return tx .update(agents) .set({ permissions: normalizeAgentPermissions({ ...existing.permissions, ...permissions }), @@ -61,6 +64,7 @@ export function agentService(db: Db, budgetHooks: BudgetServiceHooks = {}) { .where(eq(agents.id, id)) .returning() .then((rows) => rows[0] ?? null); + }); return updated ? getById(updated.id) : null; }, diff --git a/server/src/services/attention-retention.ts b/server/src/services/attention-retention.ts new file mode 100644 index 00000000000..4622c4883bf --- /dev/null +++ b/server/src/services/attention-retention.ts @@ -0,0 +1,204 @@ +import { and, eq, isNull, sql, type SQL } from "drizzle-orm"; +import { agents, decisionRetention, decisions, heartbeatRuns, + issueApprovals, issueRecoveryActions, issueThreadInteractions, issues, approvals, type Db } from "@paperclipai/db"; +import type { AttentionSourceKind } from "@paperclipai/shared"; +import { evaluateAgentInvokability } from "./agent-invokability.js"; +import { collapsePendingConfirmationsToNewest, listFailedRunsWithNewerRun, readRunIssueId, requiresReviewAttention, terminalBlockerRequiresAttention } from "./attention-source-eligibility.js"; +import { decisionRetentionService, DEFAULT_DECISION_ARCHIVE_DAYS } from "./decision-retention.js"; +import type { createDeliveryWorkCoordinator } from "./delivery-work-coordinator.js"; +import { DELIVERY_QUEUES } from "./delivery-work-notifications.js"; +import { attentionIssueCondition } from "./issue-visibility.js"; +import { issueService } from "./issues.js"; +import { parseIssueExecutionState } from "./issue-execution-policy.js"; +import { isProspectiveBlockedTransition } from "./routable-blocked.js"; +import { registerScopedDeadlineWork } from "./scoped-deadline-work.js"; + +const archiveDelay = DEFAULT_DECISION_ARCHIVE_DAYS * 86_400_000; +const kinds = ["decision", "issue_thread_interaction", "recovery_action", "blocker_attention", "review", + "failed_run", "budget_alert", "agent_error_alert"] as const; +type Kind = typeof kinds[number]; +type Scope = { companyId?: string; sourceKind?: AttentionSourceKind; sourceId?: string }; +type Candidate = { sourceKind: Kind; sourceId: string; companyId: string; activityAt: string; deadlineAt: number }; +type Entry = Candidate & { id: string; dueAt: number; rootId: string; variant?: "owner" | "terminal" }; +type Cursor = { sourceKind: string; sourceId: string }; + +const sourceTables: Record = { + decision: sql`decisions`, issue_thread_interaction: sql`issue_thread_interactions`, + recovery_action: sql`issue_recovery_actions`, blocker_attention: sql`issues`, review: sql`issues`, + failed_run: sql`heartbeat_runs`, budget_alert: sql`budget_incidents`, agent_error_alert: sql`agents`, +}; + +const sources: Record = { + decision: sql`select 'decision' as source_kind, d.id::text as source_id, d.company_id, d.updated_at as activity_at + from decisions d where d.status = 'open'`, + issue_thread_interaction: sql`select 'issue_thread_interaction', i.id::text, i.company_id, i.updated_at + from issue_thread_interactions i where i.status = 'pending' and i.addressee_user_id is null`, + recovery_action: sql`select 'recovery_action', r.id::text, r.company_id, r.updated_at + from issue_recovery_actions r where r.status in ('active', 'escalated') and r.owner_type in ('user', 'board')`, + blocker_attention: sql`select 'blocker_attention', i.id::text, i.company_id, i.updated_at + from issues i where i.status = 'blocked' and i.hidden_at is null and i.harness_kind is null`, + review: sql`select 'review', i.id::text, i.company_id, i.updated_at + from issues i where i.status = 'in_review' and i.hidden_at is null and i.harness_kind is null`, + failed_run: sql`select 'failed_run', r.id::text, r.company_id, coalesce(r.finished_at, r.updated_at, r.created_at) + from heartbeat_runs r where r.status in ('failed', 'timed_out') and exists ( + select 1 from heartbeat_run_events e where e.run_id = r.id and e.company_id = r.company_id + and e.event_type = 'lifecycle' and e.message like 'Bounded retry exhausted%')`, + budget_alert: sql`select 'budget_alert', b.id::text, b.company_id, b.updated_at from budget_incidents b + where b.status = 'open' and (b.threshold_type = 'hard' or (b.amount_limit > 0 + and round(b.amount_observed::numeric / b.amount_limit * 10000) / 100 >= 85))`, + agent_error_alert: sql`select 'agent_error_alert', a.id::text, a.company_id, a.updated_at from agents a where a.status = 'error'`, +}; + +export function attentionRetentionService(db: Db) { + const cursors = new Map(); + async function candidates(database: Db, scope: Scope = {}, cursor?: Cursor, sourceId?: string): Promise { + const selected = kinds.filter(kind => !scope.sourceKind || scope.sourceKind === kind); + if (!selected.length) return []; + const rows = await database.execute(sql`with source_candidates(source_kind, source_id, company_id, activity_at) as (${sql.join(selected.map(kind => sources[kind]), sql` union all `)}) + select s.source_kind as "sourceKind", s.source_id as "sourceId", s.company_id as "companyId", + s.activity_at::text as "activityAt", ceil(extract(epoch from s.activity_at) * 1000 + ${archiveDelay}) as "deadlineAt" + from source_candidates s inner join companies c on c.id = s.company_id + left join decision_retention r on r.company_id = s.company_id and r.source_kind = s.source_kind and r.source_id = s.source_id + where c.status = 'active' + and (${scope.companyId ?? null}::uuid is null or s.company_id = ${scope.companyId ?? null}::uuid) + and (${sourceId ?? scope.sourceId ?? null}::text is null or s.source_id = ${sourceId ?? scope.sourceId ?? null}) + and (s.source_kind = 'blocker_attention' or ((r.id is null or r.keep = false) and r.archived_at is null)) + ${cursor ? sql`and (s.source_kind, s.source_id) > (${cursor.sourceKind}, ${cursor.sourceId})` : sql``} + order by s.source_kind, s.source_id limit 100`) as unknown as Candidate[]; + return rows.map(row => ({ ...row, deadlineAt: Number(row.deadlineAt) })); + } + + async function visibleIssue(database: Db, companyId: string, issueId: string, conversations = false) { + const [row] = await database.select().from(issues).where(and(eq(issues.companyId, companyId), eq(issues.id, issueId), + attentionIssueCondition(conversations))); + return row; + } + + async function eligible(database: Db, candidate: Candidate, variant?: "owner" | "terminal", lock = false): Promise { + let sourceId = candidate.sourceId; + let activityAt = candidate.activityAt; + let dueAt = candidate.deadlineAt; + if (candidate.sourceKind === "decision") { + const [row] = await database.select({ originIssueId: decisions.originIssueId }).from(decisions) + .where(and(eq(decisions.companyId, candidate.companyId), eq(decisions.id, sourceId), eq(decisions.status, "open"))); + if (!row || row.originIssueId && !await visibleIssue(database, candidate.companyId, row.originIssueId)) return null; + } else if (candidate.sourceKind === "issue_thread_interaction") { + const [row] = await database.select().from(issueThreadInteractions).where(and( + eq(issueThreadInteractions.companyId, candidate.companyId), eq(issueThreadInteractions.id, sourceId), + eq(issueThreadInteractions.status, "pending"), isNull(issueThreadInteractions.addresseeUserId))); + if (!row || !await visibleIssue(database, candidate.companyId, row.issueId, true)) return null; + const roster = row.addresseeAgentId ? await database.select().from(agents).where(eq(agents.companyId, candidate.companyId)) : []; + if (row.addresseeAgentId && evaluateAgentInvokability(roster.find(agent => agent.id === row.addresseeAgentId), roster).invokable) return null; + if (row.kind === "request_confirmation") { + const siblings = await database.select().from(issueThreadInteractions).where(and( + eq(issueThreadInteractions.companyId, candidate.companyId), eq(issueThreadInteractions.issueId, row.issueId), + eq(issueThreadInteractions.status, "pending"), eq(issueThreadInteractions.kind, "request_confirmation"), + isNull(issueThreadInteractions.addresseeUserId))); + const companyAgents = roster.length ? roster : await database.select().from(agents).where(eq(agents.companyId, candidate.companyId)); + const board = siblings.filter(sibling => !sibling.addresseeAgentId + || !evaluateAgentInvokability(companyAgents.find(agent => agent.id === sibling.addresseeAgentId), companyAgents).invokable); + if (!collapsePendingConfirmationsToNewest(board).some(sibling => sibling.id === sourceId)) return null; + } + } else if (candidate.sourceKind === "recovery_action") { + const [row] = await database.select().from(issueRecoveryActions).where(and(eq(issueRecoveryActions.companyId, candidate.companyId), eq(issueRecoveryActions.id, sourceId))); + if (!row || !await visibleIssue(database, candidate.companyId, row.sourceIssueId)) return null; + } else if (candidate.sourceKind === "review") { + const row = await visibleIssue(database, candidate.companyId, sourceId); + if (!row) return null; + const state = parseIssueExecutionState(row.executionState); + const [approval] = await database.select({ id: approvals.id }).from(issueApprovals).innerJoin(approvals, eq(approvals.id, issueApprovals.approvalId)) + .where(and(eq(issueApprovals.companyId, candidate.companyId), eq(approvals.companyId, candidate.companyId), + eq(issueApprovals.issueId, sourceId), eq(approvals.status, "pending"))).limit(1); + if (!requiresReviewAttention({ hasHumanParticipant: state?.status === "pending" && state.currentParticipant?.type === "user", + hasUserAssignee: Boolean(row.assigneeUserId), hasPendingApproval: Boolean(approval), stalled: false })) { + const attention = await issueService(database).listReviewAttention(candidate.companyId, [row]); + if (!requiresReviewAttention({ hasHumanParticipant: false, hasUserAssignee: false, hasPendingApproval: false, + stalled: attention.get(sourceId)?.state === "stalled" })) return null; + } + } else if (candidate.sourceKind === "blocker_attention") { + const row = await visibleIssue(database, candidate.companyId, sourceId); + if (!row) return null; + if (variant === "owner") { + if (row.unblockDescriptor?.owner !== "board" || !isProspectiveBlockedTransition(row)) return null; + const [activity] = await database.execute(sql`select blocked_transition_at::text as at, + ceil(extract(epoch from blocked_transition_at) * 1000 + ${archiveDelay}) as due from issues + where company_id = ${candidate.companyId}::uuid and id = ${row.id}::uuid`) as unknown as Array<{ at: string; due: string }>; + activityAt = activity.at; dueAt = Number(activity.due); + } else { + const attention = (await issueService(database).listBlockerAttention(candidate.companyId, [row])).get(row.id); + if (!terminalBlockerRequiresAttention(attention)) return null; + sourceId = attention.terminalBlockerIssueId ?? row.id; + if (lock && sourceId !== candidate.sourceId) await database.execute(sql`select id from issues + where company_id = ${candidate.companyId}::uuid and id = ${sourceId}::uuid for update`); + const terminal = await visibleIssue(database, candidate.companyId, sourceId); + if (!terminal) return null; + const [activity] = await database.execute(sql`select updated_at::text as at, + ceil(extract(epoch from updated_at) * 1000 + ${archiveDelay}) as due from issues + where company_id = ${candidate.companyId}::uuid and id = ${sourceId}::uuid`) as unknown as Array<{ at: string; due: string }>; + activityAt = activity.at; dueAt = Number(activity.due); + } + const [retention] = await database.select().from(decisionRetention).where(and(eq(decisionRetention.companyId, candidate.companyId), + eq(decisionRetention.sourceKind, candidate.sourceKind), eq(decisionRetention.sourceId, sourceId))); + if (retention?.keep || retention?.archivedAt) return null; + } else if (candidate.sourceKind === "failed_run") { + const [run] = await database.select().from(heartbeatRuns).where(and(eq(heartbeatRuns.companyId, candidate.companyId), eq(heartbeatRuns.id, sourceId))); + if (!run) return null; + const [agent] = await database.select({ id: agents.id }).from(agents).where(and(eq(agents.companyId, candidate.companyId), + eq(agents.id, run.agentId), sql`${agents.status} <> 'terminated'`)); + const issueId = readRunIssueId(run.contextSnapshot); + if (!agent || issueId && !await visibleIssue(database, candidate.companyId, issueId)) return null; + if ((await listFailedRunsWithNewerRun(database, candidate.companyId, [{ ...run, issueId }])).has(run.id)) return null; + } + return { ...candidate, sourceId, activityAt, rootId: candidate.sourceId, variant, + id: `${candidate.sourceKind}:${candidate.sourceId}:${variant ?? "source"}`, dueAt }; + } + + async function load(scope?: Scope, signal?: AbortSignal) { + const key = JSON.stringify(scope ?? {}); + const rows = await candidates(db, scope, cursors.get(key)); + const entries: Entry[] = []; + for (const row of rows) { + if (signal?.aborted) return { rows: null, complete: true }; + for (const variant of row.sourceKind === "blocker_attention" ? ["owner", "terminal"] as const : [undefined]) { + const entry = await eligible(db, row, variant); + if (entry) entries.push(entry); + } + } + if (rows.length === 100) { + const last = rows[rows.length - 1]!; + cursors.set(key, { sourceKind: last.sourceKind, sourceId: last.sourceId }); + } else cursors.delete(key); + return { rows: entries, complete: rows.length < 100 }; + } + + async function check(entry: Entry) { + return db.transaction(async transaction => { + const database = transaction as unknown as Db; + await database.execute(sql`select id from ${sourceTables[entry.sourceKind]} + where company_id = ${entry.companyId}::uuid and id = ${entry.rootId}::uuid for update`); + const [candidate] = await candidates(database, { companyId: entry.companyId, sourceKind: entry.sourceKind }, undefined, entry.rootId); + if (!candidate) return null; + const current = await eligible(database, candidate, entry.variant, true); + if (!current || current.dueAt > Date.now()) return current; + const retention = decisionRetentionService(database); + const item = { sourceKind: current.sourceKind, subject: { id: current.sourceId }, activityAt: current.activityAt }; + const state = (await retention.syncItems(current.companyId, [item])).get(`${current.sourceKind}:${current.sourceId}`)!; + await retention.autoArchive({ companyId: current.companyId, items: [{ ...item, + keep: state.keep, archivedAt: state.archivedAt?.toISOString() ?? null, retentionVersion: state.version }] }); + return null; + }); + } + return { load, check }; +} + +export function registerAttentionRetentionWork(coordinator: ReturnType, + service: ReturnType, canRun: () => boolean) { + return registerScopedDeadlineWork(coordinator, { + queue: DELIVERY_QUEUES.attentionRetention, + decodeScope: hint => hint === "*" ? undefined : JSON.parse(hint) as Scope, + matches: (entry, scope) => (!scope?.companyId || scope.companyId === entry.companyId) + && (!scope?.sourceKind || scope.sourceKind === entry.sourceKind) + && (!scope?.sourceId || scope.sourceId === entry.sourceId), + load: service.load, check: service.check, + }, canRun); +} diff --git a/server/src/services/attention-source-eligibility.ts b/server/src/services/attention-source-eligibility.ts new file mode 100644 index 00000000000..6e3f484c03a --- /dev/null +++ b/server/src/services/attention-source-eligibility.ts @@ -0,0 +1,108 @@ +import { and, eq, gt, inArray, sql, type SQL } from "drizzle-orm"; +import { heartbeatRuns, type Db } from "@paperclipai/db"; + +export function collapsePendingConfirmationsToNewest(rows: T[]) { + const independent = (row: T) => Boolean(row.payload && typeof row.payload === "object" && "qualityReview" in row.payload && row.payload.qualityReview); + const newestByGroup = new Map(); + for (const row of rows) { + if (row.kind !== "request_confirmation" || independent(row)) continue; + const groupKey = `${row.issueId}:${row.kind}`; + const newest = newestByGroup.get(groupKey); + if ( + !newest + || row.createdAt.getTime() > newest.createdAt.getTime() + || (row.createdAt.getTime() === newest.createdAt.getTime() && row.id > newest.id) + ) { + newestByGroup.set(groupKey, row); + } + } + + return rows.filter((row) => ( + row.kind !== "request_confirmation" + || independent(row) + || newestByGroup.get(`${row.issueId}:${row.kind}`)?.id === row.id + )); +} + +export function readRunIssueId(contextSnapshot: Record | null) { + const issueId = contextSnapshot?.issueId ?? contextSnapshot?.taskId; + return typeof issueId === "string" && issueId.length > 0 ? issueId : null; +} + +type FailedRunProbe = { id: string; agentId: string; issueId: string | null; createdAt: Date }; + +const FAILED_RUN_PROBE_BATCH_SIZE = 1000; + +export async function listFailedRunsWithNewerRun(db: Db, companyId: string, failedRuns: FailedRunProbe[]) { + const withNewerRun = new Set(); + const keyed = failedRuns.filter(run => run.issueId !== null); + for (let start = 0; start < keyed.length; start += FAILED_RUN_PROBE_BATCH_SIZE) { + const batch = keyed.slice(start, start + FAILED_RUN_PROBE_BATCH_SIZE); + for (const id of await listFailedRunBatchWithNewerRun(db, companyId, batch)) withNewerRun.add(id); + } + const unkeyed = failedRuns.filter(run => run.issueId === null); + if (unkeyed.length === 0) return withNewerRun; + const agentIds = [...new Set(unkeyed.map(run => run.agentId))]; + const oldestCreatedAt = unkeyed.reduce((oldest, run) => run.createdAt < oldest ? run.createdAt : oldest, + unkeyed[0]!.createdAt); + const latestByAgent = new Map(); + for (let start = 0; start < agentIds.length; start += FAILED_RUN_PROBE_BATCH_SIZE) { + const rows = await db.select({ + agentId: heartbeatRuns.agentId, + createdAt: sql`max(${heartbeatRuns.createdAt})`.mapWith(heartbeatRuns.createdAt), + }).from(heartbeatRuns).where(and( + eq(heartbeatRuns.companyId, companyId), + inArray(heartbeatRuns.agentId, agentIds.slice(start, start + FAILED_RUN_PROBE_BATCH_SIZE)), + gt(heartbeatRuns.createdAt, oldestCreatedAt), + sql`coalesce(${heartbeatRuns.contextSnapshot} ->> 'issueId', ${heartbeatRuns.contextSnapshot} ->> 'taskId', '') = ''`, + )).groupBy(heartbeatRuns.agentId); + for (const row of rows) latestByAgent.set(row.agentId, row.createdAt); + } + for (const run of unkeyed) { + if ((latestByAgent.get(run.agentId)?.getTime() ?? 0) > run.createdAt.getTime()) withNewerRun.add(run.id); + } + return withNewerRun; +} + +async function listFailedRunBatchWithNewerRun(db: Db, companyId: string, failedRuns: FailedRunProbe[]) { + if (failedRuns.length === 0) return []; + const failed = sql.join( + failedRuns.map((run) => sql`(${run.id}::uuid, ${run.agentId}::uuid, ${run.issueId}::text, ${new Date(run.createdAt.getTime() + 1).toISOString()}::timestamptz)`), + sql`, `, + ); + const runIssueId = sql`(${heartbeatRuns.contextSnapshot} ->> 'issueId')`; + const runTaskId = sql`(${heartbeatRuns.contextSnapshot} ->> 'taskId')`; + const newerRunExists = (match: SQL) => sql`exists ( + select 1 from ${heartbeatRuns} + where ${heartbeatRuns.companyId} = ${companyId} + and ${heartbeatRuns.agentId} = failed.agent_id + and ${heartbeatRuns.createdAt} >= failed.newer_from + and ${match} + )`; + const rows = await db.execute(sql` + select failed.id + from (values ${failed}) as failed (id, agent_id, issue_id, newer_from) + where ${newerRunExists(sql`${runIssueId} = failed.issue_id`)} + or ${newerRunExists(sql`${runTaskId} = failed.issue_id and ${runIssueId} is null`)} + `) as unknown as Array<{ id: string }>; + return rows.map((row) => row.id); +} + + +export function requiresReviewAttention(input: { + hasHumanParticipant: boolean; hasUserAssignee: boolean; hasPendingApproval: boolean; stalled: boolean; +}) { + return input.hasHumanParticipant || input.hasUserAssignee || input.hasPendingApproval || input.stalled; +} + +export function terminalBlockerRequiresAttention( + attention: T | null | undefined, +): attention is T & { state: "stalled" | "needs_attention" } { + return Boolean(attention && !attention.blockingTreeLive && ["stalled", "needs_attention"].includes(attention.state ?? "")); +} diff --git a/server/src/services/attention.ts b/server/src/services/attention.ts index 8d5dedd3b03..9f18f7d8104 100644 --- a/server/src/services/attention.ts +++ b/server/src/services/attention.ts @@ -1,5 +1,6 @@ +import { collapsePendingConfirmationsToNewest, readRunIssueId, listFailedRunsWithNewerRun, requiresReviewAttention, terminalBlockerRequiresAttention } from "./attention-source-eligibility.js"; import { approvalExecutionIssueCondition, firstApprovalIssueId } from "./approval-visibility.js"; -import { and, asc, desc, eq, gt, inArray, isNotNull, isNull, notInArray, or, sql, type SQL } from "drizzle-orm"; +import { and, asc, desc, eq, inArray, isNull, notInArray, or, sql, type SQL } from "drizzle-orm"; import type { Db } from "@paperclipai/db"; import { agents, @@ -58,7 +59,7 @@ import { BLOCKER_ATTENTION_MAX_NODES, issueService, } from "./issues.js"; -import { executionIssueCondition, visibleIssueCondition } from "./issue-visibility.js"; +import { attentionIssueCondition, executionIssueCondition } from "./issue-visibility.js"; import { parseIssueExecutionState } from "./issue-execution-policy.js"; import { isProspectiveBlockedTransition } from "./routable-blocked.js"; import { issueReadSqlCondition, type AuthorizationActor } from "./authorization.js"; @@ -766,37 +767,6 @@ export function interactionResolverAudience( }; } -function collapsePendingConfirmationsToNewest(rows: T[]) { - // Quality gates bind independent outputs; a newer visual approval must not - // hide a still-pending copy approval (or affect its badge count). - const independent = (row: T) => Boolean(row.payload && typeof row.payload === "object" && "qualityReview" in row.payload && row.payload.qualityReview); - const newestByGroup = new Map(); - for (const row of rows) { - if (row.kind !== "request_confirmation" || independent(row)) continue; - const groupKey = `${row.issueId}:${row.kind}`; - const newest = newestByGroup.get(groupKey); - if ( - !newest - || row.createdAt.getTime() > newest.createdAt.getTime() - || (row.createdAt.getTime() === newest.createdAt.getTime() && row.id > newest.id) - ) { - newestByGroup.set(groupKey, row); - } - } - - return rows.filter((row) => ( - row.kind !== "request_confirmation" - || independent(row) - || newestByGroup.get(`${row.issueId}:${row.kind}`)?.id === row.id - )); -} - function budgetObservedPercent(amountObserved: number, amountLimit: number) { return amountLimit > 0 ? Math.round((amountObserved / amountLimit) * 10_000) / 100 : 0; } @@ -866,9 +836,7 @@ async function issueSummaryMap( .where(and( eq(issues.companyId, companyId), inArray(issues.id, ids), - includeConversations - ? or(executionIssueCondition(), and(visibleIssueCondition(), isNotNull(issues.conversationAgentId))) - : executionIssueCondition(), + attentionIssueCondition(includeConversations), readCondition, )); return new Map(rows.map((row) => [row.id, { @@ -1108,70 +1076,6 @@ function resolveBlockingIssue( return null; } -function readRunIssueId(contextSnapshot: Record | null) { - const issueId = contextSnapshot?.issueId ?? contextSnapshot?.taskId; - return typeof issueId === "string" && issueId.length > 0 ? issueId : null; -} - -type FailedRunProbe = { id: string; agentId: string; issueId: string | null; createdAt: Date }; - -const FAILED_RUN_PROBE_BATCH_SIZE = 1000; - -async function listFailedRunsWithNewerRun(db: Db, companyId: string, failedRuns: FailedRunProbe[]) { - const withNewerRun = new Set(); - const keyed = failedRuns.filter(run => run.issueId !== null); - for (let start = 0; start < keyed.length; start += FAILED_RUN_PROBE_BATCH_SIZE) { - const batch = keyed.slice(start, start + FAILED_RUN_PROBE_BATCH_SIZE); - for (const id of await listFailedRunBatchWithNewerRun(db, companyId, batch)) withNewerRun.add(id); - } - const unkeyed = failedRuns.filter(run => run.issueId === null); - if (unkeyed.length === 0) return withNewerRun; - const agentIds = [...new Set(unkeyed.map(run => run.agentId))]; - const oldestCreatedAt = unkeyed.reduce((oldest, run) => run.createdAt < oldest ? run.createdAt : oldest, - unkeyed[0]!.createdAt); - const latestByAgent = new Map(); - for (let start = 0; start < agentIds.length; start += FAILED_RUN_PROBE_BATCH_SIZE) { - const rows = await db.select({ - agentId: heartbeatRuns.agentId, - createdAt: sql`max(${heartbeatRuns.createdAt})`.mapWith(heartbeatRuns.createdAt), - }).from(heartbeatRuns).where(and( - eq(heartbeatRuns.companyId, companyId), - inArray(heartbeatRuns.agentId, agentIds.slice(start, start + FAILED_RUN_PROBE_BATCH_SIZE)), - gt(heartbeatRuns.createdAt, oldestCreatedAt), - sql`coalesce(${heartbeatRuns.contextSnapshot} ->> 'issueId', ${heartbeatRuns.contextSnapshot} ->> 'taskId', '') = ''`, - )).groupBy(heartbeatRuns.agentId); - for (const row of rows) latestByAgent.set(row.agentId, row.createdAt); - } - for (const run of unkeyed) { - if ((latestByAgent.get(run.agentId)?.getTime() ?? 0) > run.createdAt.getTime()) withNewerRun.add(run.id); - } - return withNewerRun; -} - -async function listFailedRunBatchWithNewerRun(db: Db, companyId: string, failedRuns: FailedRunProbe[]) { - if (failedRuns.length === 0) return []; - const failed = sql.join( - failedRuns.map((run) => sql`(${run.id}::uuid, ${run.agentId}::uuid, ${run.issueId}::text, ${new Date(run.createdAt.getTime() + 1).toISOString()}::timestamptz)`), - sql`, `, - ); - const runIssueId = sql`(${heartbeatRuns.contextSnapshot} ->> 'issueId')`; - const runTaskId = sql`(${heartbeatRuns.contextSnapshot} ->> 'taskId')`; - const newerRunExists = (match: SQL) => sql`exists ( - select 1 from ${heartbeatRuns} - where ${heartbeatRuns.companyId} = ${companyId} - and ${heartbeatRuns.agentId} = failed.agent_id - and ${heartbeatRuns.createdAt} >= failed.newer_from - and ${match} - )`; - const rows = await db.execute(sql` - select failed.id - from (values ${failed}) as failed (id, agent_id, issue_id, newer_from) - where ${newerRunExists(sql`${runIssueId} = failed.issue_id`)} - or ${newerRunExists(sql`${runTaskId} = failed.issue_id and ${runIssueId} is null`)} - `) as unknown as Array<{ id: string }>; - return rows.map((row) => row.id); -} - export function attentionService(db: Db, serviceOptions: AttentionServiceOptions = {}) { const openDecisionLimit = Math.min( Math.max(Math.trunc(serviceOptions.openDecisionLimit ?? OPEN_DECISION_DEFAULT_LIMIT), 1), @@ -1636,8 +1540,7 @@ export function attentionService(db: Db, serviceOptions: AttentionServiceOptions })); } const blockerAttention = issue.blockerAttention; - if (blockerAttention?.state !== "stalled" && blockerAttention?.state !== "needs_attention") continue; - if (blockerAttention.blockingTreeLive) continue; + if (!terminalBlockerRequiresAttention(blockerAttention)) continue; const issueSummary = blockedIssueSummaries.get(issue.id) ?? null; const terminalIssueId = blockerAttention.terminalBlockerIssueId ?? issue.id; const terminalSummary = terminalBlockerSummaries.get(terminalIssueId) @@ -1739,7 +1642,8 @@ export function attentionService(db: Db, serviceOptions: AttentionServiceOptions const pendingApprovalId = pendingApprovalByIssueId.get(review.id) ?? null; const reviewAttention = reviewAttentionByIssueId.get(review.id); const stalled = reviewAttention?.state === "stalled"; - if (!hasHumanParticipant && !review.assigneeUserId && !pendingApprovalId && !stalled) continue; + if (!requiresReviewAttention({ hasHumanParticipant, hasUserAssignee: Boolean(review.assigneeUserId), + hasPendingApproval: Boolean(pendingApprovalId), stalled })) continue; const issue = reviewIssueMap.get(review.id); if (!issue) continue; const dedupKey = `review:${review.id}`; diff --git a/server/src/services/budgets.ts b/server/src/services/budgets.ts index 6777124ee90..93403926772 100644 --- a/server/src/services/budgets.ts +++ b/server/src/services/budgets.ts @@ -1,5 +1,5 @@ import { notifyApprovalWork } from "./approval-work-notifications.js"; -import { notifyCompanyRetentionWork } from "./retention-work-notifications.js"; +import { notifyAttentionRetentionWork, notifyCompanyRetentionWork } from "./retention-work-notifications.js"; import { addCents, compareCents, normalizeCents } from "@paperclipai/shared"; import { and, desc, eq, inArray, sql, or, isNull } from "drizzle-orm"; import type { Db } from "@paperclipai/db"; @@ -466,6 +466,7 @@ export function budgetServiceInTransaction(db: Db, publications: ActivityPublica windowEnd: end, }); + await notifyAttentionRetentionWork(db, { companyId: policy.companyId, sourceKind: "budget_alert" }); if (thresholdType === "hard") await notifyApprovalWork(db, policy.companyId); const approval = thresholdType === "hard" ? await db @@ -505,6 +506,7 @@ export function budgetServiceInTransaction(db: Db, publications: ActivityPublica } async function resolveOpenSoftIncidents(policyId: string) { + await notifyAttentionRetentionWork(db, { sourceKind: "budget_alert" }); await db .update(budgetIncidents) .set({ @@ -527,6 +529,7 @@ export function budgetServiceInTransaction(db: Db, publications: ActivityPublica decidedByUserId: string | null, thresholdType?: BudgetThresholdType, ) { + await notifyAttentionRetentionWork(db, { sourceKind: "budget_alert" }); const condition = and(eq(budgetIncidents.policyId, policyId), eq(budgetIncidents.status, "open"), thresholdType ? eq(budgetIncidents.thresholdType, thresholdType) : undefined); const openRows = await db @@ -937,6 +940,7 @@ export function budgetServiceInTransaction(db: Db, publications: ActivityPublica } const now = new Date(); + await notifyAttentionRetentionWork(db, { companyId, sourceKind: "budget_alert" }); await notifyDeliveryWork(db, DELIVERY_QUEUES.budgetEnforcement, companyId); await db .update(budgetPolicies) diff --git a/server/src/services/company-portability.ts b/server/src/services/company-portability.ts index dc7dca311d6..fff45394e08 100644 --- a/server/src/services/company-portability.ts +++ b/server/src/services/company-portability.ts @@ -1,3 +1,4 @@ +import { notifyAttentionRetentionWork } from "./retention-work-notifications.js"; import { createAgentLifecycle } from "./agent-lifecycle.js"; import { agentAppearanceSchema, agentHarnessType, paperclipRunnerProfileForHarness, agentRunner, type AgentRunnerChoice } from "@paperclipai/shared"; import { resolveNewAgentRunnerForCompany } from "./agent-runner-selection.js"; @@ -6465,7 +6466,9 @@ export function companyPortabilityService(db: Db, storage?: StorageService) { } if (relationRows.length > 0) { const relationCompanyId = targetCompany.id; - await db + await db.transaction(async tx => { + await notifyAttentionRetentionWork(tx, { companyId: relationCompanyId }); + return tx .insert(issueRelations) .values(relationRows.map((row) => ({ companyId: relationCompanyId, @@ -6476,6 +6479,7 @@ export function companyPortabilityService(db: Db, storage?: StorageService) { createdByUserId: actorUserId ?? null, }))) .onConflictDoNothing(); + }); } } diff --git a/server/src/services/decision-retention.ts b/server/src/services/decision-retention.ts index fb22df2a583..b5ca6c9f1fb 100644 --- a/server/src/services/decision-retention.ts +++ b/server/src/services/decision-retention.ts @@ -1,3 +1,4 @@ +import { isProspectiveBlockedTransition } from "./routable-blocked.js"; import { notifyRetentionWork } from "./retention-work-notifications.js"; import { decisionArchiveDeliveryService, type ArchiveNotificationBatch } from "./decision-archive-delivery.js"; import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js"; @@ -15,7 +16,6 @@ import { issueRecoveryActions, issueThreadInteractions, issues, - joinRequests, } from "@paperclipai/db"; import type { AttentionArchiveManifestEntry, @@ -145,7 +145,7 @@ export function decisionRetentionService( const existing = new Set(rows.map(row => sourceKey(row.sourceKind, row.sourceId))); const missing = unique.filter(item => !existing.has(itemSourceKey(item))); if (missing.length) { - for (const kind of new Set(missing.map(item => item.sourceKind))) await notifyRetentionWork(tx, kind); + for (const item of missing) await notifyRetentionWork(tx, item.sourceKind, { companyId, sourceId: item.subject.id }); await tx.insert(decisionRetention).values(missing.map(item => ({ companyId, sourceKind: item.sourceKind, sourceId: item.subject.id, sourceActivityAt: new Date(item.activityAt), }))).onConflictDoNothing({ target: [decisionRetention.companyId, decisionRetention.sourceKind, decisionRetention.sourceId] }); @@ -155,7 +155,7 @@ export function decisionRetentionService( for (const item of unique) { const key = itemSourceKey(item), row = byKey.get(key), activityAt = new Date(item.activityAt); if (!row || row.sourceActivityAt.getTime() === activityAt.getTime()) continue; - await notifyRetentionWork(tx, item.sourceKind); + await notifyRetentionWork(tx, item.sourceKind, { companyId, sourceId: item.subject.id }); const [updated] = await tx.update(decisionRetention).set({ sourceActivityAt: activityAt, version: sql`${decisionRetention.version} + 1`, updatedAt: new Date(), }).where(and(eq(decisionRetention.id, row.id), ne(decisionRetention.sourceActivityAt, activityAt))).returning(); @@ -169,9 +169,9 @@ export function decisionRetentionService( return sourceKey(item.sourceKind, item.subject.id); } - async function writeRetention(kind: AttentionSourceKind, write: (tx: Db) => Promise) { + async function writeRetention(companyId: string, kind: AttentionSourceKind, sourceId: string, write: (tx: Db) => Promise) { return db.transaction(async tx => { - await notifyRetentionWork(tx, kind); + await notifyRetentionWork(tx, kind, { companyId, sourceId }); return write(tx as unknown as Db); }); } @@ -185,13 +185,28 @@ export function decisionRetentionService( } async function materializeSourceState(database: Db, input: { companyId: string; sourceKind: AttentionSourceKind; sourceId: string }) { - const table = input.sourceKind === "approval" ? approvals : input.sourceKind === "join_request" ? joinRequests : null; - if (!table) return; - const [source] = await database.select({ updatedAt: table.updatedAt }).from(table) - .where(and(eq(table.companyId, input.companyId), eq(table.id, input.sourceId))).for("update"); + const tables: Record = { + approval: "approvals", join_request: "join_requests", decision: "decisions", + issue_thread_interaction: "issue_thread_interactions", recovery_action: "issue_recovery_actions", + failed_run: "heartbeat_runs", productivity_review: "issues", blocker_attention: "issues", review: "issues", + budget_alert: "budget_incidents", agent_error_alert: "agents", + }; + if (input.sourceKind === "blocker_attention") { + const [source] = await database.select().from(issues).where(and(eq(issues.companyId, input.companyId), eq(issues.id, input.sourceId))).for("update"); + if (!source) throw notFound("Attention source not found"); + const activityAt = source.unblockDescriptor?.owner === "board" && isProspectiveBlockedTransition(source) + ? source.blockedTransitionAt : source.updatedAt; + await decisionRetentionService(database).syncItems(input.companyId, [{ + sourceKind: input.sourceKind, subject: { id: input.sourceId }, activityAt: activityAt.toISOString(), + }]); + return; + } + const activity = input.sourceKind === "failed_run" ? sql`coalesce(finished_at, updated_at, created_at)` : sql`updated_at`; + const [source] = await database.execute(sql`select ${activity} as "activityAt" from ${sql.identifier(tables[input.sourceKind])} + where company_id = ${input.companyId}::uuid and id = ${input.sourceId}::uuid for update`) as unknown as Array<{ activityAt: Date | string }>; if (!source) throw notFound("Attention source not found"); await decisionRetentionService(database).syncItems(input.companyId, [{ - sourceKind: input.sourceKind, subject: { id: input.sourceId }, activityAt: source.updatedAt.toISOString(), + sourceKind: input.sourceKind, subject: { id: input.sourceId }, activityAt: new Date(source.activityAt).toISOString(), }]); } @@ -206,7 +221,7 @@ export function decisionRetentionService( if (!(await canReadDecisionSource(db, input.authActor, input.companyId, input.sourceKind, input.sourceId))) { throw notFound("Attention source not found"); } - const updated = await writeRetention(input.sourceKind, async tx => { + const updated = await writeRetention(input.companyId, input.sourceKind, input.sourceId, async tx => { await materializeSourceState(tx, input); return tx.update(decisionRetention).set({ keep: input.keep, @@ -247,7 +262,7 @@ export function decisionRetentionService( throw notFound("Attention source not found"); } const now = new Date(); - const updated = await writeRetention(input.sourceKind, async tx => { + const updated = await writeRetention(input.companyId, input.sourceKind, input.sourceId, async tx => { await materializeSourceState(tx, input); return tx.update(decisionRetention).set(input.archived ? { archivedAt: now, @@ -324,7 +339,7 @@ export function decisionRetentionService( } const now = new Date(); for (const entry of input.manifest) { - await notifyRetentionWork(txDb, entry.sourceKind as AttentionSourceKind); + await notifyRetentionWork(txDb, entry.sourceKind as AttentionSourceKind, { companyId: input.companyId, sourceId: entry.sourceId }); const archived = await txDb.update(decisionRetention).set({ archivedAt: now, archivedReason: "accepted_proposal", @@ -368,7 +383,7 @@ export function decisionRetentionService( await db.transaction(async (tx) => { const txDb = tx as unknown as Db; for (const item of candidates) { - await notifyRetentionWork(txDb, item.sourceKind); + await notifyRetentionWork(txDb, item.sourceKind, { companyId: input.companyId, sourceId: item.subject.id }); const updated = await txDb.update(decisionRetention).set({ archivedAt: now, archivedReason: "idle_ttl", diff --git a/server/src/services/decision-work-notifications.ts b/server/src/services/decision-work-notifications.ts index 081edc8fc39..0abc8e95829 100644 --- a/server/src/services/decision-work-notifications.ts +++ b/server/src/services/decision-work-notifications.ts @@ -1,11 +1,14 @@ +import { notifyAttentionRetentionWork } from "./retention-work-notifications.js"; import type { Db } from "@paperclipai/db"; import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js"; -export function notifyDecisionCreated(transaction: object) { +export async function notifyDecisionCreated(transaction: object) { + await notifyAttentionRetentionWork(transaction, { sourceKind: "decision" }); return notifyDeliveryWork(transaction, DELIVERY_QUEUES.decisionExpiry); } export async function notifyDecisionChange(transaction: object) { + await notifyAttentionRetentionWork(transaction, { sourceKind: "decision" }); await notifyDeliveryWork(transaction, DELIVERY_QUEUES.decisionContinuation); await notifyDeliveryWork(transaction, DELIVERY_QUEUES.decisionExpiry); } diff --git a/server/src/services/delivery-work-notifications.ts b/server/src/services/delivery-work-notifications.ts index 9195f1fcb12..8951cdffa50 100644 --- a/server/src/services/delivery-work-notifications.ts +++ b/server/src/services/delivery-work-notifications.ts @@ -35,6 +35,7 @@ export const DELIVERY_QUEUES = { executionDisposition: "execution-disposition", environmentCleanup: "environment-cleanup", approvalQueueSeeding: "approval-queue-seeding", + attentionRetention: "attention-retention", approvalRetention: "approval-retention", joinRequestRetention: "join-request-retention", decisionTargets: "decision-targets", diff --git a/server/src/services/execution-control-notifications.ts b/server/src/services/execution-control-notifications.ts index 22c8913bbcd..a96ca761ec2 100644 --- a/server/src/services/execution-control-notifications.ts +++ b/server/src/services/execution-control-notifications.ts @@ -1,7 +1,9 @@ +import { notifyAttentionRetentionWork } from "./retention-work-notifications.js"; import type { Db } from "@paperclipai/db"; import { DELIVERY_QUEUES, notifyDeliveryWork, type DeliveryQueue } from "./delivery-work-notifications.js"; export async function notifyExecutionRunWork(transaction: object) { + await notifyAttentionRetentionWork(transaction); for (const queue of [DELIVERY_QUEUES.executionStatus, DELIVERY_QUEUES.executionFinalization, DELIVERY_QUEUES.executionReplacement, DELIVERY_QUEUES.executionDisposition, DELIVERY_QUEUES.issueMonitors]) { await notifyDeliveryWork(transaction, queue); @@ -14,6 +16,7 @@ export function writeExecutionControl( write: (tx: Db) => PromiseLike, ): Promise { return db.transaction(async transaction => { + await notifyAttentionRetentionWork(transaction); await notifyDeliveryWork(transaction, DELIVERY_QUEUES.issueMonitors); for (const topic of typeof queue === "string" ? [queue] : queue) await notifyDeliveryWork(transaction, topic); return write(transaction as unknown as Db); diff --git a/server/src/services/grok-bot/workspace-inspection.ts b/server/src/services/grok-bot/workspace-inspection.ts index f1db9a4a851..73cad27c8aa 100644 --- a/server/src/services/grok-bot/workspace-inspection.ts +++ b/server/src/services/grok-bot/workspace-inspection.ts @@ -1,3 +1,4 @@ +import { notifyAttentionRetentionWork } from "../retention-work-notifications.js"; import { and, eq, inArray, isNotNull, or, sql } from "drizzle-orm"; import { dotRunnerAssignments, environmentLeases, heartbeatRuns, issueRecoveryActions, issues, nativeRunFinalizations, type Db } from "@paperclipai/db"; import { authorizeExternalBotOperator } from "../external-bot-authority.js"; @@ -74,6 +75,7 @@ export async function inspectGrokWorkspaceObligations(db: Db, companyId: string, return { ...rest, externalWorkspaceInspection: inspection }; }; await tx.update(heartbeatRuns).set({ resultJson: retireFileObligation(run.resultJson!) }).where(eq(heartbeatRuns.id, run.id)); + if (actions.length) await notifyAttentionRetentionWork(tx, { companyId, sourceKind: "recovery_action" }); for (const action of actions) { await tx.update(issueRecoveryActions).set({ evidence: retireFileObligation(action.evidence), updatedAt: new Date(), nextAction: "Inspect Grok and verify its connection before sending a new task message. The old work will not be replayed.", diff --git a/server/src/services/heartbeat-run-events.ts b/server/src/services/heartbeat-run-events.ts index ebce78a8266..22219811668 100644 --- a/server/src/services/heartbeat-run-events.ts +++ b/server/src/services/heartbeat-run-events.ts @@ -1,3 +1,4 @@ +import { notifyAttentionRetentionWork } from "./retention-work-notifications.js"; import { readNativeEventEvidenceOutcome } from "./native-runtime/evidence-classifier.js"; import { notifyNativeStatusReassessment } from "./native-status-reassessment-notifications.js"; import { isNativeResponseEvidence, notifyNativeResponsePresentation } from "./native-response-presentation-notifications.js"; @@ -74,10 +75,11 @@ export async function appendHeartbeatRunEvent( db: Db, input: AppendHeartbeatRunEventInput, ): Promise { + const projectsRetention = input.eventType === "lifecycle" && Boolean(input.message?.startsWith("Bounded retry exhausted")); const projectsReassessment = readNativeEventEvidenceOutcome(input.payload?.prpEvent) !== null; const projectsProgress = isSafeNativeChatProgressEventType(input.eventType); const projectsResponse = isNativeResponseEvidence(input.eventType, input.payload?.prpEvent); - if (!input.nativeSource && !input.retryExhaustion && !projectsProgress && !projectsResponse && !projectsReassessment) { + if (!input.nativeSource && !input.retryExhaustion && !projectsProgress && !projectsResponse && !projectsReassessment && !projectsRetention) { const allocated = db.$with("allocated_run_event", {}).as(sql` update ${heartbeatRuns} set next_event_seq = ${heartbeatRuns.nextEventSeq} + 1, @@ -187,6 +189,7 @@ export async function appendHeartbeatRunEvent( if (projectsResponse && run.runtimeMode === "native") { await notifyNativeResponsePresentation(tx, { companyId: run.companyId, runId: input.runId }); } + if (projectsRetention) await notifyAttentionRetentionWork(tx, { companyId: input.companyId, sourceKind: "failed_run" }); const seq = await allocateHeartbeatRunEventSeq( tx as unknown as Db, input.runId, diff --git a/server/src/services/heartbeat/agent-status.ts b/server/src/services/heartbeat/agent-status.ts index f034cd06594..23cef4b2f87 100644 --- a/server/src/services/heartbeat/agent-status.ts +++ b/server/src/services/heartbeat/agent-status.ts @@ -1,3 +1,4 @@ +import { notifyAttentionRetentionWork } from "../retention-work-notifications.js"; import { DELIVERY_QUEUES, notifyDeliveryWork } from "../delivery-work-notifications.js"; import { agents, heartbeatRuns, type Db } from "@paperclipai/db"; import { and, asc, desc, eq, lte, sql } from "drizzle-orm"; @@ -91,6 +92,7 @@ export function createHeartbeatAgentStatus(db: Db) { )).limit(1); const nextStatus = running ? "running" : settledStatus(outcome, options?.keepIdleOnFailure); const now = new Date(); + await notifyAttentionRetentionWork(tx, { companyId: existing.companyId }); await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers); const [updated] = await tx.update(agents).set({ status: nextStatus, @@ -140,6 +142,7 @@ export function createHeartbeatAgentStatus(db: Db) { eq(heartbeatRuns.agentId, candidate.id))) .orderBy(desc(heartbeatRuns.finishedAt), desc(heartbeatRuns.createdAt), desc(heartbeatRuns.id)).limit(1); const nextStatus = settledStatus(lastRun?.status); + await notifyAttentionRetentionWork(tx, { companyId: candidate.companyId }); const [agent] = await tx.update(agents).set({ status: nextStatus, errorReason: nextStatus === "error" ? errorReason(lastRun?.error) : null, updatedAt: now, }).where(and(eq(agents.id, candidate.id), eq(agents.companyId, candidate.companyId), diff --git a/server/src/services/heartbeat/retries.ts b/server/src/services/heartbeat/retries.ts index 946f5e309f8..63fa7dc4cc6 100644 --- a/server/src/services/heartbeat/retries.ts +++ b/server/src/services/heartbeat/retries.ts @@ -1,3 +1,4 @@ +import { notifyAttentionRetentionWork } from "../retention-work-notifications.js"; import { deriveTaskKeyWithHeartbeatFallback } from "./run-state.js"; import { infrastructureRetryDelayMs } from "../system-recovery-state.js"; import { @@ -997,6 +998,7 @@ export function createHeartbeatRetries(db: Db, dependencies: HeartbeatRetryDepen retryContextSnapshot.previousRunId = continuation.previousRunId; } + await notifyAttentionRetentionWork(tx, { companyId: run.companyId }); const wakeupRequest = await tx .insert(agentWakeupRequests) .values({ diff --git a/server/src/services/issue-recovery-actions.ts b/server/src/services/issue-recovery-actions.ts index 5fb28bcb6d8..355e0bc4399 100644 --- a/server/src/services/issue-recovery-actions.ts +++ b/server/src/services/issue-recovery-actions.ts @@ -1,3 +1,4 @@ +import { notifyAttentionRetentionWork } from "./retention-work-notifications.js"; import { notifyNativeRecoveryNotices } from "./native-recovery-notice-notifications.js"; import { notifyUnsafeWorkspaceExportRecovery } from "./unsafe-workspace-export-notifications.js"; import { writeExecutionControl } from "./execution-control-notifications.js"; @@ -322,6 +323,7 @@ export function issueRecoveryActionService(db: DbOrTransaction) { ): Promise { try { const created = await db.transaction(async (tx) => { + await notifyAttentionRetentionWork(tx, { companyId: input.companyId }); await notifyNativeRecoveryNotices(tx, { companyId: input.companyId, issueId: input.sourceIssueId }); await notifyDeliveryWork(tx, DELIVERY_QUEUES.executionDisposition); await notifyDeliveryWork(tx, DELIVERY_QUEUES.executionContinuation); diff --git a/server/src/services/issue-review-state-notifications.ts b/server/src/services/issue-review-state-notifications.ts index 679b7e37475..353114a9bf0 100644 --- a/server/src/services/issue-review-state-notifications.ts +++ b/server/src/services/issue-review-state-notifications.ts @@ -1,9 +1,11 @@ +import { notifyAttentionRetentionWork } from "./retention-work-notifications.js"; import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js"; import { notifyExternalConversationSettlement } from "./external-conversation-settlement-notifications.js"; import { notifyNativeReviewResponse } from "./native-review-response-notifications.js"; import { notifyNativeStatusReassessment } from "./native-status-reassessment-notifications.js"; export async function notifyIssueReviewState(tx: object, scope: { companyId: string; issueId?: string; runId?: string; agentId?: string }) { + await notifyAttentionRetentionWork(tx, { companyId: scope.companyId }); await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors); await notifyExternalConversationSettlement(tx, scope); await notifyNativeReviewResponse(tx, scope); diff --git a/server/src/services/issue-title.ts b/server/src/services/issue-title.ts index af6716e3aa4..5a7e3ffe0eb 100644 --- a/server/src/services/issue-title.ts +++ b/server/src/services/issue-title.ts @@ -1,3 +1,4 @@ +import { notifyAttentionRetentionWork } from "./retention-work-notifications.js"; import { and, eq } from "drizzle-orm"; import { heartbeatRuns, issues, type Db } from "@paperclipai/db"; import { setIssueTitleSchema, type SetIssueTitle } from "@paperclipai/shared"; @@ -76,6 +77,7 @@ export async function setIssueTitle( || (issue.title === title && !issue.titleNeedsGeneration)) { return { result: await saveReceipt({ id: issue.id, title: issue.title, titleNeedsGeneration: issue.titleNeedsGeneration, changed: false }), publication: null }; } + await notifyAttentionRetentionWork(tx, { companyId }); await tx.update(issues).set({ title, titleNeedsGeneration: false, updatedAt: new Date() }).where(predicate); await issueReferenceService(tx).syncIssue(issueId, tx); const { publication } = await persistActivity(tx, { diff --git a/server/src/services/issue-visibility.ts b/server/src/services/issue-visibility.ts index 44a04a089e6..b54832f29ee 100644 --- a/server/src/services/issue-visibility.ts +++ b/server/src/services/issue-visibility.ts @@ -1,5 +1,5 @@ import { nonIdleSlackIssueCondition } from "./slack-conversation-state.js"; -import { and, isNull, type SQL } from "drizzle-orm"; +import { and, isNull, isNotNull, or, type SQL } from "drizzle-orm"; import { issues } from "@paperclipai/db"; export function visibleIssueCondition(): SQL { @@ -14,3 +14,9 @@ export function visibleIssueSql(alias = "issues") { export function executionIssueCondition(): SQL { return and(visibleIssueCondition(), isNull(issues.conversationAgentId), nonIdleSlackIssueCondition())!; } + +export function attentionIssueCondition(includeConversations = false): SQL { + return includeConversations + ? or(executionIssueCondition(), and(visibleIssueCondition(), isNotNull(issues.conversationAgentId)))! + : executionIssueCondition(); +} diff --git a/server/src/services/issues.ts b/server/src/services/issues.ts index 42c65bb5d02..4ee7e681907 100644 --- a/server/src/services/issues.ts +++ b/server/src/services/issues.ts @@ -1,3 +1,4 @@ +import { notifyAttentionRetentionWork } from "./retention-work-notifications.js"; import { notifyExternalConversationSettlement } from "./external-conversation-settlement-notifications.js"; import { notifyNativeRecoveryNotices } from "./native-recovery-notice-notifications.js"; import { notifyNativeRunProjection } from "./native-run-projection-notifications.js"; @@ -7626,6 +7627,7 @@ export function issueService(db: Db) { await assertNoBlockingCycles(companyId, issueId, deduped, dbOrTx); } + await notifyAttentionRetentionWork(dbOrTx, { companyId }); await dbOrTx .delete(issueRelations) .where( @@ -10606,6 +10608,7 @@ export function issueService(db: Db) { } if (values.monitorNextCheckAt) await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors); await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers); + await notifyAttentionRetentionWork(tx, { companyId }); const [issue] = await tx.insert(issues).values(values).returning(); await attachOwnedDraftImages(tx, issue, issue.description, { userId: issueData.createdByUserId, agentId: issueData.createdByAgentId }); if (issue.visibility === "private" && !inheritedPrivateSubtree && issueData.createdByAgentId) { @@ -11491,6 +11494,7 @@ export function issueService(db: Db) { || issueData.parentId !== undefined || issueData.executionWorkspaceId !== undefined) { await notifyTerminalWorkspaceWork(tx, `issue:${id}`); } + await notifyAttentionRetentionWork(tx, { companyId: receiptExisting.companyId }); if (patch.status !== undefined || patch.assigneeAgentId !== undefined || patch.assigneeUserId !== undefined || patch.hiddenAt !== undefined || patch.conversationAgentId !== undefined) { await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers); @@ -11952,6 +11956,7 @@ export function issueService(db: Db) { await requestRestorePointDeletion(tx, { sourceIssueId: id }); const [owner] = await tx.select({ companyId: issues.companyId }).from(issues).where(eq(issues.id, id)); if (owner) { + await notifyAttentionRetentionWork(tx, { companyId: owner.companyId }); await notifyExternalConversationSettlement(tx, { companyId: owner.companyId }); await notifyApprovalWork(tx, owner.companyId); } @@ -12084,6 +12089,7 @@ export function issueService(db: Db) { ) : isNull(issues.executionRunId); const updated = await db.transaction(async tx => { + await notifyAttentionRetentionWork(tx, { companyId: issueCompany.companyId }); await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors); await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers); return tx @@ -12213,6 +12219,7 @@ export function issueService(db: Db) { } const staleExecutionRunId = current.executionRunId; const adopted = await db.transaction(async tx => { + await notifyAttentionRetentionWork(tx, { companyId: issueCompany.companyId }); await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors); await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers); return tx @@ -12468,6 +12475,7 @@ export function issueService(db: Db) { existing.status === "in_progress" ? "todo" : existing.status; await notifyTerminalWorkspaceWork(tx, `issue:${id}`); await notifyNativeStatusReassessment(tx, { companyId: existing.companyId, issueId: id }); + await notifyAttentionRetentionWork(tx, { companyId: existing.companyId }); await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors); await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers); const updated = await tx @@ -12520,6 +12528,7 @@ export function issueService(db: Db) { patch.assigneeAgentId = null; } + await notifyAttentionRetentionWork(tx, { companyId: existing.companyId }); await notifyTerminalWorkspaceWork(tx, `issue:${id}`); if (existing.checkoutRunId || existing.executionRunId || options.clearAssignee) { await notifyExternalConversationSettlement(tx, { companyId: existing.companyId, issueId: id }); diff --git a/server/src/services/native-finalization-notifications.ts b/server/src/services/native-finalization-notifications.ts index 72d4cbcac49..1feb1ba9703 100644 --- a/server/src/services/native-finalization-notifications.ts +++ b/server/src/services/native-finalization-notifications.ts @@ -1,3 +1,4 @@ +import { notifyAttentionRetentionWork } from "./retention-work-notifications.js"; import { notifyNativeEnvironmentSettlement } from "./native-environment-settlement-notifications.js"; import { notifyExternalAgentSettlement } from "../modules/external-agents/index.js"; import { notifyExternalConversationSettlement } from "./external-conversation-settlement-notifications.js"; @@ -14,6 +15,7 @@ import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notificatio export type NativeFinalizationWorkScope = { companyId: string; runId?: string; issueId?: string }; export async function notifyNativeFinalizationWork(transaction: object, scope: NativeFinalizationWorkScope) { + await notifyAttentionRetentionWork(transaction, { companyId: scope.companyId }); if (scope.issueId || scope.runId) { await notifyNativeEnvironmentSettlement(transaction, scope); await notifyExternalAgentSettlement(transaction, scope); diff --git a/server/src/services/native-runtime/native-provider-capacity-retry.ts b/server/src/services/native-runtime/native-provider-capacity-retry.ts index 5b54bd0cdfc..dc9cb0dac66 100644 --- a/server/src/services/native-runtime/native-provider-capacity-retry.ts +++ b/server/src/services/native-runtime/native-provider-capacity-retry.ts @@ -1,3 +1,4 @@ +import { notifyAttentionRetentionWork } from "../retention-work-notifications.js"; import { randomUUID } from "node:crypto"; import { and, eq } from "drizzle-orm"; import { agentWakeupRequests, heartbeatRuns, type Db } from "@paperclipai/db"; @@ -49,6 +50,7 @@ export async function materializeNativeProviderCapacityRetry(input: { scheduledRetryAt: dueAt.toISOString(), executionRetryAccounting: accountingForScheduledRetry(run, NATIVE_PROVIDER_OVERLOADED_CODE, attempt), }; + await notifyAttentionRetentionWork(input.tx, { companyId: input.companyId }); const [wake] = await input.tx.insert(agentWakeupRequests).values({ companyId: run.companyId, agentId: run.agentId, source: "automation", triggerDetail: "system", reason: NATIVE_PROVIDER_OVERLOADED_CODE, status: "queued", runId: retryId, diff --git a/server/src/services/native-runtime/native-review-participant.ts b/server/src/services/native-runtime/native-review-participant.ts index 573574dd19d..f57f3a7c889 100644 --- a/server/src/services/native-runtime/native-review-participant.ts +++ b/server/src/services/native-runtime/native-review-participant.ts @@ -1,3 +1,4 @@ +import { notifyAttentionRetentionWork } from "../retention-work-notifications.js"; import { agents, heartbeatRuns, @@ -240,6 +241,7 @@ export async function claimNativeReviewExecutionLock(db: Db, input: { if (!run || !["queued", "running"].includes(run.status) || !admitted || !expected || admitted.nativeReviewInteractionId !== expected.nativeReviewInteractionId || admitted.nativeReviewDecisionId !== expected.nativeReviewDecisionId) return false; + await notifyAttentionRetentionWork(tx, { companyId: input.companyId }); const claimed = await tx.update(issues).set({ executionRunId: input.runId, executionAgentNameKey: input.agentNameKey, diff --git a/server/src/services/native-runtime/native-workspace-reconstruction.ts b/server/src/services/native-runtime/native-workspace-reconstruction.ts index 5273b30216a..8ccad19395d 100644 --- a/server/src/services/native-runtime/native-workspace-reconstruction.ts +++ b/server/src/services/native-runtime/native-workspace-reconstruction.ts @@ -1,3 +1,4 @@ +import { notifyAttentionRetentionWork } from "../retention-work-notifications.js"; import { DELIVERY_QUEUES, notifyDeliveryWork } from "../delivery-work-notifications.js"; import { notifyUnsafeWorkspaceExportRecovery } from "../unsafe-workspace-export-notifications.js"; import { and, eq, gt, inArray, ne, notInArray, or, sql } from "drizzle-orm"; @@ -179,6 +180,7 @@ export async function reconstructNativeWorkspace(input: { ...(source.interveningExecutions.length ? { interveningExecutions: source.interveningExecutions } : {}) } }; await markExecutionReconciliation(db, source.action, decision, input.actorId); const now = new Date(); + await notifyAttentionRetentionWork(tx, { companyId: input.companyId }); await notifyUnsafeWorkspaceExportRecovery(tx, { companyId: input.companyId, issueId: input.issueId }); await tx.update(issueRecoveryActions).set({ status: "resolved", outcome: "handed_back", resolvedAt: now, updatedAt: now, resolutionNote: "workspace_reconstruction_authorized", nextAction: "Continue from saved sources. The original unexported edits are lost; verify the selected attachments before rebuilding. Do not repeat completed external actions or reset the original work limits.", diff --git a/server/src/services/native-runtime/paperclip-runner-tool-authority.ts b/server/src/services/native-runtime/paperclip-runner-tool-authority.ts index 02917045861..f2dbe3088e4 100644 --- a/server/src/services/native-runtime/paperclip-runner-tool-authority.ts +++ b/server/src/services/native-runtime/paperclip-runner-tool-authority.ts @@ -1,4 +1,5 @@ import { nativeRunKeyScope, nativeSkillTestToolAllowed, assertNativeSkillTestTool } from "./native-skill-test-scope.js"; +import { notifyAttentionRetentionWork } from "../retention-work-notifications.js"; import { requiredQualityReviewSchema } from "@paperclipai/shared"; import { priorRecoveryApiReceipt } from "../recovery-effect-receipts.js"; import { DELIVERY_QUEUES, notifyDeliveryWork } from "../delivery-work-notifications.js"; @@ -2067,6 +2068,7 @@ export class PaperclipRunnerToolAuthority { replayMonitor = summarizeIssueMonitor(target, normalizeIssueExecutionPolicy(target.executionPolicy)); return { taskId: target.id, status: target.status, monitor: replayMonitor }; } + await notifyAttentionRetentionWork(tx, { companyId: this.binding.companyId }); const patch = prepareIssueMonitorUpdate(target, input.monitor, actor); await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors); const [updated] = await tx.update(issues).set({ ...patch, updatedAt: new Date() }) diff --git a/server/src/services/native-runtime/superseded-external-review-requests.ts b/server/src/services/native-runtime/superseded-external-review-requests.ts index f04db57f7d7..9697b436ed3 100644 --- a/server/src/services/native-runtime/superseded-external-review-requests.ts +++ b/server/src/services/native-runtime/superseded-external-review-requests.ts @@ -1,3 +1,4 @@ +import { notifyAttentionRetentionWork } from "../retention-work-notifications.js"; import { and, eq, isNull, ne, sql } from "drizzle-orm"; import { completionContracts, heartbeatRuns, issues, issueThreadInteractions, statusDecisionEffects, statusDecisions, workAssessments, type Db } from "@paperclipai/db"; import { hasPublishedGitHubReview } from "../chat-channels/github/review-wait.js"; @@ -70,6 +71,7 @@ export async function supersedeExternalReviewRequestsInTransaction( publications: ActivityPublication[], ) { for (const { interaction } of candidates) { + await notifyAttentionRetentionWork(tx, { companyId: binding.companyId }); const reason = "A later published review no longer requests human input."; const [retired] = await tx.update(issueThreadInteractions).set({ status: "cancelled", result: { version: 1, outcome: "withdrawn", reason }, resolvedAt: new Date(), updatedAt: new Date(), diff --git a/server/src/services/native-status-reassessment-notifications.ts b/server/src/services/native-status-reassessment-notifications.ts index fb845523bb4..3a5ec8b6c37 100644 --- a/server/src/services/native-status-reassessment-notifications.ts +++ b/server/src/services/native-status-reassessment-notifications.ts @@ -1,8 +1,10 @@ +import { notifyAttentionRetentionWork } from "./retention-work-notifications.js"; import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js"; export type NativeStatusReassessmentScope = { companyId: string; issueId?: string; runId?: string; agentId?: string }; -export function notifyNativeStatusReassessment(tx: object, scope: NativeStatusReassessmentScope) { +export async function notifyNativeStatusReassessment(tx: object, scope: NativeStatusReassessmentScope) { + await notifyAttentionRetentionWork(tx, { companyId: scope.companyId }); return notifyDeliveryWork(tx, DELIVERY_QUEUES.nativeStatusReassessment, JSON.stringify({ companyId: scope.companyId, issueId: scope.issueId, runId: scope.runId, agentId: scope.agentId })); } diff --git a/server/src/services/plugin-managed-agents.ts b/server/src/services/plugin-managed-agents.ts index e2e2c965e8c..91a81fb5120 100644 --- a/server/src/services/plugin-managed-agents.ts +++ b/server/src/services/plugin-managed-agents.ts @@ -1,3 +1,4 @@ +import { notifyAttentionRetentionWork } from "./retention-work-notifications.js"; import { createAgentLifecycle } from "./agent-lifecycle.js"; import { and, eq, isNull, ne } from "drizzle-orm"; import type { Db } from "@paperclipai/db"; @@ -340,6 +341,7 @@ export function pluginManagedAgentService( // and its canonical history. Transfer only its proven stable-key marker to // the replacement installation, atomically with the new scoped bindings. await upsertBinding(companyId, declaration, agentId, {}, adapterType, tx); + await notifyAttentionRetentionWork(tx, { companyId, sourceKind: "agent_error_alert" }); await tx.update(agents).set({ metadata: managedMetadata(options.pluginId, options.pluginKey, declaration, agent.metadata), updatedAt: new Date(), diff --git a/server/src/services/retention-work-notifications.ts b/server/src/services/retention-work-notifications.ts index ffedb0b1a4d..0331f53f258 100644 --- a/server/src/services/retention-work-notifications.ts +++ b/server/src/services/retention-work-notifications.ts @@ -3,12 +3,19 @@ import type { Db } from "@paperclipai/db"; import type { AttentionSourceKind } from "@paperclipai/shared"; import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js"; -export async function notifyRetentionWork(transaction: object, kind: AttentionSourceKind) { +export async function notifyAttentionRetentionWork(transaction: object, scope: { companyId?: string; sourceKind?: AttentionSourceKind; sourceId?: string } = {}) { + await notifyDeliveryWork(transaction, DELIVERY_QUEUES.attentionRetention, JSON.stringify(scope)); +} + +export async function notifyRetentionWork(transaction: object, kind: AttentionSourceKind, scope: { companyId?: string; sourceId?: string } = {}) { if (kind === "approval") await notifyDeliveryWork(transaction, DELIVERY_QUEUES.approvalRetention); if (kind === "join_request") await notifyDeliveryWork(transaction, DELIVERY_QUEUES.joinRequestRetention); + if (kind !== "approval" && kind !== "join_request") await notifyAttentionRetentionWork(transaction, { ...scope, sourceKind: kind, + sourceId: kind === "blocker_attention" ? undefined : scope.sourceId }); } export async function notifyCompanyRetentionWork(transaction: object, companyId: string) { + await notifyAttentionRetentionWork(transaction, { companyId }); await notifyApprovalWork(transaction, companyId); await notifyRetentionWork(transaction, "join_request"); } diff --git a/server/src/services/slack-conversation-lifecycle.ts b/server/src/services/slack-conversation-lifecycle.ts index 0fd213dc084..fddf863b69e 100644 --- a/server/src/services/slack-conversation-lifecycle.ts +++ b/server/src/services/slack-conversation-lifecycle.ts @@ -1,3 +1,4 @@ +import { notifyAttentionRetentionWork } from "./retention-work-notifications.js"; import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js"; import { notifyUnsafeWorkspaceExportRecovery } from "./unsafe-workspace-export-notifications.js"; import { and, asc, desc, eq, gt, inArray, isNull, sql } from "drizzle-orm"; @@ -126,6 +127,7 @@ export async function settleExternalConversation(db: Db, companyId: string, issu const ready = await readSettlement(tx, companyId, issueId, true); if (!ready) return false; const { binding, issue, run } = ready; + await notifyAttentionRetentionWork(tx, { companyId }); const changed = await tx.update(chatConversations).set({ state: "waiting", updatedAt: new Date() }) .where(and(eq(chatConversations.id, binding.conversation.id), eq(chatConversations.companyId, companyId), eq(chatConversations.state, "active"))) .returning({ id: chatConversations.id }); diff --git a/server/src/services/slack-conversation-state.ts b/server/src/services/slack-conversation-state.ts index dedaa8d6676..5fbf7a2d43e 100644 --- a/server/src/services/slack-conversation-state.ts +++ b/server/src/services/slack-conversation-state.ts @@ -1,3 +1,4 @@ +import { notifyAttentionRetentionWork } from "./retention-work-notifications.js"; import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js"; import { notifyExternalConversationSettlement } from "./external-conversation-settlement-notifications.js"; import { chatConversations, issues, type Db } from "@paperclipai/db"; @@ -45,6 +46,7 @@ export function nonIdleSlackIssueCondition() { /** Called inside message admission while its transaction owns the issue lock. */ export async function resumeExternalConversation(tx: Db, companyId: string, issueId: string) { + await notifyAttentionRetentionWork(tx, { companyId }); await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers); await notifyExternalConversationSettlement(tx, { companyId, issueId }); // Only the verified idle convention may leave In Review automatically. From 7edb163a80d1aca60e6c780c9472befa523fcd5a Mon Sep 17 00:00:00 2001 From: Dotta Date: Fri, 30 Oct 2026 16:49:17 -0500 Subject: [PATCH 10/74] fix(server): cancel merge observation HTTP requests on shutdown Bound GitHub object and repository probes, keep readiness independent of external reads, and page pending cards through the owning parser. Co-Authored-By: Paperclip --- ...026-10-30-merged-pr-confirmations-sleep.md | 34 ++++++- .../external-objects-service.test.ts | 9 +- .../merged-pr-confirmation-work.test.ts | 88 +++++++++++++++++++ .../server-startup-feedback-export.test.ts | 17 +++- server/src/index.ts | 4 +- server/src/services/external-objects.ts | 1 + .../github-external-object-provider.ts | 18 ++-- .../src/services/github-pull-request-merge.ts | 17 ++-- .../src/services/issue-thread-interactions.ts | 38 +++++--- .../services/merged-pr-confirmation-work.ts | 14 ++- 10 files changed, 200 insertions(+), 40 deletions(-) diff --git a/doc/plans/2026-10-30-merged-pr-confirmations-sleep.md b/doc/plans/2026-10-30-merged-pr-confirmations-sleep.md index fb8b091f841..7b8699456fb 100644 --- a/doc/plans/2026-10-30-merged-pr-confirmations-sleep.md +++ b/doc/plans/2026-10-30-merged-pr-confirmations-sleep.md @@ -13,10 +13,12 @@ it receives no GitHub payloads. No database migration is required. ## Durable truth and producers Pending `request_confirmation` interaction rows remain the durable work source. -The candidate query excludes terminal and hidden issues, linked tool actions, -and text without merge intent or a PR reference. The existing fail-closed parser -then excludes additional actions, governed actions, and document/attachment targets. -No accepted or cancelled history keeps a deadline alive. +The candidate query pages pending cards in bounded groups of 100 and excludes +terminal and hidden issues and linked tool actions. The existing fail-closed +parser is the sole text eligibility source: it requires merge intent and a PR +reference and excludes additional actions, governed actions, and document/attachment targets. +No accepted or cancelled history keeps a deadline alive. A full page without any +merge candidates still advances its keyset cursor; later candidates are not lost. Ordinary creation registers the topic before inserting a confirmation in its transaction. Existing confirmation-resolution notifications invalidate its issue @@ -53,6 +55,14 @@ This is candidate-backed external observation. It does not eliminate all polling register a Cloud wake alarm, or prove that the host/database suspends. Existing sleep-safety blockers remain in force. +Startup registers this observation without waiting for live GitHub requests. +The internal resolver invocation accepts an optional host-owned abort signal; +that signal is not part of external-object records or plugin/RPC payloads. +The GitHub owner combines cancellation with a ten-second request budget and +passes it to the actual object fetch and repository-visibility probes, including +response body reads. Shutdown aborts the request and waits for its cancellation; +no post-abort acceptance is committed. A timeout leaves the durable card pending. + ## Acceptance evidence Focused tests use real PostgreSQL, the actual service producers, transaction @@ -68,6 +78,22 @@ provider failure across drain; standby/resume; restart; tenant-scoped and duplic merge hints; latest/superseded cards during provider observation; hidden/terminal issues; URL targets; no-continuation policy; and shutdown during an in-flight read. Existing startup and normalized GitHub event tests exercise the changed consumers. +Controlled local HTTP tests hold headers, partial response bodies, 404 probes, +and public-visibility probes. They observe server-side response close after +shutdown within one second, rather than only racing a promise. A shortened +request budget times out a real request; the saved card then accepts after +restart when that same HTTP fixture serves a merged response. Existing GitHub +provider and external-object service tests preserve their authorization behavior. + +## Verification follow-up + +The original head passed full-workspace typecheck and build. A later required +Linux CI shard failed the unchanged managed-skill maintenance CLI fixture in +`managed-skill-content.test.ts`: it returned `unavailable_or_corrupt_package` +where `local_storage` was expected. Both the isolated case and the complete +unchanged 17-test file pass against fresh local PostgreSQL. The Linux cause is +not established; this is retained as an unresolved CI failure, not called a flake. +The current-head required CI result remains the completion gate. ## Existing limitations outside this migration diff --git a/server/src/__tests__/external-objects-service.test.ts b/server/src/__tests__/external-objects-service.test.ts index e9022599779..af3b4667deb 100644 --- a/server/src/__tests__/external-objects-service.test.ts +++ b/server/src/__tests__/external-objects-service.test.ts @@ -1,3 +1,4 @@ +import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js"; import { randomUUID } from "node:crypto"; import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest"; import { eq } from "drizzle-orm"; @@ -26,7 +27,8 @@ import type { PaperclipPluginManifestV1 } from "@paperclipai/shared"; import type { PluginWorkerManager } from "../services/plugin-worker-manager.js"; import { createGitHubExternalObjectProvider } from "../services/github-external-object-provider.js"; -const embeddedPostgresSupport = await getEmbeddedPostgresTestSupport(); +const external = resolveExternalTestDatabaseUrl("PAPERCLIP_EXTERNAL_OBJECTS_TEST_DATABASE_URL"); +const embeddedPostgresSupport = external ? { supported: true } : await getEmbeddedPostgresTestSupport(); const describeEmbeddedPostgres = embeddedPostgresSupport.supported ? describe : describe.skip; if (!embeddedPostgresSupport.supported) { @@ -709,8 +711,8 @@ describeEmbeddedPostgres("externalObjectService", () => { let tempDb: Awaited> | null = null; beforeAll(async () => { - tempDb = await startEmbeddedPostgresTestDatabase("paperclip-external-objects-"); - db = createDb(tempDb.connectionString); + if (!external) tempDb = await startEmbeddedPostgresTestDatabase("paperclip-external-objects-"); + db = createDb(external ?? tempDb!.connectionString, { maxConnections: 6 }); }, 20_000); afterEach(async () => { @@ -725,6 +727,7 @@ describeEmbeddedPostgres("externalObjectService", () => { }); afterAll(async () => { + await db?.$client.end({ timeout: 1 }); await tempDb?.cleanup(); }); diff --git a/server/src/__tests__/merged-pr-confirmation-work.test.ts b/server/src/__tests__/merged-pr-confirmation-work.test.ts index 51195e6157b..95565fd96e7 100644 --- a/server/src/__tests__/merged-pr-confirmation-work.test.ts +++ b/server/src/__tests__/merged-pr-confirmation-work.test.ts @@ -1,3 +1,5 @@ +import { createServer } from "node:http"; +import { createPullRequestMergeStateResolver } from "../services/github-pull-request-merge.js"; import { randomUUID } from "node:crypto"; import { setTimeout as delay } from "node:timers/promises"; import { eq, sql } from "drizzle-orm"; @@ -262,6 +264,92 @@ const support = external ? { supported: true } : await getEmbeddedPostgresTestSu expect(wakeup).toHaveBeenCalledTimes(1); expect(errors).toEqual([]); }); + it.each(["headers", "body", "not-found probe", "public visibility probe"] as const)("cancels an actual unresponsive HTTP %s read promptly at stop", async phase => { + vi.useRealTimers(); + const card = await create(); + let observed!: () => void, closed!: () => void; + const requested = new Promise(resolve => { observed = resolve; }); + const disconnected = new Promise(resolve => { closed = resolve; }); + const signals: AbortSignal[] = []; + const server = createServer((request, response) => { + if (phase === "not-found probe" && request.url?.includes("/pulls/")) { response.writeHead(404).end(); return; } + response.on("close", closed); + if (phase === "body") { response.writeHead(200, { "content-type": "application/json" }); response.write('{"merged":'); } + observed(); + }); + await new Promise(resolve => { server.listen(0, "127.0.0.1", resolve); }); + const address = server.address(); + if (!address || typeof address === "string") throw new Error("Expected test HTTP address"); + const resolveState = createPullRequestMergeStateResolver(db, { + requestTimeoutMs: 10_000, + credentialProvider: () => phase === "public visibility probe" ? { token: "fixture", source: "server_env", secretName: null } : null, + fetch: (url, init) => { signals.push(init!.signal!); return fetch(`http://127.0.0.1:${address.port}${new URL(url).pathname}`, init); }, + }); + const real = issueThreadInteractionService(db, { resolvePullRequestState: resolveState, wakeup }); + coordinator = createDeliveryWorkCoordinator({ owner: db, canRun: () => true, onError: error => errors.push(error) }); + try { + const startupAt = performance.now(); + const worker = registerMergeConfirmationWork(coordinator, real, () => true); + expect(performance.now() - startupAt).toBeLessThan(100); + await Promise.race([requested, delay(2000).then(() => { throw new Error("HTTP observation did not start"); })]); + const stoppingAt = performance.now(); + await coordinator.stop(); coordinator = undefined; + expect(performance.now() - stoppingAt).toBeLessThan(1000); + await worker.ready; + await Promise.race([disconnected, delay(1000).then(() => { throw new Error("HTTP request remained open after stop"); })]); + expect(signals.at(-1)?.aborted).toBe(true); + expect((await stored(card.id)).status).toBe("pending"); expect(wakeup).not.toHaveBeenCalled(); expect(errors).toEqual([]); + } finally { + await coordinator?.stop(); coordinator = undefined; + server.closeAllConnections(); await new Promise(resolve => { server.close(() => resolve()); }); + } + }); + + it("times out a real unresponsive HTTP read and recovers its durable candidate on restart", async () => { + vi.useRealTimers(); const card = await create(); + let serveMerge = false, closed!: () => void; + const disconnected = new Promise(resolve => { closed = resolve; }); + const server = createServer((_request, response) => { + if (serveMerge) { response.writeHead(200, { "content-type": "application/json" }).end(JSON.stringify({ merged: true, state: "closed" })); return; } + response.on("close", closed); + }); + await new Promise(resolve => { server.listen(0, "127.0.0.1", resolve); }); + const address = server.address(); if (!address || typeof address === "string") throw new Error("Expected test HTTP address"); + const real = issueThreadInteractionService(db, { resolvePullRequestState: createPullRequestMergeStateResolver(db, { + requestTimeoutMs: 100, tokenProvider: () => null, + fetch: (url, init) => fetch(`http://127.0.0.1:${address.port}${new URL(url).pathname}`, init), + }), wakeup }); + coordinator = createDeliveryWorkCoordinator({ owner: db, canRun: () => true, onError: error => errors.push(error) }); + try { + const startedAt = performance.now(); + await registerMergeConfirmationWork(coordinator, real, () => true).ready; + expect(performance.now() - startedAt).toBeLessThan(2000); + await Promise.race([disconnected, delay(1000).then(() => { throw new Error("Timed out HTTP request remained open"); })]); + expect(errors).toHaveLength(1); expect((await stored(card.id)).status).toBe("pending"); expect(wakeup).not.toHaveBeenCalled(); + expect(coordinator.nextWakeAt()).not.toBeNull(); + await coordinator.stop(); coordinator = undefined; serveMerge = true; + coordinator = createDeliveryWorkCoordinator({ owner: db, canRun: () => true, onError: error => errors.push(error) }); + await registerMergeConfirmationWork(coordinator, real, () => true).ready; + expect((await stored(card.id)).status).toBe("accepted"); expect(wakeup).toHaveBeenCalledTimes(1); + } finally { + await coordinator?.stop(); coordinator = undefined; + server.closeAllConnections(); await new Promise(resolve => { server.close(() => resolve()); }); + } + }); + + it("pages beyond a full pending-card page without duplicating the merge parser in SQL", async () => { + const card = await create(); + const id = BigInt(`0x${card.id.replaceAll("-", "")}`); + await db.insert(issueThreadInteractions).values(Array.from({ length: 100 }, (_, index) => { + const hex = (id - BigInt(index + 1)).toString(16).padStart(32, "0"); + return { id: `${hex.slice(0,8)}-${hex.slice(8,12)}-${hex.slice(12,16)}-${hex.slice(16,20)}-${hex.slice(20)}`, + companyId, issueId, kind: "request_confirmation", title: "Approve the plan?", payload: { version: 1 as const, prompt: "Approve the plan?" } }; + })); + await start(); expect(provider).not.toHaveBeenCalled(); + await waitFor(async () => expect((await stored(card.id)).status).toBe("accepted")); await quiet(); + expect(provider).toHaveBeenCalledTimes(1); expect(wakeup).toHaveBeenCalledTimes(1); expect(errors).toEqual([]); + }); + it("limits normalized merge hints to the owning company and matching PR and tolerates duplicates", async () => { provider.mockResolvedValue("open"); const own = await create(); const foreignCompany = randomUUID(), foreignIssue = randomUUID(); diff --git a/server/src/__tests__/server-startup-feedback-export.test.ts b/server/src/__tests__/server-startup-feedback-export.test.ts index 9fc7234b2e6..4e7e3562156 100644 --- a/server/src/__tests__/server-startup-feedback-export.test.ts +++ b/server/src/__tests__/server-startup-feedback-export.test.ts @@ -180,6 +180,7 @@ const { const heartbeatServiceFactoryMock = vi.fn(() => heartbeatServiceMock); const issueThreadInteractionServiceMock = { sweepSupersededPendingRequestConfirmations: vi.fn(async () => ({ expired: 0 })), + mergedPullRequestConfirmationPage: vi.fn(async () => ({ rows: [], complete: true })), mergedPullRequestConfirmationCandidates: vi.fn(async () => []), sweepMergedPullRequestConfirmations: vi.fn(async () => ({ checked: 0, @@ -1335,7 +1336,7 @@ describe("startServer feedback export wiring", () => { expect(heartbeatServiceMock.tickAgentTimers).toHaveBeenCalledTimes(1); expect(externalObjectsServiceMock.refreshDueObjectsForActiveCompanies).toHaveBeenCalledTimes(1); - expect(issueThreadInteractionServiceMock.mergedPullRequestConfirmationCandidates).toHaveBeenCalledTimes(1); + expect(issueThreadInteractionServiceMock.mergedPullRequestConfirmationPage).toHaveBeenCalledTimes(1); expect(issueThreadInteractionServiceMock.sweepMergedPullRequestConfirmations).not.toHaveBeenCalled(); expect(deliveryWorkRegisterMock.mock.calls.some(([queue]) => queue === DELIVERY_QUEUES.mergedPullRequestConfirmations)).toBe(true); expect(executionWorkspaceServiceMock.sweepTerminalWorkspaces).toHaveBeenCalledTimes(1); @@ -1424,6 +1425,20 @@ describe("startServer feedback export wiring", () => { } }); + it("becomes ready without waiting for the merge-confirmation observation", async () => { + loadConfigMock.mockReturnValue(buildTestConfig({ heartbeatSchedulerEnabled: true })); + let release!: () => void; + const blocked = new Promise<{ rows: never[]; complete: boolean }>(resolve => { release = () => resolve({ rows: [], complete: true }); }); + issueThreadInteractionServiceMock.mergedPullRequestConfirmationPage.mockReturnValueOnce(blocked); + const timer = vi.spyOn(globalThis, "setInterval").mockImplementation(() => 1 as unknown as ReturnType); + let started = false; + const starting = startServer().then(server => { started = true; return server; }); + try { + await vi.waitFor(() => expect(issueThreadInteractionServiceMock.mergedPullRequestConfirmationPage).toHaveBeenCalled()); + await settleStartupHousekeeping(); expect(started).toBe(true); expect(getStartupRecoveryState().phase).toBe("ready"); + } finally { release(); await starting; timer.mockRestore(); } + }); + it.each(["retained cleanup", "workspace finalization", "environment settlement", "native run projection", "native result commit", "native recovery notices", "external conversation settlement"] as const)("becomes ready while %s is held but joins it at shutdown", async owner => { loadConfigMock.mockReturnValue(buildTestConfig({ heartbeatSchedulerEnabled: true })); let releaseCleanup!: () => void; diff --git a/server/src/index.ts b/server/src/index.ts index c1804998ef5..86074131dc3 100644 --- a/server/src/index.ts +++ b/server/src/index.ts @@ -1601,8 +1601,8 @@ async function startServerWithDatabaseTeardown( : config.workspaceCompactionDelayHours * 60 * 60 * 1000, terminalWorkspaceInspectionBackoffMs: TERMINAL_WORKSPACE_RETRY_MS, }); - await registerMergeConfirmationWork(deliveryWork, mergedPullRequestConfirmations, - () => !isIdleTaskDrainActive() && !heartbeatSchedulerStopped && !isWarmStandby()).ready; + registerMergeConfirmationWork(deliveryWork, mergedPullRequestConfirmations, + () => !isIdleTaskDrainActive() && !heartbeatSchedulerStopped && !isWarmStandby()); const gitRestorePointWorker = registerGitRestorePointWork(deliveryWork, gitRestorePointRetentionService(db as any, { storage: storageService, retentionDays: config.gitRestorePointRetentionDays, diff --git a/server/src/services/external-objects.ts b/server/src/services/external-objects.ts index 56a8fb91834..306f043d753 100644 --- a/server/src/services/external-objects.ts +++ b/server/src/services/external-objects.ts @@ -84,6 +84,7 @@ export interface ExternalObjectResolver { resolve(input: { companyId: string; object: ExternalObjectRecord; + signal?: AbortSignal; }): Promise; } diff --git a/server/src/services/github-external-object-provider.ts b/server/src/services/github-external-object-provider.ts index 6dedac97efb..83b47354035 100644 --- a/server/src/services/github-external-object-provider.ts +++ b/server/src/services/github-external-object-provider.ts @@ -15,6 +15,7 @@ type FetchLike = (url: string, init?: RequestInit) => Promise; export interface GitHubExternalObjectProviderOptions { fetch?: FetchLike; + requestTimeoutMs?: number; tokenProvider?: (companyId: string) => Promise | string | null; /** * Resolve the credential and its source. Prefer this over `tokenProvider` when the @@ -243,12 +244,12 @@ type RepoProbeOutcome = { async function probeRepoVisibility( identity: GitHubObjectIdentity, - opts: { fetch: FetchLike; headers?: Record | undefined }, + opts: { fetch: FetchLike; headers?: Record | undefined; signal?: AbortSignal }, ): Promise { try { const probe = await opts.fetch( `${gitHubApiBase(identity.host)}/repos/${encodeURIComponent(identity.owner)}/${encodeURIComponent(identity.repo)}`, - { headers: opts.headers }, + { headers: opts.headers, signal: opts.signal }, ); if (probe.ok) { const body = await safeJson(probe); @@ -427,12 +428,13 @@ export function createGitHubExternalObjectProvider( companyId: string, identity: GitHubObjectIdentity, headers: Record, + signal: AbortSignal, ): Promise { const credentialFingerprint = createHash("sha256").update(headers.authorization ?? "anonymous").digest("hex"); const key = `${companyId}\u0000${credentialFingerprint}\u0000${identity.owner.toLowerCase()}\u0000${identity.repo.toLowerCase()}`; const cached = probeCache.get(key); if (cached && cached.expiresAt > Date.now()) return cached.outcome; - const outcome = await probeRepoVisibility(identity, { fetch: fetchImpl, headers }); + const outcome = await probeRepoVisibility(identity, { fetch: fetchImpl, headers, signal }); if (outcome.visibility === "visible" || outcome.visibility === "hidden") { probeCache.set(key, { outcome, expiresAt: Date.now() + GITHUB_OBJECT_TTL_SECONDS * 1000 }); } @@ -464,7 +466,9 @@ export function createGitHubExternalObjectProvider( return { providerKey: "github", objectType, - async resolve({ companyId, object }) { + async resolve({ companyId, object, signal: ownerSignal }) { + const deadline = AbortSignal.timeout(opts.requestTimeoutMs ?? 10_000); + const signal = ownerSignal ? AbortSignal.any([ownerSignal, deadline]) : deadline; const identity = parseGitHubObject(object); if (!identity || identity.objectType !== objectType) { return { @@ -509,7 +513,7 @@ export function createGitHubExternalObjectProvider( // "public" verdict must not authorize a later read of its private contents. // The cache is still used below to classify a 404, where a stale verdict can // only make the object look deleted — it can never expose private data. - const probe = await probeRepoVisibility(identity, { fetch: fetchImpl, headers }); + const probe = await probeRepoVisibility(identity, { fetch: fetchImpl, headers, signal }); if (probe.visibility === "hidden") { return { ok: false, @@ -554,7 +558,7 @@ export function createGitHubExternalObjectProvider( let response: Response; try { - response = await fetchImpl(url, { headers }); + response = await fetchImpl(url, { headers, signal }); } catch { return { ok: false, @@ -573,7 +577,7 @@ export function createGitHubExternalObjectProvider( // Probe the repository metadata to classify: if the repo itself is visible // the object truly is gone; if it is not, surface credentials-needed // instead of permanently freezing a reference as "Not found". - const probe = await cachedProbeRepoVisibility(companyId, identity, headers); + const probe = await cachedProbeRepoVisibility(companyId, identity, headers, signal); if (probe.visibility === "visible") { return { ok: true, snapshot: notFoundSnapshot(identity, etag) }; } diff --git a/server/src/services/github-pull-request-merge.ts b/server/src/services/github-pull-request-merge.ts index bb92d37d7c2..1cdff01dc1f 100644 --- a/server/src/services/github-pull-request-merge.ts +++ b/server/src/services/github-pull-request-merge.ts @@ -1,5 +1,5 @@ import type { Db } from "@paperclipai/db"; -import { createGitHubExternalObjectProvider } from "./github-external-object-provider.js"; +import { createGitHubExternalObjectProvider, type GitHubExternalObjectProviderOptions } from "./github-external-object-provider.js"; export type GitHubPullRequestReference = { host: "github.com"; @@ -25,11 +25,13 @@ export type PullRequestMergeDetails = { export type PullRequestMergeStateResolver = ( companyId: string, reference: GitHubPullRequestReference, + signal?: AbortSignal, ) => Promise; export type PullRequestMergeDetailsResolver = ( companyId: string, reference: GitHubPullRequestReference, + signal?: AbortSignal, ) => Promise; export const PULL_REQUEST_CACHE_MAX_ENTRIES = 1_000; @@ -86,14 +88,15 @@ function readRecord(value: unknown): Record | null { : null; } -export function createPullRequestMergeDetailsResolver(db: Db): PullRequestMergeDetailsResolver { - const resolver = createGitHubExternalObjectProvider(db).resolvers +export function createPullRequestMergeDetailsResolver(db: Db, options?: GitHubExternalObjectProviderOptions): PullRequestMergeDetailsResolver { + const resolver = createGitHubExternalObjectProvider(db, options).resolvers .find((candidate) => candidate.objectType === "pull_request") ?? null; - return async (companyId, reference) => { + return async (companyId, reference, signal) => { if (!resolver) return { state: "unknown", headRef: null, headSha: null }; const result = await resolver.resolve({ companyId, + signal, object: { externalId: `${reference.owner}/${reference.repo}#pull/${reference.number}`, sanitizedCanonicalUrl: `https://github.com/${reference.owner}/${reference.repo}/pull/${reference.number}`, @@ -123,7 +126,7 @@ export function createPullRequestMergeDetailsResolver(db: Db): PullRequestMergeD }; } -export function createPullRequestMergeStateResolver(db: Db): PullRequestMergeStateResolver { - const resolveDetails = createPullRequestMergeDetailsResolver(db); - return async (companyId, reference) => (await resolveDetails(companyId, reference)).state; +export function createPullRequestMergeStateResolver(db: Db, options?: GitHubExternalObjectProviderOptions): PullRequestMergeStateResolver { + const resolveDetails = createPullRequestMergeDetailsResolver(db, options); + return async (companyId, reference, signal) => (await resolveDetails(companyId, reference, signal)).state; } diff --git a/server/src/services/issue-thread-interactions.ts b/server/src/services/issue-thread-interactions.ts index b306f637e27..b49ca48d5c1 100644 --- a/server/src/services/issue-thread-interactions.ts +++ b/server/src/services/issue-thread-interactions.ts @@ -204,6 +204,7 @@ export type IssueThreadInteractionServiceOptions = { resolvePullRequestState?: ( companyId: string, reference: GitHubPullRequestReference, + signal?: AbortSignal, ) => Promise; wakeup?: InteractionWakeup; pullRequestCacheTtlMs?: number; @@ -2045,10 +2046,11 @@ export function issueThreadInteractionService( async function resolvePullRequestState( companyId: string, reference: GitHubPullRequestReference, + signal?: AbortSignal, ): Promise { if (opts.resolvePullRequestState) - return opts.resolvePullRequestState(companyId, reference); - return defaultPullRequestStateResolver?.(companyId, reference) ?? "unknown"; + return opts.resolvePullRequestState(companyId, reference, signal); + return defaultPullRequestStateResolver?.(companyId, reference, signal) ?? "unknown"; } async function resolvePullRequestStates( @@ -2057,6 +2059,7 @@ export function issueThreadInteractionService( companyId: string; reference: GitHubPullRequestReference; }>, + signal?: AbortSignal, ) { const states = new Map(); const pending = entries.slice(); @@ -2064,11 +2067,13 @@ export function issueThreadInteractionService( await Promise.all( Array.from({ length: workerCount }, async () => { while (pending.length > 0) { + signal?.throwIfAborted(); const entry = pending.shift(); if (!entry) return; const state = await resolvePullRequestState( entry.companyId, entry.reference, + signal, ); states.set(entry.key, state); } @@ -2076,11 +2081,7 @@ export function issueThreadInteractionService( ); return states; } - async function mergedPullRequestConfirmationCandidates(scope?: MergeConfirmationScope) { - const text = sql`concat_ws(' ', ${issueThreadInteractions.title}, ${issueThreadInteractions.summary}, - ${issueThreadInteractions.payload}->>'prompt', ${issueThreadInteractions.payload}->>'acceptLabel', - ${issueThreadInteractions.payload}->>'detailsMarkdown', ${issueThreadInteractions.payload}->'target'->>'key', - ${issueThreadInteractions.payload}->'target'->>'label', ${issueThreadInteractions.payload}->'target'->>'href')`; + async function mergedPullRequestConfirmationPage(scope?: MergeConfirmationScope, after?: string) { const rows = await db.select({ interaction: issueThreadInteractions, issue: { id: issues.id, companyId: issues.companyId, projectId: issues.projectId, goalId: issues.goalId, status: issues.status, assigneeAgentId: issues.assigneeAgentId, @@ -2092,14 +2093,24 @@ export function issueThreadInteractionService( scope ? eq(issueThreadInteractions.companyId, scope.companyId) : undefined, scope?.issueId ? eq(issueThreadInteractions.issueId, scope.issueId) : undefined, scope?.interactionId ? eq(issueThreadInteractions.id, scope.interactionId) : undefined, - sql`${text} ~* '(merge|merging)'`, - sql`${text} ~* '(github\\.com/[^ /]+/[^ /]+/pull/[1-9][0-9]*|[a-z0-9_.-]+/[a-z0-9_.-]+#[1-9][0-9]*)'`, + after ? sql`${issueThreadInteractions.id} > ${after}::uuid` : undefined, sql`NOT EXISTS (SELECT 1 FROM ${toolActionRequests} WHERE ${toolActionRequests.interactionId} = ${issueThreadInteractions.id})`, - )); - return rows.flatMap(row => { + )).orderBy(asc(issueThreadInteractions.id)).limit(100); + return { rows: rows.flatMap(row => { const references = getMergeConfirmationPullRequestReferences(row.interaction); return references.length ? [{ ...row, references }] : []; - }); + }), after: rows.at(-1)?.interaction.id, complete: rows.length < 100 }; + } + + async function mergedPullRequestConfirmationCandidates(scope?: MergeConfirmationScope) { + const candidates: Awaited>["rows"] = []; + let after: string | undefined; + while (true) { + const page = await mergedPullRequestConfirmationPage(scope, after); + candidates.push(...page.rows); + if (page.complete) return candidates; + after = page.after; + } } async function getIdempotentInteraction(args: { @@ -3012,6 +3023,7 @@ export function issueThreadInteractionService( await emitInteractionResolvedTelemetry(db, interaction); return interaction; }, + mergedPullRequestConfirmationPage, mergedPullRequestConfirmationCandidates, sweepMergedPullRequestConfirmations: async ( mergedHints: Array<{ @@ -3064,7 +3076,7 @@ export function issueThreadInteractionService( const refreshedStates = await resolvePullRequestStates([ ...uniqueReferences.values(), - ]); + ], signal); if (scope && [...refreshedStates.values()].some(state => state === "unknown")) { throw new Error("Merge confirmation provider state is unavailable"); } diff --git a/server/src/services/merged-pr-confirmation-work.ts b/server/src/services/merged-pr-confirmation-work.ts index bfccaeea79f..6a754d5a88a 100644 --- a/server/src/services/merged-pr-confirmation-work.ts +++ b/server/src/services/merged-pr-confirmation-work.ts @@ -9,16 +9,24 @@ export const MERGE_CONFIRMATION_RECONCILE_MS = 5 * 60_000; export function registerMergeConfirmationWork( coordinator: ReturnType, service: Pick, - "mergedPullRequestConfirmationCandidates" | "sweepMergedPullRequestConfirmations">, + "mergedPullRequestConfirmationCandidates" | "mergedPullRequestConfirmationPage" | "sweepMergedPullRequestConfirmations">, canRun: () => boolean, ) { + const cursors = new Map(); return registerScopedDeadlineWork(coordinator, { queue: DELIVERY_QUEUES.mergedPullRequestConfirmations, decodeScope: hint => hint === "*" ? undefined : JSON.parse(hint) as MergeConfirmationScope, matches: (entry, scope?: MergeConfirmationScope) => !scope || (entry.companyId === scope.companyId && (!scope.issueId || entry.issueId === scope.issueId) && (!scope.interactionId || entry.id === scope.interactionId)), - load: async (scope?: MergeConfirmationScope) => (await service.mergedPullRequestConfirmationCandidates(scope)) - .map(({ interaction }) => ({ id: interaction.id, companyId: interaction.companyId, issueId: interaction.issueId, dueAt: Date.now() })), + load: async (scope?: MergeConfirmationScope) => { + const key = scope ? JSON.stringify(scope) : "*"; + const page = await service.mergedPullRequestConfirmationPage(scope, cursors.get(key)); + if (page.complete) cursors.delete(key); + else cursors.set(key, page.after!); + return { complete: page.complete, rows: page.rows.map(({ interaction }) => ({ + id: interaction.id, companyId: interaction.companyId, issueId: interaction.issueId, dueAt: Date.now(), + })) }; + }, check: async (entry, signal) => { const scope = { companyId: entry.companyId, issueId: entry.issueId, interactionId: entry.id }; await service.sweepMergedPullRequestConfirmations([], scope, signal); From 1b12f8f732bd1c6d848797c145b1768f5b76e3a1 Mon Sep 17 00:00:00 2001 From: Dotta Date: Fri, 30 Oct 2026 16:54:51 -0500 Subject: [PATCH 11/74] test(skills): verify Cloud CLI reads through shared object storage Use the real S3 provider with a controlled HTTP fixture and a stable test key. Preserve read-only/no-seeding behavior and verify corrupt, missing, and restored object outcomes under managed Cloud policy. Co-Authored-By: Paperclip --- .../__tests__/managed-skill-content.test.ts | 90 ++++++++++++++++--- 1 file changed, 77 insertions(+), 13 deletions(-) diff --git a/server/src/__tests__/managed-skill-content.test.ts b/server/src/__tests__/managed-skill-content.test.ts index cd4ddb8135f..59566d743e3 100644 --- a/server/src/__tests__/managed-skill-content.test.ts +++ b/server/src/__tests__/managed-skill-content.test.ts @@ -1,3 +1,5 @@ +import { createServer } from "node:http"; +import { createS3StorageProvider } from "../storage/s3-provider.js"; import { skillInstallationMaintenance } from "../services/skill-installation-maintenance.js"; import { removeRuntimeAssetTree } from "../services/native-runtime/runtime-asset-retention.js"; import { eq } from "drizzle-orm"; @@ -79,26 +81,88 @@ describe.skipIf(!support.supported)("managed skills use committed versions", () it("loads persisted Cloud configuration for read-only maintenance without seeding it", async () => { const companyId = await company(); - await companySkillService(db).createLocalSkill(companyId, { name: "Cloud CLI", markdown: "# Cloud CLI" }); + const objects = new Map(); + const server = createServer((request, response) => { + void (async () => { + const key = new URL(request.url!, "http://fixture").pathname; + if (request.method === "PUT") { + const chunks: Buffer[] = []; + for await (const chunk of request) chunks.push(Buffer.from(chunk)); + let body = Buffer.concat(chunks); + if (request.headers["content-encoding"]?.includes("aws-chunked")) { + const decoded: Buffer[] = []; + let offset = 0; + while (offset < body.length) { + const end = body.indexOf("\r\n", offset); + const size = Number.parseInt(body.subarray(offset, end).toString().split(";")[0]!, 16); + if (size === 0) break; + decoded.push(body.subarray(end + 2, end + 2 + size)); offset = end + 2 + size + 2; + } + body = Buffer.concat(decoded); + } + objects.set(key, body); response.writeHead(200, { etag: '"fixture"' }).end(); return; + } + const body = objects.get(key); + if (!body) { response.writeHead(404).end(); return; } + response.writeHead(200, { "content-length": body.length, "content-type": "application/octet-stream", etag: '"fixture"' }); + response.end(request.method === "HEAD" ? undefined : body); + })().catch(() => { response.writeHead(500).end(); }); + }); + await new Promise(resolve => { server.listen(0, "127.0.0.1", resolve); }); + const address = server.address(); if (!address || typeof address === "string") throw new Error("Expected fixture address"); + const endpoint = `http://127.0.0.1:${address.port}`; + const credentials = { AWS_ACCESS_KEY_ID: "fixture-access-key", AWS_SECRET_ACCESS_KEY: "fixture-secret-key", AWS_SESSION_TOKEN: "" }; + const previousCredentials = Object.fromEntries(Object.keys(credentials).map(key => [key, process.env[key]])); + Object.assign(process.env, credentials); + vi.mocked(storageModule.getStorageService).mockReturnValue(createStorageService(createS3StorageProvider({ + bucket: "managed-skill-fixture", region: "us-east-1", endpoint, forcePathStyle: true, + }))); const key = "cloud-runtime-config:v1"; - const run = (phase: string) => promisify(execFile)(process.execPath, [ - "--import", fileURLToPath(import.meta.resolve("tsx")), - fileURLToPath(new URL("../../scripts/migrate-skill-packages.ts", import.meta.url)), - `--phase=${phase}`, `--company=${companyId}`, "--limit=50", - ], { env: { ...process.env, DATABASE_URL: database.connectionString, - PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN: "disposable-cloud-cli-fixture", - PAPERCLIP_STORAGE_PROVIDER: "local_disk", PAPERCLIP_STORAGE_LOCAL_DIR: path.join(root, "objects") }, - timeout: 60_000, maxBuffer: 1024 * 1024 }); - await expect(run("eligibility")).rejects.toThrow("Initialize Cloud runtime configuration"); - expect(await db.select().from(instanceSettings).where(eq(instanceSettings.singletonKey, key))).toHaveLength(0); - await db.insert(instanceSettings).values({ singletonKey: key, general: { version: 1, revision: 0, config: null } }); try { + await companySkillService(db).createLocalSkill(companyId, { name: "Cloud CLI", markdown: "# Cloud CLI" }); + const run = (phase: string) => promisify(execFile)(process.execPath, [ + "--import", fileURLToPath(import.meta.resolve("tsx")), + fileURLToPath(new URL("../../scripts/migrate-skill-packages.ts", import.meta.url)), + `--phase=${phase}`, `--company=${companyId}`, "--limit=50", + ], { env: { ...process.env, ...credentials, DATABASE_URL: database.connectionString, + PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN: "disposable-cloud-cli-fixture", + PAPERCLIP_SECRETS_MASTER_KEY: "a".repeat(64), PAPERCLIP_STORAGE_PROVIDER: "s3", + PAPERCLIP_STORAGE_S3_BUCKET: "managed-skill-fixture", PAPERCLIP_STORAGE_S3_REGION: "us-east-1", + PAPERCLIP_STORAGE_S3_ENDPOINT: endpoint, PAPERCLIP_STORAGE_S3_FORCE_PATH_STYLE: "true", PAPERCLIP_STORAGE_S3_PREFIX: "" }, + timeout: 60_000, maxBuffer: 1024 * 1024 }); + await expect(run("eligibility")).rejects.toThrow("Initialize Cloud runtime configuration"); + expect(await db.select().from(instanceSettings).where(eq(instanceSettings.singletonKey, key))).toHaveLength(0); + await db.insert(instanceSettings).values({ singletonKey: key, general: { version: 1, revision: 0, config: null } }); const eligibility = JSON.parse((await run("eligibility")).stdout.trim().split("\n").at(-1)!); expect(eligibility.results).toEqual([expect.objectContaining({ status: "version_backed" })]); + const before = [...objects.entries()].map(([key, body]) => [key, Buffer.from(body)] as const); const verification = JSON.parse((await run("verification")).stdout.trim().split("\n").at(-1)!); - expect(verification.results).toEqual([expect.objectContaining({ status: "local_storage" })]); + expect(verification.results).toEqual([expect.objectContaining({ status: "remote_storage" })]); + expect([...objects.entries()]).toEqual(before); + const archiveKey = [...objects.keys()].find(key => key.endsWith("/files.tar.gz"))!; + const archive = objects.get(archiveKey)!; + const corrupted = Buffer.from(archive); corrupted[0] = corrupted[0]! ^ 1; + objects.set(archiveKey, corrupted); + const corrupt = JSON.parse((await run("verification")).stdout.trim().split("\n").at(-1)!); + expect(corrupt.results).toEqual([expect.objectContaining({ status: "corrupt_package" })]); + objects.set(archiveKey, archive); + const manifestKey = [...objects.keys()].find(key => key.endsWith("/manifest.json"))!; + const manifest = objects.get(manifestKey)!; objects.delete(manifestKey); + const unavailable = JSON.parse((await run("verification")).stdout.trim().split("\n").at(-1)!); + expect(unavailable.results).toEqual([expect.objectContaining({ status: "unavailable_or_corrupt_package" })]); + objects.set(manifestKey, manifest); + const recovered = JSON.parse((await run("verification")).stdout.trim().split("\n").at(-1)!); + expect(recovered.results).toEqual([expect.objectContaining({ status: "remote_storage" })]); + expect([...objects.entries()].sort()).toEqual([...before].sort()); + expect(await db.select().from(instanceSettings).where(eq(instanceSettings.singletonKey, key))) + .toEqual([expect.objectContaining({ general: { version: 1, revision: 0, config: null } })]); } finally { await db.delete(instanceSettings).where(eq(instanceSettings.singletonKey, key)); + vi.mocked(storageModule.getStorageService).mockReturnValue(createStorageService(createLocalDiskStorageProvider(path.join(root, "objects")))); + for (const [key, value] of Object.entries(previousCredentials)) { + if (value === undefined) delete process.env[key]; else process.env[key] = value; + } + server.closeAllConnections(); await new Promise(resolve => { server.close(() => resolve()); }); } }, 180_000); From 51c554275e8786c39c186e008c5d83c0e3ba1b2a Mon Sep 17 00:00:00 2001 From: Dotta Date: Fri, 30 Oct 2026 16:55:32 -0500 Subject: [PATCH 12/74] docs: record bounded HTTP and Cloud fixture verification Co-Authored-By: Paperclip --- .../2026-10-30-merged-pr-confirmations-sleep.md | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/doc/plans/2026-10-30-merged-pr-confirmations-sleep.md b/doc/plans/2026-10-30-merged-pr-confirmations-sleep.md index 7b8699456fb..1c691c19509 100644 --- a/doc/plans/2026-10-30-merged-pr-confirmations-sleep.md +++ b/doc/plans/2026-10-30-merged-pr-confirmations-sleep.md @@ -91,9 +91,18 @@ The original head passed full-workspace typecheck and build. A later required Linux CI shard failed the unchanged managed-skill maintenance CLI fixture in `managed-skill-content.test.ts`: it returned `unavailable_or_corrupt_package` where `local_storage` was expected. Both the isolated case and the complete -unchanged 17-test file pass against fresh local PostgreSQL. The Linux cause is -not established; this is retained as an unresolved CI failure, not called a flake. -The current-head required CI result remains the completion gate. +unchanged 17-test file pass against fresh local PostgreSQL. The original Linux cause is +not established; that failure is retained and is not called a flake. + +After merging current master, the same fixture failed locally because the new +managed-Cloud policy from PR #18350 correctly refuses local disk. A separate +test-only prerequisite commit changes that fixture to the real S3 provider with +a controlled HTTP endpoint, valid fixture credentials, and a stable test key. +It verifies remote reads without seeding or mutating runtime configuration, +rejects corrupt and missing objects, and recovers after restoring canonical bytes. +The full existing 17-test file passes with these failure controls. Production +configuration validation remains unchanged. Current-head required CI remains +the completion gate. ## Existing limitations outside this migration From bdb9b66ac6d8c9e3ac81131513379e5a0d547e39 Mon Sep 17 00:00:00 2001 From: Dotta Date: Fri, 30 Oct 2026 16:54:51 -0500 Subject: [PATCH 13/74] test(skills): verify Cloud CLI reads through shared object storage Use the real S3 provider with a controlled HTTP fixture and a stable test key. Preserve read-only/no-seeding behavior and verify corrupt, missing, and restored object outcomes under managed Cloud policy. Co-Authored-By: Paperclip --- .../__tests__/managed-skill-content.test.ts | 90 ++++++++++++++++--- 1 file changed, 77 insertions(+), 13 deletions(-) diff --git a/server/src/__tests__/managed-skill-content.test.ts b/server/src/__tests__/managed-skill-content.test.ts index cd4ddb8135f..59566d743e3 100644 --- a/server/src/__tests__/managed-skill-content.test.ts +++ b/server/src/__tests__/managed-skill-content.test.ts @@ -1,3 +1,5 @@ +import { createServer } from "node:http"; +import { createS3StorageProvider } from "../storage/s3-provider.js"; import { skillInstallationMaintenance } from "../services/skill-installation-maintenance.js"; import { removeRuntimeAssetTree } from "../services/native-runtime/runtime-asset-retention.js"; import { eq } from "drizzle-orm"; @@ -79,26 +81,88 @@ describe.skipIf(!support.supported)("managed skills use committed versions", () it("loads persisted Cloud configuration for read-only maintenance without seeding it", async () => { const companyId = await company(); - await companySkillService(db).createLocalSkill(companyId, { name: "Cloud CLI", markdown: "# Cloud CLI" }); + const objects = new Map(); + const server = createServer((request, response) => { + void (async () => { + const key = new URL(request.url!, "http://fixture").pathname; + if (request.method === "PUT") { + const chunks: Buffer[] = []; + for await (const chunk of request) chunks.push(Buffer.from(chunk)); + let body = Buffer.concat(chunks); + if (request.headers["content-encoding"]?.includes("aws-chunked")) { + const decoded: Buffer[] = []; + let offset = 0; + while (offset < body.length) { + const end = body.indexOf("\r\n", offset); + const size = Number.parseInt(body.subarray(offset, end).toString().split(";")[0]!, 16); + if (size === 0) break; + decoded.push(body.subarray(end + 2, end + 2 + size)); offset = end + 2 + size + 2; + } + body = Buffer.concat(decoded); + } + objects.set(key, body); response.writeHead(200, { etag: '"fixture"' }).end(); return; + } + const body = objects.get(key); + if (!body) { response.writeHead(404).end(); return; } + response.writeHead(200, { "content-length": body.length, "content-type": "application/octet-stream", etag: '"fixture"' }); + response.end(request.method === "HEAD" ? undefined : body); + })().catch(() => { response.writeHead(500).end(); }); + }); + await new Promise(resolve => { server.listen(0, "127.0.0.1", resolve); }); + const address = server.address(); if (!address || typeof address === "string") throw new Error("Expected fixture address"); + const endpoint = `http://127.0.0.1:${address.port}`; + const credentials = { AWS_ACCESS_KEY_ID: "fixture-access-key", AWS_SECRET_ACCESS_KEY: "fixture-secret-key", AWS_SESSION_TOKEN: "" }; + const previousCredentials = Object.fromEntries(Object.keys(credentials).map(key => [key, process.env[key]])); + Object.assign(process.env, credentials); + vi.mocked(storageModule.getStorageService).mockReturnValue(createStorageService(createS3StorageProvider({ + bucket: "managed-skill-fixture", region: "us-east-1", endpoint, forcePathStyle: true, + }))); const key = "cloud-runtime-config:v1"; - const run = (phase: string) => promisify(execFile)(process.execPath, [ - "--import", fileURLToPath(import.meta.resolve("tsx")), - fileURLToPath(new URL("../../scripts/migrate-skill-packages.ts", import.meta.url)), - `--phase=${phase}`, `--company=${companyId}`, "--limit=50", - ], { env: { ...process.env, DATABASE_URL: database.connectionString, - PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN: "disposable-cloud-cli-fixture", - PAPERCLIP_STORAGE_PROVIDER: "local_disk", PAPERCLIP_STORAGE_LOCAL_DIR: path.join(root, "objects") }, - timeout: 60_000, maxBuffer: 1024 * 1024 }); - await expect(run("eligibility")).rejects.toThrow("Initialize Cloud runtime configuration"); - expect(await db.select().from(instanceSettings).where(eq(instanceSettings.singletonKey, key))).toHaveLength(0); - await db.insert(instanceSettings).values({ singletonKey: key, general: { version: 1, revision: 0, config: null } }); try { + await companySkillService(db).createLocalSkill(companyId, { name: "Cloud CLI", markdown: "# Cloud CLI" }); + const run = (phase: string) => promisify(execFile)(process.execPath, [ + "--import", fileURLToPath(import.meta.resolve("tsx")), + fileURLToPath(new URL("../../scripts/migrate-skill-packages.ts", import.meta.url)), + `--phase=${phase}`, `--company=${companyId}`, "--limit=50", + ], { env: { ...process.env, ...credentials, DATABASE_URL: database.connectionString, + PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN: "disposable-cloud-cli-fixture", + PAPERCLIP_SECRETS_MASTER_KEY: "a".repeat(64), PAPERCLIP_STORAGE_PROVIDER: "s3", + PAPERCLIP_STORAGE_S3_BUCKET: "managed-skill-fixture", PAPERCLIP_STORAGE_S3_REGION: "us-east-1", + PAPERCLIP_STORAGE_S3_ENDPOINT: endpoint, PAPERCLIP_STORAGE_S3_FORCE_PATH_STYLE: "true", PAPERCLIP_STORAGE_S3_PREFIX: "" }, + timeout: 60_000, maxBuffer: 1024 * 1024 }); + await expect(run("eligibility")).rejects.toThrow("Initialize Cloud runtime configuration"); + expect(await db.select().from(instanceSettings).where(eq(instanceSettings.singletonKey, key))).toHaveLength(0); + await db.insert(instanceSettings).values({ singletonKey: key, general: { version: 1, revision: 0, config: null } }); const eligibility = JSON.parse((await run("eligibility")).stdout.trim().split("\n").at(-1)!); expect(eligibility.results).toEqual([expect.objectContaining({ status: "version_backed" })]); + const before = [...objects.entries()].map(([key, body]) => [key, Buffer.from(body)] as const); const verification = JSON.parse((await run("verification")).stdout.trim().split("\n").at(-1)!); - expect(verification.results).toEqual([expect.objectContaining({ status: "local_storage" })]); + expect(verification.results).toEqual([expect.objectContaining({ status: "remote_storage" })]); + expect([...objects.entries()]).toEqual(before); + const archiveKey = [...objects.keys()].find(key => key.endsWith("/files.tar.gz"))!; + const archive = objects.get(archiveKey)!; + const corrupted = Buffer.from(archive); corrupted[0] = corrupted[0]! ^ 1; + objects.set(archiveKey, corrupted); + const corrupt = JSON.parse((await run("verification")).stdout.trim().split("\n").at(-1)!); + expect(corrupt.results).toEqual([expect.objectContaining({ status: "corrupt_package" })]); + objects.set(archiveKey, archive); + const manifestKey = [...objects.keys()].find(key => key.endsWith("/manifest.json"))!; + const manifest = objects.get(manifestKey)!; objects.delete(manifestKey); + const unavailable = JSON.parse((await run("verification")).stdout.trim().split("\n").at(-1)!); + expect(unavailable.results).toEqual([expect.objectContaining({ status: "unavailable_or_corrupt_package" })]); + objects.set(manifestKey, manifest); + const recovered = JSON.parse((await run("verification")).stdout.trim().split("\n").at(-1)!); + expect(recovered.results).toEqual([expect.objectContaining({ status: "remote_storage" })]); + expect([...objects.entries()].sort()).toEqual([...before].sort()); + expect(await db.select().from(instanceSettings).where(eq(instanceSettings.singletonKey, key))) + .toEqual([expect.objectContaining({ general: { version: 1, revision: 0, config: null } })]); } finally { await db.delete(instanceSettings).where(eq(instanceSettings.singletonKey, key)); + vi.mocked(storageModule.getStorageService).mockReturnValue(createStorageService(createLocalDiskStorageProvider(path.join(root, "objects")))); + for (const [key, value] of Object.entries(previousCredentials)) { + if (value === undefined) delete process.env[key]; else process.env[key] = value; + } + server.closeAllConnections(); await new Promise(resolve => { server.close(() => resolve()); }); } }, 180_000); From 9e530436ac7cd1da94aa6e67b756cc2b3a6b66ba Mon Sep 17 00:00:00 2001 From: Dotta Date: Fri, 30 Oct 2026 16:59:00 -0500 Subject: [PATCH 14/74] fix(agents): revalidate error-clear authority before write intent Co-Authored-By: Paperclip --- .../agent-lifecycle/adapters/records.ts | 23 ++++++++----------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/server/src/modules/agent-lifecycle/adapters/records.ts b/server/src/modules/agent-lifecycle/adapters/records.ts index db842941836..96892987b01 100644 --- a/server/src/modules/agent-lifecycle/adapters/records.ts +++ b/server/src/modules/agent-lifecycle/adapters/records.ts @@ -81,22 +81,19 @@ export function agentRecords(db: Db, effects: LifecycleEffects) { } const updated = await effects.transaction(db, existing.companyId, async tx => { + const [current] = await tx.select({ status: agents.status, lifecycleState: agents.lifecycleState }) + .from(agents).where(and(eq(agents.id, id), eq(agents.companyId, existing.companyId))).for("no key update"); + if (current?.status !== "error" || current.lifecycleState !== "ready") { + throw conflict("Only agents in error status can have their error cleared"); + } await effects.beforeStatusChange(tx, existing.companyId, id, "error", "idle"); - return tx.update(agents).set({ - status: "idle", - pauseReason: null, - pausedAt: null, - errorReason: null, - updatedAt: new Date(), - }) - .where(and(eq(agents.id, id), eq(agents.status, "error"), eq(agents.lifecycleState, "ready"))) - .returning() - .then((rows) => rows[0] ?? null); + const [saved] = await tx.update(agents).set({ + status: "idle", pauseReason: null, pausedAt: null, errorReason: null, updatedAt: new Date(), + }).where(and(eq(agents.id, id), eq(agents.status, "error"), eq(agents.lifecycleState, "ready"))).returning(); + if (!saved) throw conflict("Only agents in error status can have their error cleared"); + return saved; }); - if (!updated) { - throw conflict("Only agents in error status can have their error cleared"); - } return getById(updated.id); }, From 0136ff7708d775ff7d9600dbf72cf4c7d263516b Mon Sep 17 00:00:00 2001 From: Dotta Date: Fri, 30 Oct 2026 16:54:51 -0500 Subject: [PATCH 15/74] test(skills): verify Cloud CLI reads through shared object storage Use the real S3 provider with a controlled HTTP fixture and a stable test key. Preserve read-only/no-seeding behavior and verify corrupt, missing, and restored object outcomes under managed Cloud policy. Co-Authored-By: Paperclip --- .../__tests__/managed-skill-content.test.ts | 90 ++++++++++++++++--- 1 file changed, 77 insertions(+), 13 deletions(-) diff --git a/server/src/__tests__/managed-skill-content.test.ts b/server/src/__tests__/managed-skill-content.test.ts index cd4ddb8135f..59566d743e3 100644 --- a/server/src/__tests__/managed-skill-content.test.ts +++ b/server/src/__tests__/managed-skill-content.test.ts @@ -1,3 +1,5 @@ +import { createServer } from "node:http"; +import { createS3StorageProvider } from "../storage/s3-provider.js"; import { skillInstallationMaintenance } from "../services/skill-installation-maintenance.js"; import { removeRuntimeAssetTree } from "../services/native-runtime/runtime-asset-retention.js"; import { eq } from "drizzle-orm"; @@ -79,26 +81,88 @@ describe.skipIf(!support.supported)("managed skills use committed versions", () it("loads persisted Cloud configuration for read-only maintenance without seeding it", async () => { const companyId = await company(); - await companySkillService(db).createLocalSkill(companyId, { name: "Cloud CLI", markdown: "# Cloud CLI" }); + const objects = new Map(); + const server = createServer((request, response) => { + void (async () => { + const key = new URL(request.url!, "http://fixture").pathname; + if (request.method === "PUT") { + const chunks: Buffer[] = []; + for await (const chunk of request) chunks.push(Buffer.from(chunk)); + let body = Buffer.concat(chunks); + if (request.headers["content-encoding"]?.includes("aws-chunked")) { + const decoded: Buffer[] = []; + let offset = 0; + while (offset < body.length) { + const end = body.indexOf("\r\n", offset); + const size = Number.parseInt(body.subarray(offset, end).toString().split(";")[0]!, 16); + if (size === 0) break; + decoded.push(body.subarray(end + 2, end + 2 + size)); offset = end + 2 + size + 2; + } + body = Buffer.concat(decoded); + } + objects.set(key, body); response.writeHead(200, { etag: '"fixture"' }).end(); return; + } + const body = objects.get(key); + if (!body) { response.writeHead(404).end(); return; } + response.writeHead(200, { "content-length": body.length, "content-type": "application/octet-stream", etag: '"fixture"' }); + response.end(request.method === "HEAD" ? undefined : body); + })().catch(() => { response.writeHead(500).end(); }); + }); + await new Promise(resolve => { server.listen(0, "127.0.0.1", resolve); }); + const address = server.address(); if (!address || typeof address === "string") throw new Error("Expected fixture address"); + const endpoint = `http://127.0.0.1:${address.port}`; + const credentials = { AWS_ACCESS_KEY_ID: "fixture-access-key", AWS_SECRET_ACCESS_KEY: "fixture-secret-key", AWS_SESSION_TOKEN: "" }; + const previousCredentials = Object.fromEntries(Object.keys(credentials).map(key => [key, process.env[key]])); + Object.assign(process.env, credentials); + vi.mocked(storageModule.getStorageService).mockReturnValue(createStorageService(createS3StorageProvider({ + bucket: "managed-skill-fixture", region: "us-east-1", endpoint, forcePathStyle: true, + }))); const key = "cloud-runtime-config:v1"; - const run = (phase: string) => promisify(execFile)(process.execPath, [ - "--import", fileURLToPath(import.meta.resolve("tsx")), - fileURLToPath(new URL("../../scripts/migrate-skill-packages.ts", import.meta.url)), - `--phase=${phase}`, `--company=${companyId}`, "--limit=50", - ], { env: { ...process.env, DATABASE_URL: database.connectionString, - PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN: "disposable-cloud-cli-fixture", - PAPERCLIP_STORAGE_PROVIDER: "local_disk", PAPERCLIP_STORAGE_LOCAL_DIR: path.join(root, "objects") }, - timeout: 60_000, maxBuffer: 1024 * 1024 }); - await expect(run("eligibility")).rejects.toThrow("Initialize Cloud runtime configuration"); - expect(await db.select().from(instanceSettings).where(eq(instanceSettings.singletonKey, key))).toHaveLength(0); - await db.insert(instanceSettings).values({ singletonKey: key, general: { version: 1, revision: 0, config: null } }); try { + await companySkillService(db).createLocalSkill(companyId, { name: "Cloud CLI", markdown: "# Cloud CLI" }); + const run = (phase: string) => promisify(execFile)(process.execPath, [ + "--import", fileURLToPath(import.meta.resolve("tsx")), + fileURLToPath(new URL("../../scripts/migrate-skill-packages.ts", import.meta.url)), + `--phase=${phase}`, `--company=${companyId}`, "--limit=50", + ], { env: { ...process.env, ...credentials, DATABASE_URL: database.connectionString, + PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN: "disposable-cloud-cli-fixture", + PAPERCLIP_SECRETS_MASTER_KEY: "a".repeat(64), PAPERCLIP_STORAGE_PROVIDER: "s3", + PAPERCLIP_STORAGE_S3_BUCKET: "managed-skill-fixture", PAPERCLIP_STORAGE_S3_REGION: "us-east-1", + PAPERCLIP_STORAGE_S3_ENDPOINT: endpoint, PAPERCLIP_STORAGE_S3_FORCE_PATH_STYLE: "true", PAPERCLIP_STORAGE_S3_PREFIX: "" }, + timeout: 60_000, maxBuffer: 1024 * 1024 }); + await expect(run("eligibility")).rejects.toThrow("Initialize Cloud runtime configuration"); + expect(await db.select().from(instanceSettings).where(eq(instanceSettings.singletonKey, key))).toHaveLength(0); + await db.insert(instanceSettings).values({ singletonKey: key, general: { version: 1, revision: 0, config: null } }); const eligibility = JSON.parse((await run("eligibility")).stdout.trim().split("\n").at(-1)!); expect(eligibility.results).toEqual([expect.objectContaining({ status: "version_backed" })]); + const before = [...objects.entries()].map(([key, body]) => [key, Buffer.from(body)] as const); const verification = JSON.parse((await run("verification")).stdout.trim().split("\n").at(-1)!); - expect(verification.results).toEqual([expect.objectContaining({ status: "local_storage" })]); + expect(verification.results).toEqual([expect.objectContaining({ status: "remote_storage" })]); + expect([...objects.entries()]).toEqual(before); + const archiveKey = [...objects.keys()].find(key => key.endsWith("/files.tar.gz"))!; + const archive = objects.get(archiveKey)!; + const corrupted = Buffer.from(archive); corrupted[0] = corrupted[0]! ^ 1; + objects.set(archiveKey, corrupted); + const corrupt = JSON.parse((await run("verification")).stdout.trim().split("\n").at(-1)!); + expect(corrupt.results).toEqual([expect.objectContaining({ status: "corrupt_package" })]); + objects.set(archiveKey, archive); + const manifestKey = [...objects.keys()].find(key => key.endsWith("/manifest.json"))!; + const manifest = objects.get(manifestKey)!; objects.delete(manifestKey); + const unavailable = JSON.parse((await run("verification")).stdout.trim().split("\n").at(-1)!); + expect(unavailable.results).toEqual([expect.objectContaining({ status: "unavailable_or_corrupt_package" })]); + objects.set(manifestKey, manifest); + const recovered = JSON.parse((await run("verification")).stdout.trim().split("\n").at(-1)!); + expect(recovered.results).toEqual([expect.objectContaining({ status: "remote_storage" })]); + expect([...objects.entries()].sort()).toEqual([...before].sort()); + expect(await db.select().from(instanceSettings).where(eq(instanceSettings.singletonKey, key))) + .toEqual([expect.objectContaining({ general: { version: 1, revision: 0, config: null } })]); } finally { await db.delete(instanceSettings).where(eq(instanceSettings.singletonKey, key)); + vi.mocked(storageModule.getStorageService).mockReturnValue(createStorageService(createLocalDiskStorageProvider(path.join(root, "objects")))); + for (const [key, value] of Object.entries(previousCredentials)) { + if (value === undefined) delete process.env[key]; else process.env[key] = value; + } + server.closeAllConnections(); await new Promise(resolve => { server.close(() => resolve()); }); } }, 180_000); From b1104c6dd7d2ae7ec4ce6a1ab23a652bd2fb2988 Mon Sep 17 00:00:00 2001 From: Dotta Date: Fri, 30 Oct 2026 17:04:31 -0500 Subject: [PATCH 16/74] fix(server): scope run retention hints to affected companies Co-Authored-By: Paperclip --- server/src/__tests__/execution-control-work.test.ts | 4 ++-- server/src/services/execution-control-notifications.ts | 9 ++++++--- .../src/services/execution-control-reconciliation.ts | 2 +- server/src/services/execution-recovery-resolution.ts | 4 ++-- server/src/services/explicit-native-continuation.ts | 2 +- server/src/services/heartbeat.ts | 2 +- server/src/services/heartbeat/run-lifecycle.ts | 2 +- server/src/services/heartbeat/runtime-execution.ts | 4 ++-- server/src/services/issue-recovery-actions.ts | 10 +++++----- server/src/services/legacy-execution-recovery.ts | 2 +- .../native-runtime/native-failed-workspace-recovery.ts | 2 +- .../services/native-runtime/native-restart-recovery.ts | 2 +- .../services/native-runtime/native-run-finalizer.ts | 4 ++-- .../services/native-runtime/native-run-projection.ts | 2 +- .../services/native-runtime/native-safe-replacement.ts | 4 ++-- .../services/native-runtime/native-session-executor.ts | 6 +++--- .../services/native-runtime/native-superseded-retry.ts | 2 +- server/src/services/system-recovery.ts | 2 +- server/src/services/task-retry.ts | 2 +- 19 files changed, 35 insertions(+), 32 deletions(-) diff --git a/server/src/__tests__/execution-control-work.test.ts b/server/src/__tests__/execution-control-work.test.ts index 0046d8574aa..14cd9456eb5 100644 --- a/server/src/__tests__/execution-control-work.test.ts +++ b/server/src/__tests__/execution-control-work.test.ts @@ -383,11 +383,11 @@ describe.skipIf(!support.supported)("execution-control durable work", () => { const due = new Date(Date.now() + 120_000), earlier = new Date(Date.now() + 60_000); for (const at of [due, earlier]) { await writeExecutionControl(db, DELIVERY_QUEUES.executionFinalization, tx => tx.update(heartbeatRuns) - .set({ executionControlDeadlineAt: at }).where(eq(heartbeatRuns.id, run.id))); + .set({ executionControlDeadlineAt: at }).where(eq(heartbeatRuns.id, run.id)), { companyId }); await settle(); expect(c.nextWakeAt()).toBe(at.getTime()); } await writeExecutionControl(db, DELIVERY_QUEUES.executionFinalization, tx => tx.update(heartbeatRuns) - .set({ executionControlDeadlineAt: null }).where(eq(heartbeatRuns.id, run.id))); + .set({ executionControlDeadlineAt: null }).where(eq(heartbeatRuns.id, run.id)), { companyId }); await quiet(c); }); it("delivers a reconciled continuation when marking and resolving commit together", async () => { diff --git a/server/src/services/execution-control-notifications.ts b/server/src/services/execution-control-notifications.ts index a96ca761ec2..40393216e2e 100644 --- a/server/src/services/execution-control-notifications.ts +++ b/server/src/services/execution-control-notifications.ts @@ -2,8 +2,10 @@ import { notifyAttentionRetentionWork } from "./retention-work-notifications.js" import type { Db } from "@paperclipai/db"; import { DELIVERY_QUEUES, notifyDeliveryWork, type DeliveryQueue } from "./delivery-work-notifications.js"; -export async function notifyExecutionRunWork(transaction: object) { - await notifyAttentionRetentionWork(transaction); +export type ExecutionWorkScope = { companyId: string; agentId?: string; runId?: string }; + +export async function notifyExecutionRunWork(transaction: object, scope?: ExecutionWorkScope) { + if (scope) await notifyAttentionRetentionWork(transaction, scope); for (const queue of [DELIVERY_QUEUES.executionStatus, DELIVERY_QUEUES.executionFinalization, DELIVERY_QUEUES.executionReplacement, DELIVERY_QUEUES.executionDisposition, DELIVERY_QUEUES.issueMonitors]) { await notifyDeliveryWork(transaction, queue); @@ -14,9 +16,10 @@ export function writeExecutionControl( db: Db | Parameters[0]>[0], queue: DeliveryQueue | readonly DeliveryQueue[], write: (tx: Db) => PromiseLike, + scope: ExecutionWorkScope | undefined, ): Promise { return db.transaction(async transaction => { - await notifyAttentionRetentionWork(transaction); + if (scope) await notifyAttentionRetentionWork(transaction, scope); await notifyDeliveryWork(transaction, DELIVERY_QUEUES.issueMonitors); for (const topic of typeof queue === "string" ? [queue] : queue) await notifyDeliveryWork(transaction, topic); return write(transaction as unknown as Db); diff --git a/server/src/services/execution-control-reconciliation.ts b/server/src/services/execution-control-reconciliation.ts index 2f2a435982d..621fd210cfc 100644 --- a/server/src/services/execution-control-reconciliation.ts +++ b/server/src/services/execution-control-reconciliation.ts @@ -196,7 +196,7 @@ export async function reconcileAbandonedExecutionControl( await notifyNativeEnvironmentSettlement(tx, { companyId: run.companyId, runId: run.id }); await notifyCancelledWorkspacePreservation(tx, { companyId: run.companyId, runId: run.id }); await notifyUnsafeWorkspaceExportRunChange(tx, run); - await notifyExecutionRunWork(tx); + await notifyExecutionRunWork(tx, { companyId: run.companyId }); const [updatedRun] = await tx .update(heartbeatRuns) .set({ diff --git a/server/src/services/execution-recovery-resolution.ts b/server/src/services/execution-recovery-resolution.ts index f3f5a3e4310..d56e7d926f1 100644 --- a/server/src/services/execution-recovery-resolution.ts +++ b/server/src/services/execution-recovery-resolution.ts @@ -546,7 +546,7 @@ export async function settleUnrecoverableExecutions( companyId: run.companyId, issueId: task.id, runId: run.id, }).limit(1).for("update"); if (!hold) return; - await notifyExecutionRunWork(tx); + await notifyExecutionRunWork(tx, { companyId: run.companyId }); const note = "Exact session recovery stopped. The recorded workspace recovery holds this execution without replaying its actions."; await tx.update(issueRecoveryActions).set({ status: "cancelled", outcome: "cancelled", resolvedAt: now, updatedAt: now, wakePolicy: null, monitorPolicy: null, @@ -654,7 +654,7 @@ export async function settleUnrecoverableExecutions( replay: "not_authorized", }, }); - await notifyExecutionRunWork(tx); + await notifyExecutionRunWork(tx, { companyId: run.companyId }); await tx .update(heartbeatRuns) .set({ executionStatusDeliveryId: randomUUID() }) diff --git a/server/src/services/explicit-native-continuation.ts b/server/src/services/explicit-native-continuation.ts index 074ffc3bdf5..084a25e457f 100644 --- a/server/src/services/explicit-native-continuation.ts +++ b/server/src/services/explicit-native-continuation.ts @@ -475,7 +475,7 @@ export async function admitExplicitNativeContinuation(input: { }).where(and(eq(agentWakeupRequests.companyId, companyId), eq(agentWakeupRequests.id, input.queuedCommentInterruptId), eq(agentWakeupRequests.agentId, agentId), eq(agentWakeupRequests.status, "deferred_issue_execution"))); if (cancelledStartupIds.size) await db.transaction(async tx => { - await notifyExecutionRunWork(tx); + await notifyExecutionRunWork(tx, { companyId, agentId }); for (const runId of cancelledStartupIds) { await tx.update(nativeRunFinalizations).set({ phase: "terminal_failure", failureCode: "native_startup_cancelled", nextAttemptAt: null, diff --git a/server/src/services/heartbeat.ts b/server/src/services/heartbeat.ts index eb655fdef25..a4cec7b81b0 100644 --- a/server/src/services/heartbeat.ts +++ b/server/src/services/heartbeat.ts @@ -3724,7 +3724,7 @@ export function heartbeatService( await writeExecutionControl(db, DELIVERY_QUEUES.executionFinalization, tx => tx .update(heartbeatRuns) .set({ executionControlDeadlineAt: null }) - .where(eq(heartbeatRuns.id, run.id))); + .where(eq(heartbeatRuns.id, run.id)), { companyId: run.companyId }); } nativeOwnershipHeld = nativeOwnershipHeld || diff --git a/server/src/services/heartbeat/run-lifecycle.ts b/server/src/services/heartbeat/run-lifecycle.ts index 00a27c1faf2..9b9eaad3061 100644 --- a/server/src/services/heartbeat/run-lifecycle.ts +++ b/server/src/services/heartbeat/run-lifecycle.ts @@ -1044,7 +1044,7 @@ export function createHeartbeatLifecycle(db: Db, dependencies: HeartbeatLifecycl if (previous && isHeartbeatRunTerminalStatus(status)) await notifyRetainedNativeSessionCleanup(tx, { companyId: previous.companyId, runId: previous.id }); if (isHeartbeatRunTerminalStatus(status)) await notifyDeliveryWork(tx, DELIVERY_QUEUES.accounting, companyId); - await notifyExecutionRunWork(tx); + await notifyExecutionRunWork(tx, companyId ? { companyId } : undefined); if (previous) await notifyChatRunMilestones(tx, { companyId: previous.companyId, runId: previous.id, issueId: previous.nativeIssueId ?? previous.issueId ?? readNonEmptyString(parseObject(previous.contextSnapshot).issueId) ?? undefined }); return write(tx); diff --git a/server/src/services/heartbeat/runtime-execution.ts b/server/src/services/heartbeat/runtime-execution.ts index b5b64abc948..bdbbfa07752 100644 --- a/server/src/services/heartbeat/runtime-execution.ts +++ b/server/src/services/heartbeat/runtime-execution.ts @@ -966,7 +966,7 @@ export async function executeHeartbeatRuntime(db: Db, input: HeartbeatRuntimeExe eq(heartbeatRuns.id, run.id), eq(heartbeatRuns.status, "running"), ), - )); + ), { companyId: run.companyId }); const workspaceFinalizeStatus = hasWorkspaceRestoreFailure(adapterResult.resultJson) ? "failed" : "succeeded"; await recordWorkspaceFinalize(workspaceFinalizeStatus); if (adapterResult.nativeFinalization) { @@ -1042,7 +1042,7 @@ export async function executeHeartbeatRuntime(db: Db, input: HeartbeatRuntimeExe eq(heartbeatRuns.id, run.id), eq(heartbeatRuns.status, "running"), ), - )); + ), { companyId: run.companyId }); if ( issueRef && context.resumeSessionGoalHeartbeat === true && diff --git a/server/src/services/issue-recovery-actions.ts b/server/src/services/issue-recovery-actions.ts index 355e0bc4399..a1ccb8e8e7a 100644 --- a/server/src/services/issue-recovery-actions.ts +++ b/server/src/services/issue-recovery-actions.ts @@ -394,7 +394,7 @@ export function issueRecoveryActionService(db: DbOrTransaction) { const [updated] = await writeExecutionControl(db, [DELIVERY_QUEUES.executionDisposition, DELIVERY_QUEUES.executionContinuation], tx => tx.update(issueRecoveryActions).set({ nextAction: input.nextAction, updatedAt: now }) .where(and(eq(issueRecoveryActions.id, existing.id), inArray(issueRecoveryActions.status, [...ACTIVE_RECOVERY_ACTION_STATUSES]), - sql`${issueRecoveryActions.evidence}->'workspaceOwnerStop' is null`)).returning()); + sql`${issueRecoveryActions.evidence}->'workspaceOwnerStop' is null`)).returning(), { companyId: input.companyId }); if (updated) return toReadModel(updated); } return existing; @@ -469,7 +469,7 @@ export function issueRecoveryActionService(db: DbOrTransaction) { ), ) .returning(); - }); + }, { companyId: input.companyId }); if (!exhausted) { return retryUpsertSourceScoped(input, retryCount); } @@ -541,7 +541,7 @@ export function issueRecoveryActionService(db: DbOrTransaction) { ), ) .returning(); - }); + }, { companyId: input.companyId }); if (!updated) { return retryUpsertSourceScoped(input, retryCount); } @@ -556,7 +556,7 @@ export function issueRecoveryActionService(db: DbOrTransaction) { .insert(issueRecoveryActions) .values(buildInsertValues(input, ownerType, now)) .returning(); - }); + }, { companyId: input.companyId }); return toReadModel(created!); } catch (error) { if (!isUniqueRecoveryActionConflict(error)) throw error; @@ -610,7 +610,7 @@ export function issueRecoveryActionService(db: DbOrTransaction) { }) .where(and(...predicates)) .returning(); - }); + }, { companyId: input.companyId }); return updated ? toReadModel(updated) : null; } diff --git a/server/src/services/legacy-execution-recovery.ts b/server/src/services/legacy-execution-recovery.ts index 427704de63f..c462cdf1fca 100644 --- a/server/src/services/legacy-execution-recovery.ts +++ b/server/src/services/legacy-execution-recovery.ts @@ -135,7 +135,7 @@ export async function terminalizeLegacyExecution(input: { }; if (issueId) await notifyUnsafeWorkspaceExportRecovery(tx, { companyId: current.companyId, issueId }); await notifyInstructionCollection(tx, { companyId: current.companyId, runId: current.id }); - await notifyExecutionRunWork(tx); + await notifyExecutionRunWork(tx, { companyId: current.companyId }); let [updated] = await tx .update(heartbeatRuns) .set({ diff --git a/server/src/services/native-runtime/native-failed-workspace-recovery.ts b/server/src/services/native-runtime/native-failed-workspace-recovery.ts index 5c3268f6afc..a1c6c914b33 100644 --- a/server/src/services/native-runtime/native-failed-workspace-recovery.ts +++ b/server/src/services/native-runtime/native-failed-workspace-recovery.ts @@ -65,7 +65,7 @@ export async function preserveNativeFailedWorkspace(db: Db, selected: Pick>'{systemRecovery,revision}' = ${String(previous.revision)}`)) .returning({ id: issueRecoveryActions.id }); return Boolean(updated); - }); + }, { companyId: source.companyId }); } export async function loadSystemRecovery(db: Db, source: WorkspaceRecoverySourceIdentity) { diff --git a/server/src/services/task-retry.ts b/server/src/services/task-retry.ts index e9f0bd4edbf..17075c34eb9 100644 --- a/server/src/services/task-retry.ts +++ b/server/src/services/task-retry.ts @@ -334,7 +334,7 @@ export async function requestTaskRetry(input: { details: { requestId: active?.id ?? id, sourceRunId: latest.id, attachedToExistingWork: Boolean(active || busy) } }); return { requestId: active?.id ?? id, accepted: true, stale: false, observedRunId: latest.id, chatActionId: intent.chatActionId }; - }); + }, { companyId: input.companyId }); } /** Chat owns admission and authorization. The task receipt follows that From 7b0da01e3a6b6fc04a4eb887fc6da752febd35f1 Mon Sep 17 00:00:00 2001 From: Dotta Date: Fri, 30 Oct 2026 17:07:07 -0500 Subject: [PATCH 17/74] test(server): preserve UUID entropy in restore company prefixes The 60-company retained-history fixture collided on Rad9867 under companies_issue_prefix_idx in CI job 124331069055. Use the complete generated company UUID for its unique prefix while preserving all retention assertions. Co-Authored-By: Paperclip --- server/src/__tests__/git-restore-point-ledger.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/__tests__/git-restore-point-ledger.test.ts b/server/src/__tests__/git-restore-point-ledger.test.ts index 8b2e894dc44..0f30dea022b 100644 --- a/server/src/__tests__/git-restore-point-ledger.test.ts +++ b/server/src/__tests__/git-restore-point-ledger.test.ts @@ -73,7 +73,7 @@ describeEmbeddedPostgres("git restore point retention", () => { } = {}) { const companyId = input.companyId ?? randomUUID(); if (!input.companyId) { - await db.insert(companies).values({ id: companyId, name: "Restore", issuePrefix: `R${companyId.slice(0, 6)}` }); + await db.insert(companies).values({ id: companyId, name: "Restore", issuePrefix: `R${companyId.replaceAll("-", "")}` }); } const issueStatus = input.issueStatus ?? "done"; const closedAt = new Date(now.getTime() - (input.closedDaysAgo ?? 10) * DAY_MS); From 5a8e8dc5b76e334b4ef723e062d4c3dd160c6d7e Mon Sep 17 00:00:00 2001 From: Dotta Date: Fri, 30 Oct 2026 16:10:20 -0500 Subject: [PATCH 18/74] fix: release settled deadline work holds during drain Preserve dirty generations with an in-memory resume deadline while allowing settled transaction holds to end during drain. Keep unresolved database intent fenced by the coordinator. Co-Authored-By: Paperclip --- server/src/__tests__/deadline-work.test.ts | 85 +++++++++++++++++++++- server/src/services/deadline-work.ts | 13 ++-- 2 files changed, 92 insertions(+), 6 deletions(-) diff --git a/server/src/__tests__/deadline-work.test.ts b/server/src/__tests__/deadline-work.test.ts index 7b852977345..7657a7e9d99 100644 --- a/server/src/__tests__/deadline-work.test.ts +++ b/server/src/__tests__/deadline-work.test.ts @@ -1,4 +1,5 @@ -import { installDatabaseWorkSignals } from "../../../packages/db/src/work-signals.js"; +import { idleWorkSnapshot } from "../services/task-admission.js"; +import { installDatabaseWorkSignals, signalDatabaseWork, databaseWorkPending } from "../../../packages/db/src/work-signals.js"; import { afterEach, expect, it, vi } from "vitest"; import { createDeliveryWorkCoordinator } from "../services/delivery-work-coordinator.js"; import { registerDeadlineWork } from "../services/deadline-work.js"; @@ -39,3 +40,85 @@ it.each([ expect(run).toHaveBeenCalledTimes(2); expect(onError).not.toHaveBeenCalled(); }); + +it("releases a settled first-write hold during drain and retains a probe-free resume wake", async () => { + vi.useFakeTimers(); + let draining = false; + const owner = installDatabaseWorkSignals({ + execute: vi.fn(async () => [{ status: "committed" }]), + transaction: async (callback: (tx: object) => Promise) => callback({ execute: async () => [{ xid: "1" }] }), + }); + const run = vi.fn(async () => {}), next = vi.fn(async () => null); + coordinator = createDeliveryWorkCoordinator({ owner, canRun: () => !draining, canReconcile: () => true, onError: vi.fn() }); + const worker = registerDeadlineWork(coordinator, DELIVERY_QUEUES.agentTimers, { run, next, canRun: () => !draining, retryMs: 30_000 }); + await worker.ready; + draining = true; + const before = idleWorkSnapshot().active; + await owner.transaction(async tx => { await signalDatabaseWork(tx, DELIVERY_QUEUES.agentTimers); }); + await vi.advanceTimersByTimeAsync(0); + expect(idleWorkSnapshot().active).toBe(before); + expect(coordinator.nextWakeAt()).not.toBeNull(); + await vi.advanceTimersByTimeAsync(3 * 30_000); + expect(run).toHaveBeenCalledOnce(); expect(next).toHaveBeenCalledOnce(); expect(owner.execute).not.toHaveBeenCalled(); + draining = false; + await vi.advanceTimersByTimeAsync(30_000); + expect(run).toHaveBeenCalledTimes(2); expect(coordinator.nextWakeAt()).toBeNull(); +}); + +it("retains failed work through drain and preserves a future domain deadline", async () => { + vi.useFakeTimers(); + let draining = false; + const owner = installDatabaseWorkSignals({ execute: vi.fn(async () => []) }); + const run = vi.fn(async () => { if (run.mock.calls.length === 1) { draining = true; throw new Error("retry debt"); } }); + const deadline = Date.now() + 600_000; + const next = vi.fn(async () => deadline); + const onError = vi.fn(); + coordinator = createDeliveryWorkCoordinator({ owner, canRun: () => !draining, canReconcile: () => true, onError }); + const worker = registerDeadlineWork(coordinator, DELIVERY_QUEUES.agentTimers, { run, next, canRun: () => !draining, retryMs: 30_000 }); + await worker.ready; + await vi.advanceTimersByTimeAsync(90_000); + expect(run).toHaveBeenCalledOnce(); expect(next).not.toHaveBeenCalled(); expect(owner.execute).not.toHaveBeenCalled(); + draining = false; await vi.advanceTimersByTimeAsync(30_000); + expect(run).toHaveBeenCalledTimes(2); expect(coordinator.nextWakeAt()).toBe(deadline); + draining = true; await vi.advanceTimersByTimeAsync(30_000); + expect(coordinator.nextWakeAt()).toBe(deadline); expect(onError).toHaveBeenCalledOnce(); +}); + +it("does no domain probes in warm standby and reconstructs after admission opens", async () => { + vi.useFakeTimers(); + let standby = true; + const owner = installDatabaseWorkSignals({ execute: vi.fn(async () => []) }); + const run = vi.fn(async () => {}), next = vi.fn(async () => null); + coordinator = createDeliveryWorkCoordinator({ owner, canRun: () => !standby, canReconcile: () => !standby, onError: vi.fn() }); + const worker = registerDeadlineWork(coordinator, DELIVERY_QUEUES.agentTimers, { run, next, canRun: () => !standby, retryMs: 30_000 }); + await worker.ready; await vi.advanceTimersByTimeAsync(90_000); + expect(run).not.toHaveBeenCalled(); expect(next).not.toHaveBeenCalled(); expect(owner.execute).not.toHaveBeenCalled(); + standby = false; await vi.advanceTimersByTimeAsync(30_000); + expect(run).toHaveBeenCalledOnce(); expect(coordinator.nextWakeAt()).toBeNull(); +}); + +it("does not release an ambiguous database intent while dispatch is drained", async () => { + vi.useFakeTimers(); + let draining = false, committed = false; + const owner = installDatabaseWorkSignals({ + execute: vi.fn(async () => [{ status: committed ? "committed" : "in progress" }]), + transaction: async (callback: (tx: object) => Promise) => { + await callback({ execute: async () => [{ xid: "2" }] }); + throw new Error("lost acknowledgement"); + }, + }); + coordinator = createDeliveryWorkCoordinator({ owner, canRun: () => !draining, canReconcile: () => true, onError: vi.fn() }); + const run = vi.fn(async () => {}); + const worker = registerDeadlineWork(coordinator, DELIVERY_QUEUES.agentTimers, { run, next: async () => null, canRun: () => !draining, retryMs: 30_000 }); + await worker.ready; draining = true; + const before = idleWorkSnapshot().active; + await expect(owner.transaction(async tx => { await signalDatabaseWork(tx, DELIVERY_QUEUES.agentTimers); })).rejects.toThrow("lost acknowledgement"); + await vi.advanceTimersByTimeAsync(0); + expect(idleWorkSnapshot().active).toBe(before + 1); + expect(databaseWorkPending(owner, DELIVERY_QUEUES.agentTimers)).toBe(true); + committed = true; await vi.advanceTimersByTimeAsync(30_000); + expect(databaseWorkPending(owner, DELIVERY_QUEUES.agentTimers)).toBe(false); + expect(idleWorkSnapshot().active).toBe(before); + draining = false; await vi.advanceTimersByTimeAsync(30_000); + expect(run).toHaveBeenCalledTimes(2); expect(coordinator.nextWakeAt()).toBeNull(); +}); diff --git a/server/src/services/deadline-work.ts b/server/src/services/deadline-work.ts index 306f340d9fc..15ba76abdae 100644 --- a/server/src/services/deadline-work.ts +++ b/server/src/services/deadline-work.ts @@ -26,7 +26,7 @@ export function registerDeadlineWork( pausePassiveMaintenance: true, onSettled: hints => { requested++; task.onSettled?.(hints); }, run: async signal => { - if (!task.canRun()) { requested++; next = null; return; } + if (!task.canRun()) { requested++; return; } if (Date.now() < earliest) { next = earliest; return; } if (Date.now() < coalesceUntil && (next === null || next > Date.now())) { next = next === null ? coalesceUntil : Math.min(next, coalesceUntil); @@ -39,8 +39,6 @@ export function registerDeadlineWork( if (signal.aborted || !task.canRun()) { next = null; return; } const deadline = await task.next(); - - if (!task.canRun()) { next = null; return; } earliest = deadline === null || deadline > Date.now() ? (task.minimumSpacingMs ? Date.now() + task.minimumSpacingMs : 0) @@ -49,8 +47,13 @@ export function registerDeadlineWork( coalesceUntil = Date.now() + (task.notificationCoalesceMs ?? 0); reconciled = generation; }, - hasPending: async () => requested !== reconciled, - nextRunAt: () => next, + // A settled producer must release its admission hold during drain. Keep + // the unreconciled generation and an in-memory resume wake, without asking + // the domain to probe its database while dispatch is closed. + hasPending: async () => task.canRun() && requested !== reconciled, + nextRunAt: () => !task.canRun() && requested !== reconciled + ? Math.min(next ?? Infinity, Date.now() + task.retryMs) + : next, }); return { ...worker, wake: () => { requested++; worker.wake(); } }; } From c86e5b58db55dd90a95db141ac760b44803a46fe Mon Sep 17 00:00:00 2001 From: Dotta Date: Fri, 30 Oct 2026 17:09:27 -0500 Subject: [PATCH 19/74] perf(github): activate connection work from events and deadlines Retain Cloud event replay through durable enrollment and signed tenant hints. Save credential binding changes and retry deadlines in existing grant metadata. Preserve receipt idempotency, lifecycle admission and mixed-version fallback. Co-Authored-By: Paperclip --- .../2026-10-30-github-connection-sleep.md | 83 ++++++ packages/db/src/schema/tool_access.ts | 2 + packages/shared/src/types/tool-access.ts | 3 + packages/shared/src/validators/tool-access.ts | 2 + .../cloud-control-task-drain.test.ts | 22 +- .../github-connection-events.test.ts | 8 +- .../__tests__/github-connection-work.test.ts | 192 ++++++++++++++ .../server-startup-feedback-export.test.ts | 35 ++- server/src/app.ts | 2 + server/src/index.ts | 40 +-- server/src/middleware/cloud-control.ts | 1 + server/src/routes/connection-event-wake.ts | 13 + server/src/services/cloud-runtime-identity.ts | 1 + server/src/services/companies.ts | 4 + server/src/services/connection-event-wake.ts | 12 + .../services/delivery-work-notifications.ts | 2 + .../services/github-connection-events.test.ts | 35 ++- .../src/services/github-connection-events.ts | 92 ++++--- .../github-connection-notifications.ts | 7 + server/src/services/github-connection-work.ts | 36 +++ .../src/services/paperclip-cloud-connector.ts | 12 +- .../src/services/provider-retry-after.test.ts | 26 ++ server/src/services/provider-retry-after.ts | 6 + server/src/services/tool-access.ts | 5 +- .../tool-access/connections/grants.ts | 2 + .../tool-access/github-access-discovery.ts | 2 + .../tool-access/github-grant-continuity.ts | 249 +++++++++++------- .../tool-access/oauth/cloud-callback.ts | 59 ++--- .../tool-access/oauth/grant-refresh.ts | 6 +- 29 files changed, 740 insertions(+), 219 deletions(-) create mode 100644 doc/plans/2026-10-30-github-connection-sleep.md create mode 100644 server/src/__tests__/github-connection-work.test.ts create mode 100644 server/src/routes/connection-event-wake.ts create mode 100644 server/src/services/connection-event-wake.ts create mode 100644 server/src/services/github-connection-notifications.ts create mode 100644 server/src/services/github-connection-work.ts create mode 100644 server/src/services/provider-retry-after.test.ts create mode 100644 server/src/services/provider-retry-after.ts diff --git a/doc/plans/2026-10-30-github-connection-sleep.md b/doc/plans/2026-10-30-github-connection-sleep.md new file mode 100644 index 00000000000..11271110a7a --- /dev/null +++ b/doc/plans/2026-10-30-github-connection-sleep.md @@ -0,0 +1,83 @@ +# GitHub connection delivery and credential continuity + +GitHub event delivery and credential continuity have separate obligations. +The event worker uses the shared delivery coordinator. It performs a startup +reconciliation and responds to committed configuration changes or authenticated +Cloud hints. Credential work registers the actual next grant expiry, provider +continuity check, or saved retry deadline. The aggregate heartbeat no longer +calls either service, including when the heartbeat scheduler is disabled. + +## Event delivery contract + +The Cloud webhook target queue remains durable truth. The tenant leases sealed +events, validates their bindings, persists company-scoped receipts, applies the +existing installation or pull-request behavior, and acknowledges completed work. +A receipt row lock prevents overlapping receivers from applying an installation +delta twice. Completed receipts remain passive history. The existing normalized +merged-pull-request callback remains unchanged. + +A new tenant sends signed event-lease claim `ewv=1`. A compatible Cloud broker +records delivery version 1 on the existing instance registration before returning +`eventWakeVersion=1`. Only then does the tenant stop configured-empty polling. +Cloud reconstructs unacknowledged targets after restart and retries the exact +registered tenant when its current queue lease permits another delivery. A +successful HTTP hint does not acknowledge the target. Missing, stopped or drained +tenants therefore retain their work. The hint carries no provider payload or +client-selected destination. The tenant checks a signed, stack-bound, single-use +`connector-events:wake` control assertion on the exact POST endpoint. + +Old Cloud and self-hosted registrations retain scoped reconciliation with the +existing backoff. No connector configuration or no registration history settles +to no recurring owner SQL or provider traffic. Disabled consumers can acknowledge +queued events as ignored using their trusted local registration history; they +cannot apply events to an inactive connection. Removing polling does not remove +the host sleep safety blockers. Worker quiet is not proof of host suspension. + +## Durable credential debt + +A grant stores a unique `bindingSync` revision and removed installation IDs before +Cloud registration calls. Successful delivery clears only the locked current +revision. Failure retains the saved intent and an exponential retry deadline, +including provider Retry-After advice. A restart loads that deadline from the +grant. Binding-only retries use the existing credential refresh lease and current +secret resolver without repeating account/repository discovery. The initial OAuth +callback uses the same saved debt and settlement path after connection activation. + +Credential expiry remains enforced by the existing use path. Scheduled continuity +checks run one hour before recorded access expiry or after the existing 30-day +continuity window. A successful renewal that leaves the credential short-lived +or unchanged retains a one-hour retry obligation instead of causing rapid calls. +Changed credential metadata invalidates the old retry revision, so an earlier new +expiry can preempt it. Disabled connections and archived companies do not perform +credential work. Reactivation uses the normal committed configuration signal. + +Supported local producers are the owning connection, OAuth, grant and company +services in one tenant process. Their notifications belong to the outer database +transaction. A savepoint does not commit a wake, and rollback cancels the write. +Arbitrary SQL writers must use these notification contracts or restart the owner; +process-local signals do not promise independent-writer or replica notification. +External event delivery remains durable in Cloud across processes and restarts. + +## Paired rollout and rollback + +1. Apply the companion Cloud registration-column migration and deploy its delivery + worker with the control signer present. Keep new enrollment disabled initially. +2. Deploy this tenant version. It keeps fallback polling against older Cloud. +3. Enable Cloud enrollment. Version 1 activation is durable and idempotent. Turning + the new-enrollment flag off must still service all existing version 1 records. +4. Before removing the Cloud delivery worker, restart affected tenants with + `PAPERCLIP_GITHUB_EVENT_POLLING=1`. They arm fallback before sending signed + `ewv=0`. Confirm the Cloud registration is cleared. A failed handoff leaves + fallback active and Cloud retry intact. Alternatively, restore the old polling + tenant image first while retaining the compatible Cloud worker. +5. Do not deploy a Cloud downgrade that removes responsibility for still-enrolled + tenants. There is no capability expiry, renewal poll, or periodic tenant hint. + +## Evidence + +Focused tests use isolated PostgreSQL plus controlled HTTP provider simulations. +They cover first service write after quiet, saved results, terminal receipts, +concurrent replay, signed callbacks, outer commit/rollback, drain/resume, startup +wiring in both scheduler branches, credential backoff and changed deadlines. +Cloud acceptance uses its actual broker/store/notifier boundaries and a controlled +HTTP tenant. These tests do not establish a deployed fleet rollout or cost savings. diff --git a/packages/db/src/schema/tool_access.ts b/packages/db/src/schema/tool_access.ts index b8b317ddb91..bbe6c30e584 100644 --- a/packages/db/src/schema/tool_access.ts +++ b/packages/db/src/schema/tool_access.ts @@ -231,6 +231,8 @@ export const connectionGrants = pgTable( appSlug?: string; accessRevision?: string; lastAccessRefreshAt?: string; + bindingSync?: { id: string; removeInstallationIds: string[] }; + continuityRetry?: { at: string; attempts: number; revision: string }; lastWebhookAt?: string; webhookHealth?: "pending" | "healthy" | "unhealthy"; }; diff --git a/packages/shared/src/types/tool-access.ts b/packages/shared/src/types/tool-access.ts index 0fbd422fdad..5627ca6a174 100644 --- a/packages/shared/src/types/tool-access.ts +++ b/packages/shared/src/types/tool-access.ts @@ -266,6 +266,9 @@ export interface ConnectionGrant { appSlug?: string; accessRevision?: string; lastAccessRefreshAt?: string; + /** Unsettled Cloud binding changes; retained until the exact revision is delivered. */ + bindingSync?: { id: string; removeInstallationIds: string[] }; + continuityRetry?: { at: string; attempts: number; revision: string }; lastWebhookAt?: string; webhookHealth?: "pending" | "healthy" | "unhealthy"; }; diff --git a/packages/shared/src/validators/tool-access.ts b/packages/shared/src/validators/tool-access.ts index bafe22e80e1..914af74e7ab 100644 --- a/packages/shared/src/validators/tool-access.ts +++ b/packages/shared/src/validators/tool-access.ts @@ -240,6 +240,8 @@ export const connectionGrantSchema = z.object({ managementUrl: z.string().url().max(2000).optional(), appSlug: z.string().regex(/^[a-z0-9-]{1,100}$/).optional(), lastAccessRefreshAt: z.string().datetime().optional(), + bindingSync: z.object({ id: z.string().uuid(), removeInstallationIds: z.array(z.string().regex(/^[1-9][0-9]{0,30}$/)).max(100) }).optional(), + continuityRetry: z.object({ at: z.string().datetime(), attempts: z.number().int().min(1).max(16), revision: z.string().max(2000) }).optional(), lastWebhookAt: z.string().datetime().optional(), webhookHealth: z.enum(["pending", "healthy", "unhealthy"]).optional(), }).optional(), diff --git a/server/src/__tests__/cloud-control-task-drain.test.ts b/server/src/__tests__/cloud-control-task-drain.test.ts index 3f0ab8e1951..46c2962edc5 100644 --- a/server/src/__tests__/cloud-control-task-drain.test.ts +++ b/server/src/__tests__/cloud-control-task-drain.test.ts @@ -1,7 +1,9 @@ +import { connectionEventWakeRoutes } from "../routes/connection-event-wake.js"; +import { registerConnectionEventWake } from "../services/connection-event-wake.js"; import { generateKeyPairSync, sign } from "node:crypto"; import express from "express"; import request from "supertest"; -import { afterEach, beforeEach, describe, expect, it } from "vitest"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { CLOUD_CONTROL_AUDIENCE, CLOUD_CONTROL_HEADER, @@ -184,6 +186,7 @@ describe("verifyCloudControlAssertion", () => { }); describe("cloudControlMiddleware", () => { + const eventOwner = {}; const savedEnv: Record = {}; beforeEach(() => { @@ -208,6 +211,7 @@ describe("cloudControlMiddleware", () => { next(); }); app.use(cloudControlMiddleware()); + app.use(connectionEventWakeRoutes(eventOwner)); app.all("/api/instance/task-drain", (req, res) => { res.json({ actor: req.actor }); }); @@ -241,6 +245,22 @@ describe("cloudControlMiddleware", () => { return `${header}.${payload}.${signature}`; } + it("accepts only signed exact-action event hints and leaves unavailable receivers retryable", async () => { + const app = createApp(); const path = "/api/instance/connector-events/wake"; + expect((await request(app).post(path)).status).toBe(403); + expect((await request(app).post(path).set(CLOUD_CONTROL_HEADER, freshAssertion("lifecycle:read"))).status).toBe(401); + expect((await request(app).post(path).set(CLOUD_CONTROL_HEADER, freshAssertion("connector-events:wake"))).status).toBe(503); + const wake = vi.fn(); const stop = registerConnectionEventWake(eventOwner, wake); + try { + const token = freshAssertion("connector-events:wake"); + expect((await request(app).post(path).set(CLOUD_CONTROL_HEADER, token)).status).toBe(202); + expect((await request(app).post(path).set(CLOUD_CONTROL_HEADER, token)).status).toBe(401); + expect((await request(app).post(path).set(CLOUD_CONTROL_HEADER, freshAssertion("connector-events:wake"))).status).toBe(202); + expect(wake).toHaveBeenCalledTimes(2); + } finally { stop(); } + expect((await request(app).post(path).set(CLOUD_CONTROL_HEADER, freshAssertion("connector-events:wake"))).status).toBe(503); + }); + it("installs a synthetic instance-admin board actor for a valid assertion, per method", async () => { const app = createApp(); for (const [method, action] of [ diff --git a/server/src/__tests__/github-connection-events.test.ts b/server/src/__tests__/github-connection-events.test.ts index 7c7b7025243..52cc835a444 100644 --- a/server/src/__tests__/github-connection-events.test.ts +++ b/server/src/__tests__/github-connection-events.test.ts @@ -35,7 +35,7 @@ describeEmbeddedPostgres("GitHub connection event delivery", () => { beforeAll(async () => { if (!externalTestDatabaseUrl) tempDb = await startEmbeddedPostgresTestDatabase("paperclip-github-events-"); if (externalTestDatabaseUrl) await applyPendingMigrations(externalTestDatabaseUrl); - db = createDb(externalTestDatabaseUrl ?? tempDb!.connectionString); + db = createDb(externalTestDatabaseUrl ?? tempDb!.connectionString, { maxConnections: 4 }); }, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS); afterEach(async () => { @@ -343,7 +343,11 @@ describeEmbeddedPostgres("GitHub connection event delivery", () => { throw new Error("fixture live subscriber failed"); }); try { - await expect(service.pollOnce()).resolves.toMatchObject({ processed: 1, duplicate: 0, failed: 0 }); + const competing = githubConnectionEventService(db, { connector, now: () => currentTime }); + const outcomes = await Promise.all([service.pollOnce(), competing.pollOnce()]); + expect(outcomes.reduce((sum, result) => sum + result.processed, 0)).toBe(1); + expect(outcomes.reduce((sum, result) => sum + result.duplicate, 0)).toBe(1); + expect(outcomes.every(result => result.failed === 0)).toBe(true); } finally { unsubscribe(); } await health.expectDeadline(currentTime.getTime() + 900_000 - (refreshed ? 60_000 : 0)); await health.expectQuiet(); diff --git a/server/src/__tests__/github-connection-work.test.ts b/server/src/__tests__/github-connection-work.test.ts new file mode 100644 index 00000000000..4e789eedb2b --- /dev/null +++ b/server/src/__tests__/github-connection-work.test.ts @@ -0,0 +1,192 @@ +import { randomUUID, generateKeyPairSync, verify, createHash } from "node:crypto"; +import { createServer, type Server } from "node:http"; +import { once } from "node:events"; +import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest"; +import { eq } from "drizzle-orm"; +import { applyPendingMigrations, companies, connectionGrants, createDb, toolApplications, toolConnections } from "@paperclipai/db"; +import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js"; +import { createGitHubGrantContinuity, githubGrantContinuityDeadline } from "../services/tool-access/github-grant-continuity.js"; +import { createPaperclipCloudConnector } from "../services/paperclip-cloud-connector.js"; +import { githubConnectionEventService } from "../services/github-connection-events.js"; +import { createDeliveryWorkCoordinator } from "../services/delivery-work-coordinator.js"; +import { registerGitHubConnectionContinuityWork, registerGitHubConnectionEventWork } from "../services/github-connection-work.js"; +import { notifyGitHubCredentialChange } from "../services/github-connection-notifications.js"; +import { idleWorkSnapshot } from "../services/task-admission.js"; + +const databaseUrl = resolveExternalTestDatabaseUrl("PAPERCLIP_GITHUB_WORK_TEST_DATABASE_URL"); +const describeDatabase = databaseUrl ? describe : describe.skip; + +describeDatabase("GitHub coordinator with PostgreSQL and controlled Cloud HTTP (simulation)", () => { + let db: ReturnType; + let server: Server, origin: string; + let bindings: Array>, leases: number, failBindings: boolean, githubRequests: number; + const signing = generateKeyPairSync("ed25519"), sealing = generateKeyPairSync("x25519"); + const realFetch = globalThis.fetch; + let companyIds: string[] = []; + beforeAll(async () => { + await applyPendingMigrations(databaseUrl!); + db = createDb(databaseUrl!, { maxConnections: 4 }); + server = createServer(async (req, res) => { + const chunks: Buffer[] = []; for await (const chunk of req) chunks.push(Buffer.from(chunk)); + const raw = Buffer.concat(chunks).toString(); + if (req.url!.startsWith("/v1/connector/")) { + const body = JSON.parse(raw), [header, payload, signature] = body.request.split("."); + expect(verify(null, Buffer.from(`${header}.${payload}`), signing.publicKey, Buffer.from(signature, "base64url"))).toBe(true); + const claims = JSON.parse(Buffer.from(payload, "base64url").toString()); + expect(claims.iss).toBe("instance"); expect(claims.env).toBe("test"); + expect(claims.aud).toBe(`${origin}${req.url}`); + if (req.url!.endsWith("/events/lease")) { + leases++; res.setHeader("content-type", "application/json"); + res.end(JSON.stringify({ events: [], ...(claims.ewv === 1 ? { eventWakeVersion: 1 } : {}) })); return; + } + expect(claims.sh).toBe(createHash("sha256").update(body.binding).digest("base64url")); + bindings.push(JSON.parse(body.binding)); + res.writeHead(failBindings ? 503 : 200, { "content-type": "application/json" }).end("{}"); return; + } + githubRequests++; + const body = req.url === "/user" ? { id: 42, login: "octocat" } + : req.url!.startsWith("/user/installations?") ? { installations: [{ id: 101, repository_selection: "all", account: { login: "octocat" } }] } + : { total_count: 1, repositories: [{ id: 999, full_name: "octocat/repo" }] }; + res.writeHead(200, { "content-type": "application/json" }).end(JSON.stringify(body)); + }); + server.listen(0, "127.0.0.1"); await once(server, "listening"); + const address = server.address(); if (!address || typeof address === "string") throw new Error("HTTP fixture did not start"); + origin = `http://127.0.0.1:${address.port}`; + }); + afterEach(async () => { + vi.restoreAllMocks(); vi.useRealTimers(); + for (const id of companyIds) await db.delete(companies).where(eq(companies.id, id)); companyIds = []; + }); + afterAll(async () => { server.close(); await once(server, "close"); await db.$client.end({ timeout: 1 }); }); + + async function fixture(ids: string[] = []) { + bindings = []; leases = 0; failBindings = false; githubRequests = 0; + const companyId = randomUUID(); companyIds.push(companyId); + const [company] = await db.insert(companies).values({ id: companyId, name: "GitHub work test", issuePrefix: "GWT" }).returning(); + const [app] = await db.insert(toolApplications).values({ companyId, applicationKey: randomUUID(), name: "GitHub", type: "mcp_server", status: "active" }).returning(); + const [connection] = await db.insert(toolConnections).values({ companyId, applicationId: app!.id, uid: randomUUID(), name: "GitHub", status: "active", enabled: true, + transport: "mcp_remote", authKind: "oauth", credentialPolicy: "per_user", config: { sourceTemplateKey: "github", oauth: { connectorProfile: "github.code" } }, transportConfig: {} }).returning(); + const [grant] = await db.insert(connectionGrants).values({ companyId, connectionId: connection!.id, kind: "user", subjectUserId: "owner", status: "active", + credentialSecretRefs: [{ configPath: "oauth.access_token", secretId: randomUUID(), label: "Access", provider: "local_encrypted" }], + providerTenant: { oauth: { accessTokenExpiresAt: new Date(Date.now() + 8 * 3600_000).toISOString(), refreshedAt: new Date().toISOString() }, github: { + userId: "42", login: "octocat", installationCount: ids.length, repositoryCount: 1, repositorySelection: "all", installationIds: ids, + installationOwnerLogins: ["octocat"], accessRevision: randomUUID(), lastAccessRefreshAt: new Date().toISOString(), webhookHealth: "healthy", + } } }).returning(); + const connector = createPaperclipCloudConnector({ config: { baseUrl: origin, instanceId: "instance", environment: "test", + signPrivateKey: signing.privateKey.export({ format: "jwk" }).d!, sealPrivateKey: sealing.privateKey.export({ format: "jwk" }).d! } }); + type Dependencies = Parameters[1]; + const dependencies: Dependencies = { + now: () => new Date(), currentCloudConnector: () => connector, + refreshOAuthGrantCredentials: async input => (await db.select().from(connectionGrants).where(eq(connectionGrants.id, input.grantId)))[0]!, + resolveOAuthGrantSecret: (async () => ({ value: "controlled-token" })) as Dependencies["resolveOAuthGrantSecret"], + }; + const service = createGitHubGrantContinuity(db, dependencies); + return { company: company!, connection: connection!, grant: grant!, connector, service, dependencies }; + } + const fakeClock = () => vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"], shouldAdvanceTime: true }); + async function settled(coordinator: ReturnType) { + await vi.waitFor(() => expect(idleWorkSnapshot().active).toBe(0)); + expect(coordinator.nextWakeAt()).not.toBeNull(); + } + + it("ordinary grant refresh activates events from quiet, commits durable result, then performs no owner SQL/HTTP across old cadence windows", async () => { + const f = await fixture(); fakeClock(); + vi.spyOn(globalThis, "fetch").mockImplementation((url, init) => String(url).startsWith("https://api.github.com") + ? realFetch(`${origin}${new URL(String(url)).pathname}${new URL(String(url)).search}`, init) : realFetch(url, init)); + const errors: unknown[] = []; + const coordinator = createDeliveryWorkCoordinator({ owner: db, canRun: () => true, onError: error => errors.push(error) }); + const events = githubConnectionEventService(db, { connector: f.connector }); + const ew = registerGitHubConnectionEventWork(db, coordinator, events, () => true); + const cw = registerGitHubConnectionContinuityWork(db, coordinator, f.service, () => true); + try { + await Promise.all([ew.ready, cw.ready]); expect(leases).toBe(0); + await f.service.refreshManagedGitHubGrantAccess(f.connection, f.grant, { actorType: "system", actorId: "test" }); + await vi.waitFor(() => expect(leases).toBeGreaterThan(0)); + await vi.advanceTimersByTimeAsync(31_000); await settled(coordinator); + const [saved] = await db.select().from(connectionGrants).where(eq(connectionGrants.id, f.grant.id)); + expect(saved!.providerTenant!.github!.installationIds).toEqual(["101"]); + expect(saved!.providerTenant!.github!.bindingSync).toBeUndefined(); + expect(bindings).toContainEqual(expect.objectContaining({ active: true, installationId: "101", id: `${f.grant.id}_101` })); + const reads = vi.spyOn(db, "select"), writes = vi.spyOn(db, "transaction"), execute = vi.spyOn(db, "execute"); + const before = { leases, githubRequests, bindings: bindings.length }; + await vi.advanceTimersByTimeAsync(10 * 60_000); + expect(reads).not.toHaveBeenCalled(); expect(writes).not.toHaveBeenCalled(); expect(execute).not.toHaveBeenCalled(); + expect({ leases, githubRequests, bindings: bindings.length }).toEqual(before); expect(errors).toEqual([]); + } finally { await ew.stop(); await cw.stop(); await coordinator.stop(); } + }); + + it("failed binding removal persists retry/debt through restart, uses current lease deadline and settles without refetching GitHub metadata", async () => { + const f = await fixture(["101"]); fakeClock(); + await db.update(connectionGrants).set({ providerTenant: { ...f.grant.providerTenant, github: { ...f.grant.providerTenant!.github!, + bindingSync: { id: randomUUID(), removeInstallationIds: ["202"] } } } }).where(eq(connectionGrants.id, f.grant.id)); + failBindings = true; + expect(await f.service.sweepGitHubConnectionContinuity()).toMatchObject({ due: 1, failed: 1 }); + const [failed] = await db.select().from(connectionGrants).where(eq(connectionGrants.id, f.grant.id)); + expect(failed!.providerTenant!.github!.bindingSync!.removeInstallationIds).toEqual(["202"]); + const retryAt = Date.parse(failed!.providerTenant!.github!.continuityRetry!.at); + const restarted = createGitHubGrantContinuity(db, f.dependencies); + expect(await restarted.nextGitHubConnectionContinuityAt()).toBe(retryAt); + await vi.advanceTimersByTimeAsync(29_000); + expect(await restarted.sweepGitHubConnectionContinuity()).toMatchObject({ due: 0 }); + failBindings = false; await vi.advanceTimersByTimeAsync(2_000); + await Promise.all([restarted.sweepGitHubConnectionContinuity(), restarted.sweepGitHubConnectionContinuity()]); + const [saved] = await db.select().from(connectionGrants).where(eq(connectionGrants.id, f.grant.id)); + expect(saved!.providerTenant!.github!.bindingSync).toBeUndefined(); expect(saved!.providerTenant!.github!.continuityRetry).toBeUndefined(); + expect(bindings.filter(binding => binding.installationId === "202")).toHaveLength(1); + expect(githubRequests).toBe(0); + expect(await restarted.nextGitHubConnectionContinuityAt()).toBeGreaterThan(Date.now()); + await db.update(toolConnections).set({ enabled: false }).where(eq(toolConnections.id, f.connection.id)); + expect(await restarted.nextGitHubConnectionContinuityAt()).toBeNull(); + await db.update(toolConnections).set({ enabled: true }).where(eq(toolConnections.id, f.connection.id)); + await db.update(companies).set({ status: "archived" }).where(eq(companies.id, f.company.id)); + expect(await restarted.nextGitHubConnectionContinuityAt()).toBeNull(); + }); + + it("retains a real deadline when successful renewal leaves a short-lived credential unchanged", async () => { + const f = await fixture(["101"]); fakeClock(); + vi.spyOn(globalThis, "fetch").mockImplementation((url, init) => String(url).startsWith("https://api.github.com") + ? realFetch(`${origin}${new URL(String(url)).pathname}${new URL(String(url)).search}`, init) : realFetch(url, init)); + await db.update(connectionGrants).set({ providerTenant: { ...f.grant.providerTenant, + oauth: { ...f.grant.providerTenant!.oauth, accessTokenExpiresAt: new Date(Date.now() + 30 * 60_000).toISOString() } } }).where(eq(connectionGrants.id, f.grant.id)); + expect(await f.service.sweepGitHubConnectionContinuity()).toMatchObject({ due: 1, refreshed: 1 }); + const [saved] = await db.select().from(connectionGrants).where(eq(connectionGrants.id, f.grant.id)); + const retryAt = Date.parse(saved!.providerTenant!.github!.continuityRetry!.at); + expect(retryAt).toBeGreaterThanOrEqual(Date.now() + 59 * 60_000); + const restarted = createGitHubGrantContinuity(db, f.dependencies); + expect(await restarted.nextGitHubConnectionContinuityAt()).toBe(retryAt); + const requests = githubRequests; await vi.advanceTimersByTimeAsync(10 * 60_000); + expect(await restarted.sweepGitHubConnectionContinuity()).toMatchObject({ due: 0 }); expect(githubRequests).toBe(requests); + }); + + it("outer commit, rollback and first producer during drain retain the deadline and resume obligation without cross-company work", async () => { + const f = await fixture(["101"]); fakeClock(); + let enabled = false; + const errors: unknown[] = []; + const coordinator = createDeliveryWorkCoordinator({ owner: db, canRun: () => enabled, canReconcile: () => true, onError: error => errors.push(error) }); + const worker = registerGitHubConnectionContinuityWork(db, coordinator, f.service, () => enabled); + try { + await worker.ready; + const dueAt = Date.now() + 90 * 60_000; + let release!: () => void; + const held = new Promise(resolve => { release = resolve; }); + const transaction = db.transaction(async tx => { + await tx.transaction(async nested => { + await notifyGitHubCredentialChange(nested, f.connection); + await nested.update(connectionGrants).set({ providerTenant: { ...f.grant.providerTenant, + oauth: { ...f.grant.providerTenant!.oauth, accessTokenExpiresAt: new Date(dueAt + 3600_000).toISOString() } } }).where(eq(connectionGrants.id, f.grant.id)); + }); + await held; + }); + await new Promise(resolve => setImmediate(resolve)); + expect(bindings).toHaveLength(0); release(); await transaction; + await vi.advanceTimersByTimeAsync(31_000); + expect(idleWorkSnapshot().active).toBe(0); expect(bindings).toHaveLength(0); + enabled = true; await vi.advanceTimersByTimeAsync(31_000); + await vi.waitFor(() => expect(coordinator.nextWakeAt()).toBe(dueAt)); + await expect(db.transaction(async tx => { await notifyGitHubCredentialChange(tx, f.connection); + await tx.update(connectionGrants).set({ status: "revoked" }).where(eq(connectionGrants.id, f.grant.id)); throw new Error("rollback"); })).rejects.toThrow("rollback"); + await vi.advanceTimersByTimeAsync(31_000); + await vi.waitFor(() => expect(coordinator.nextWakeAt()).toBe(dueAt)); expect(errors).toEqual([]); + } finally { await worker.stop(); await coordinator.stop(); } + }); +}); diff --git a/server/src/__tests__/server-startup-feedback-export.test.ts b/server/src/__tests__/server-startup-feedback-export.test.ts index bbb020c865d..d73175ee556 100644 --- a/server/src/__tests__/server-startup-feedback-export.test.ts +++ b/server/src/__tests__/server-startup-feedback-export.test.ts @@ -11,7 +11,7 @@ import { readIdleSleepSafety } from "../services/idle-sleep-safety.js"; import * as idleLocalWork from "../services/idle-local-work.js"; import { idleBackupWakeMarker, markIdleBackupWakeRequired } from "../services/idle-database-backup.js"; import { DELIVERY_QUEUES } from "../services/delivery-work-notifications.js"; -import { decisionService, decisionRetentionService, toolAccessService, reconcileBuiltInAgentsOnStartup, reconcileCodexLocalManagedHomesOnStartup, reconcilePersistedRuntimeServicesOnStartup } from "../services/index.js"; +import { decisionService, decisionRetentionService, toolAccessService, githubConnectionEventService, reconcileBuiltInAgentsOnStartup, reconcileCodexLocalManagedHomesOnStartup, reconcilePersistedRuntimeServicesOnStartup } from "../services/index.js"; import { companies, runDatabaseBackup, type Db } from "@paperclipai/db"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; @@ -36,6 +36,13 @@ vi.mock("../services/cloud-configuration-migration.js", () => ({ migrateCloudCon vi.mock("../services/cloud-configuration-qualification.js", () => ({ qualifyCloudConfigurationForCurrentImage: cloudMigrationMocks.qualify })); vi.mock("../telemetry.js", () => ({ initTelemetry: cloudRuntimeStartupMocks.telemetry, getTelemetryClient: vi.fn(() => null) })); +// Producer transaction/notification behavior is covered by the real PostgreSQL +// GitHub worker suite; this startup seam supplies a lightweight database mock. +vi.mock("../services/delivery-work-notifications.js", async () => ({ + ...await vi.importActual("../services/delivery-work-notifications.js"), + subscribeDeliveryWork: vi.fn(() => () => {}), +})); + const ORIGINAL_PAPERCLIP_API_URL = process.env.PAPERCLIP_API_URL; const ORIGINAL_PAPERCLIP_RUNTIME_API_URL = process.env.PAPERCLIP_RUNTIME_API_URL; const ORIGINAL_PAPERCLIP_RUNTIME_API_CANDIDATES_JSON = process.env.PAPERCLIP_RUNTIME_API_CANDIDATES_JSON; @@ -432,6 +439,7 @@ vi.mock("../services/index.js", () => ({ externalObjectService: externalObjectsServiceFactoryMock, heartbeatService: heartbeatServiceFactoryMock, githubConnectionEventService: vi.fn(() => ({ + nextPollAt: vi.fn(() => null), pollOnce: vi.fn(async () => ({ leased: 0, processed: 0, @@ -483,6 +491,7 @@ vi.mock("../services/index.js", () => ({ needsAttention: 0, failed: 0, })), + nextGitHubConnectionContinuityAt: vi.fn(async () => null), sweepGitHubConnectionContinuity: vi.fn(async () => ({ checked: 0, due: 0, @@ -1280,6 +1289,30 @@ describe("startServer feedback export wiring", () => { expect(completionSweepMock).not.toHaveBeenCalled(); }); + it.each([true, false])("registers GitHub delivery/deadline workers once without aggregate polling (scheduler=%s)", async enabled => { + loadConfigMock.mockReturnValue(buildTestConfig({ heartbeatSchedulerEnabled: enabled, heartbeatSchedulerIntervalMs: 30_000 })); + const ticks: Array<() => void> = []; + const intervals = vi.spyOn(globalThis, "setInterval").mockImplementation(((callback: () => void, ms: number) => { + if (ms === 30_000) ticks.push(callback); + return 1 as unknown as ReturnType; + }) as typeof setInterval); + try { + await startServer(); await settleStartupHousekeeping(); + for (const queue of [DELIVERY_QUEUES.connectionEvents, DELIVERY_QUEUES.connectionContinuity]) { + expect(deliveryWorkRegisterMock.mock.calls.filter(([registered]) => registered === queue)).toHaveLength(1); + } + const polls = vi.mocked(githubConnectionEventService).mock.results.map(result => result.value.pollOnce); + const sweeps = vi.mocked(toolAccessService).mock.results.map(result => result.value.sweepGitHubConnectionContinuity); + const count = () => [...polls, ...sweeps].reduce((sum, call) => sum + call.mock.calls.length, 0); + const startup = count(); + expect(startup).toBe(2); + for (let i = 0; i < 3; i++) for (const tick of ticks) tick(); + await new Promise(resolve => setImmediate(resolve)); + expect(count()).toBe(startup); + expect(deliveryWorkErrors.filter(error => error.queue === DELIVERY_QUEUES.connectionEvents || error.queue === DELIVERY_QUEUES.connectionContinuity)).toEqual([]); + } finally { intervals.mockRestore(); } + }); + it("does not rescan idle tool health from the aggregate maintenance tick", async () => { loadConfigMock.mockReturnValue(buildTestConfig({ heartbeatSchedulerEnabled: true, heartbeatSchedulerIntervalMs: 30_000 })); let tick: (() => void) | undefined; diff --git a/server/src/app.ts b/server/src/app.ts index 42ab549d582..d374063f4b2 100644 --- a/server/src/app.ts +++ b/server/src/app.ts @@ -1,3 +1,4 @@ +import { connectionEventWakeRoutes } from "./routes/connection-event-wake.js"; import { registerGitHubReviewProjectionWork } from "./services/chat-channels/github/review-projection-work.js"; import { finishServiceShutdown } from "./shutdown.js"; import { promptInjectionPreventionRoutes } from "./routes/prompt-injection-prevention.js"; @@ -577,6 +578,7 @@ export async function createApp( // REPLACES whatever actor the request otherwise resolved to, and only on // the one endpoint it authorizes (see the middleware for the contract). app.use(cloudControlMiddleware()); + app.use(connectionEventWakeRoutes(db)); app.use(skillAccessRequestContext); app.use("/api/auth", authRoutes(db)); if (opts.betterAuthHandler) { diff --git a/server/src/index.ts b/server/src/index.ts index c127753889c..b2db762a808 100644 --- a/server/src/index.ts +++ b/server/src/index.ts @@ -1,3 +1,4 @@ +import { registerGitHubConnectionEventWork, registerGitHubConnectionContinuityWork } from "./services/github-connection-work.js"; import { registerProviderTraceRetentionWork } from "./services/provider-trace-retention-work.js"; import { registerRunEvidenceMigrationWork } from "./services/run-evidence-migration-work.js"; import { registerSkillVersionRetentionWork } from "./services/skill-version-retention-work.js"; @@ -1490,35 +1491,13 @@ async function startServerWithDatabaseTeardown( ?? process.env.PAPERCLIP_TOOL_RUNTIME_TRUSTED_HOST ?? null, }); - const scheduleGitHubConnectionEventPoll = (forSetup = false) => { - if (isIdleTaskDrainActive() || heartbeatSchedulerStopped) return; - const work = githubConnectionEvents.pollOnce({ forSetup }) - .then((result) => { - if (result.leased > 0 || result.failed > 0) { - logger.info(result, "GitHub connection event poll completed"); - } - }) - .catch((err) => { - logger.error({ err }, "GitHub connection event poll failed"); - }); - trackHeartbeatSchedulerWork(work); - return work; - }; + const githubCanRun = () => !isIdleTaskDrainActive() && !isWarmStandby() && !heartbeatSchedulerStopped; + const githubEventWorker = registerGitHubConnectionEventWork(db, deliveryWork, githubConnectionEvents, githubCanRun); + const githubContinuityWorker = registerGitHubConnectionContinuityWork(db, deliveryWork, tools, githubCanRun); + server.on("close", () => { void githubEventWorker.stop(); void githubContinuityWorker.stop(); }); server.on("close", registerGitHubBotCloudSetupPoller(db, async () => { - await scheduleGitHubConnectionEventPoll(true); + if (githubCanRun()) await githubConnectionEvents.pollOnce({ forSetup: true }); })); - const scheduleGitHubConnectionContinuitySweep = () => { - if (isIdleTaskDrainActive() || heartbeatSchedulerStopped) return; - trackHeartbeatSchedulerWork(tools.sweepGitHubConnectionContinuity() - .then((result) => { - if (result.due > 0 || result.failed > 0) { - logger.info(result, "GitHub connection continuity sweep completed"); - } - }) - .catch((err) => { - logger.error({ err }, "GitHub connection continuity sweep failed"); - })); - }; await timeStartupRecovery("tool_reviews", async () => { if (config.heartbeatSchedulerEnabled) { @@ -1549,8 +1528,7 @@ async function startServerWithDatabaseTeardown( () => !isIdleTaskDrainActive() && !heartbeatSchedulerStopped && !isWarmStandby()).ready : app.locals.toolGateway.cleanupExpiredSessions()).catch((err: unknown) => logger.error({ err }, "startup gateway token cleanup failed")); await timeStartupRecovery("delivery_workers", () => Promise.all([budgetEnforcementWorker, accountingWorker, fastResponseWorker, ...deliveryWorkers].map(worker => worker.ready))); - scheduleGitHubConnectionEventPoll(); - scheduleGitHubConnectionContinuitySweep(); + await Promise.all([githubEventWorker.ready, githubContinuityWorker.ready]); if (heartbeat) { const secretProposals = createSecretProposalsService(db as any); @@ -2011,8 +1989,6 @@ async function startServerWithDatabaseTeardown( if (isIdleTaskDrainActive() || heartbeatSchedulerStopped) return; scheduleMergedPullRequestConfirmationSweep(); - scheduleGitHubConnectionEventPoll(); - scheduleGitHubConnectionContinuitySweep(); if (isIdleTaskDrainActive() || heartbeatSchedulerStopped) return; trackHeartbeatSchedulerWork((async () => { @@ -2134,8 +2110,6 @@ async function startServerWithDatabaseTeardown( await startEnvironmentCleanup(); startHeartbeatSchedulerInterval(() => { scheduleExternalObjectRefreshSweep(new Date()); - scheduleGitHubConnectionEventPoll(); - scheduleGitHubConnectionContinuitySweep(); }); } diff --git a/server/src/middleware/cloud-control.ts b/server/src/middleware/cloud-control.ts index f0e44503e8b..b726aae3092 100644 --- a/server/src/middleware/cloud-control.ts +++ b/server/src/middleware/cloud-control.ts @@ -8,6 +8,7 @@ import { /** Endpoint → method → the one action a control assertion must name to take it. */ const ACTIONS_BY_ENDPOINT: Record> = { + "/api/instance/connector-events/wake": { POST: "connector-events:wake" }, "/api/instance/task-drain": { GET: "task-drain:read", POST: "task-drain:start", diff --git a/server/src/routes/connection-event-wake.ts b/server/src/routes/connection-event-wake.ts new file mode 100644 index 00000000000..c4a71e87717 --- /dev/null +++ b/server/src/routes/connection-event-wake.ts @@ -0,0 +1,13 @@ +import { Router } from "express"; +import { wakeConnectionEvents } from "../services/connection-event-wake.js"; + +export function connectionEventWakeRoutes(owner: object) { + const router = Router(); + router.post("/api/instance/connector-events/wake", (req, res) => { + if (req.actor.source !== "cloud_control") { res.status(403).json({ error: "cloud_control_required" }); return; } + res.set("Cache-Control", "no-store"); + if (!wakeConnectionEvents(owner)) { res.status(503).json({ error: "connection_event_receiver_unavailable" }); return; } + res.status(202).json({ accepted: true }); + }); + return router; +} diff --git a/server/src/services/cloud-runtime-identity.ts b/server/src/services/cloud-runtime-identity.ts index 435c7f7562b..80b3333b36f 100644 --- a/server/src/services/cloud-runtime-identity.ts +++ b/server/src/services/cloud-runtime-identity.ts @@ -451,6 +451,7 @@ export const CLOUD_CONTROL_ACTIONS = [ "task-drain:stop", "lifecycle:read", "lifecycle:unarchive-primary", + "connector-events:wake", ] as const; export type CloudControlAction = (typeof CLOUD_CONTROL_ACTIONS)[number]; diff --git a/server/src/services/companies.ts b/server/src/services/companies.ts index 0f2ed547a79..cce62d9c934 100644 --- a/server/src/services/companies.ts +++ b/server/src/services/companies.ts @@ -332,6 +332,8 @@ export function companyService(db: Db, _budgetHooks: BudgetServiceHooks = {}) { const renamedPrefix = await resolveRenamedIssuePrefix(tx, id, companyPatch); if (companyPatch.status !== undefined && companyPatch.status !== existing.status) { + await notifyDeliveryWork(tx, DELIVERY_QUEUES.connectionEvents); + await notifyDeliveryWork(tx, DELIVERY_QUEUES.connectionContinuity); await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers); await notifyDeliveryWork(tx, DELIVERY_QUEUES.skillSourceSync); await notifyDeliveryWork(tx, DELIVERY_QUEUES.routineSchedules); @@ -488,6 +490,8 @@ export function companyService(db: Db, _budgetHooks: BudgetServiceHooks = {}) { const wasAlreadyArchived = existing.status === "archived"; if (!wasAlreadyArchived) { + await notifyDeliveryWork(tx, DELIVERY_QUEUES.connectionEvents); + await notifyDeliveryWork(tx, DELIVERY_QUEUES.connectionContinuity); await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers); await notifyDeliveryWork(tx, DELIVERY_QUEUES.skillSourceSync); await notifyDeliveryWork(tx, DELIVERY_QUEUES.routineSchedules); diff --git a/server/src/services/connection-event-wake.ts b/server/src/services/connection-event-wake.ts new file mode 100644 index 00000000000..87840eb0979 --- /dev/null +++ b/server/src/services/connection-event-wake.ts @@ -0,0 +1,12 @@ +/** Process-local hint only; Cloud retains the durable target until event acknowledgment. */ +const receivers = new WeakMap void>(); +export function registerConnectionEventWake(owner: object, wake: () => void) { + receivers.set(owner, wake); + return () => { if (receivers.get(owner) === wake) receivers.delete(owner); }; +} +export function wakeConnectionEvents(owner: object): boolean { + const wake = receivers.get(owner); + if (!wake) return false; + wake(); + return true; +} diff --git a/server/src/services/delivery-work-notifications.ts b/server/src/services/delivery-work-notifications.ts index 9195f1fcb12..5e7648c8703 100644 --- a/server/src/services/delivery-work-notifications.ts +++ b/server/src/services/delivery-work-notifications.ts @@ -107,6 +107,8 @@ export const DELIVERY_QUEUES = { feedback: "feedback-exports", chatCompletion: "chat-completions", connection: "connection-continuations", + connectionEvents: "connection-events", + connectionContinuity: "connection-continuity", gitRestorePoints: "git-restore-points", nativeWorkspaceSyncRetention: "native-workspace-sync-retention", question: "question-responses", diff --git a/server/src/services/github-connection-events.test.ts b/server/src/services/github-connection-events.test.ts index 93031ecae62..2a7a15dbffc 100644 --- a/server/src/services/github-connection-events.test.ts +++ b/server/src/services/github-connection-events.test.ts @@ -3,7 +3,7 @@ import { chatEndpoints, type Db } from "@paperclipai/db"; import { githubConnectionEventService } from "./github-connection-events.js"; import type { PaperclipCloudConnector } from "./paperclip-cloud-connector.js"; -function fixture() { +function fixture(push = false) { let time = Date.parse("2026-09-04T12:00:00Z"); const bindings = [{ grant: { id: "grant", companyId: "company", kind: "user", subjectUserId: "owner", status: "active", @@ -13,13 +13,14 @@ function fixture() { const bots: Array<{ companyId: string }> = []; // Only SQL is stubbed. Backoff, lease admission and concurrent execution use // the real service; relational persistence has its own Postgres suite. + const result = (rows: T[]) => Object.assign(Promise.resolve(rows), { limit: async () => rows.slice(0, 1) }); const select = vi.fn(() => ({ from: (table: unknown) => table === chatEndpoints - ? { where: async () => bots } - : { innerJoin: () => ({ where: async () => bindings }) } })); + ? { where: () => result(bots) } + : { innerJoin: () => ({ where: () => result(bindings) }) } })); const leaseEvents = vi.fn().mockResolvedValue(null); - const connector = { leaseEvents } as unknown as PaperclipCloudConnector; + const connector = { leaseEvents, eventWakeVersion: () => push ? 1 : null } as unknown as PaperclipCloudConnector; const service = githubConnectionEventService({ select } as unknown as Db, { connector, now: () => new Date(time) }); - return { service, leaseEvents, select, bots, advance: (ms: number) => { time += ms; } }; + return { service, leaseEvents, select, bots, bindings, advance: (ms: number) => { time += ms; } }; } describe("GitHub event polling during active App setup", () => { @@ -72,3 +73,27 @@ describe("GitHub event polling during active App setup", () => { expect(f.leaseEvents).toHaveBeenCalledTimes(2); }); }); + +describe("GitHub external delivery activation", () => { + it("does no SQL/network with no connector configuration after startup or cadence windows", async () => { + const select = vi.fn(() => { throw new Error("unconfigured worker touched SQL"); }); + const service = githubConnectionEventService({ select } as unknown as Db, { env: {} }); + for (let i = 0; i < 4; i++) await service.pollOnce({ forDelivery: true }); + expect(service.nextPollAt()).toBeNull(); expect(select).not.toHaveBeenCalled(); + }); + it("settles configured-idle after enrollment and wakes only on explicit delivery hints", async () => { + const f = fixture(true); + await f.service.pollOnce(); expect(f.service.nextPollAt()).toBeNull(); + const reads = f.select.mock.calls.length; + for (let i = 0; i < 4; i++) { f.advance(300_000); await f.service.pollOnce(); } + expect(f.select).toHaveBeenCalledTimes(reads); expect(f.leaseEvents).toHaveBeenCalledTimes(1); + await f.service.pollOnce({ forDelivery: true }); expect(f.leaseEvents).toHaveBeenCalledTimes(2); + expect(f.service.nextPollAt()).toBeNull(); + }); + it("retains old-Cloud fallback and settles once all bindings are disabled", async () => { + const f = fixture(); await f.service.pollOnce(); expect(f.service.nextPollAt()).not.toBeNull(); + f.bindings.length = 0; f.advance(10_000); await f.service.pollOnce(); + expect(f.service.nextPollAt()).toBeNull(); const reads = f.select.mock.calls.length; + f.advance(3600_000); await f.service.pollOnce(); expect(f.select).toHaveBeenCalledTimes(reads); + }); +}); diff --git a/server/src/services/github-connection-events.ts b/server/src/services/github-connection-events.ts index 47bdac122a2..17fff9a993c 100644 --- a/server/src/services/github-connection-events.ts +++ b/server/src/services/github-connection-events.ts @@ -1,3 +1,4 @@ +import { notifyGitHubCredentialChange } from "./github-connection-notifications.js"; import { notifyToolConnectionHealth } from "./tool-connection-health-notifications.js"; import { dispatchGitHubBotCloudEvent } from "./chat-github-cloud-ingress.js"; import { chatEndpoints } from "@paperclipai/db"; @@ -221,6 +222,9 @@ export function githubConnectionEventService( ) { const now = options.now ?? (() => new Date()); let nextPollAt = 0; + const config = options.connector ? null : paperclipCloudConnectorConfigFromEnv(options.env); + const connector = options.connector ?? (config ? createPaperclipCloudConnector({ config }) : null); + const forcePolling = (options.env ?? process.env).PAPERCLIP_GITHUB_EVENT_POLLING === "1"; let emptyPolls = 0; let nextSetupPollAt = 0; let inFlight: Promise | null = null; @@ -236,6 +240,7 @@ export function githubConnectionEventService( eq(connectionGrants.status, "active"), eq(toolConnections.status, "active"), eq(toolConnections.enabled, true), + sql`exists (select 1 from companies where id = ${connectionGrants.companyId} and status <> 'archived')`, )); return bindingRows(rows); } @@ -318,6 +323,11 @@ export function githubConnectionEventService( // Lifecycle webhooks carry IDs, not the user token’s complete repository view. // Discard the snapshot until Refresh access verifies it again. accessRevision: randomUUID(), + ...(unavailable ? { bindingSync: { + id: randomUUID(), removeInstallationIds: [...new Set([ + ...(github.bindingSync?.removeInstallationIds ?? []), binding.installationId, + ])].filter(id => !installationIds.includes(id)), + } } : {}), repositories: undefined, installationIds, installationCount: installationIds.length, @@ -327,6 +337,7 @@ export function githubConnectionEventService( webhookHealth: unavailable ? "unhealthy" as const : "healthy" as const, }, }; + await notifyGitHubCredentialChange(database, { config: { sourceTemplateKey: "github" } }); await database.update(connectionGrants).set({ providerTenant, updatedAt: now() }) .where(and(eq(connectionGrants.id, binding.grantId), eq(connectionGrants.companyId, binding.companyId))); await notifyToolConnectionHealth(database, { companyId: binding.companyId, id: binding.connectionId }); @@ -366,16 +377,20 @@ export function githubConnectionEventService( eq(connectionEventDeliveries.providerDeliveryId, event.id), )).limit(1); if (existing?.status === "processed") return "duplicate" as const; - await db.update(connectionEventDeliveries).set({ - status: "received", - attempts: sql`${connectionEventDeliveries.attempts} + 1`, - lastError: null, - updatedAt: receiptAt, - }).where(eq(connectionEventDeliveries.id, existing!.id)); + } const postCommitPublications: ActivityPublication[] = []; + let applied = true; try { const applyAndFinalize = async (database: Db) => { + const [locked] = await database.select().from(connectionEventDeliveries).where(and( + eq(connectionEventDeliveries.companyId, companyId), eq(connectionEventDeliveries.provider, event.provider), + eq(connectionEventDeliveries.providerDeliveryId, event.id), + )).for("update").limit(1); + if (locked!.status === "processed") { applied = false; return; } + if (!receipt) await database.update(connectionEventDeliveries).set({ + status: "received", attempts: sql`${connectionEventDeliveries.attempts} + 1`, lastError: null, updatedAt: receiptAt, + }).where(eq(connectionEventDeliveries.id, locked!.id)); if (event.event === "pull_request") await applyPullRequestEvent(companyId, normalizedEvent); if (event.event === "installation" || event.event === "installation_repositories") { for (const binding of bindings) await applyInstallationEvent(database, binding, normalizedEvent); @@ -428,11 +443,7 @@ export function githubConnectionEventService( }, }, postCommitPublications); }; - if (event.event === "installation" || event.event === "installation_repositories") { - await db.transaction(async (tx) => applyAndFinalize(tx as unknown as Db)); - } else { - await applyAndFinalize(db); - } + await db.transaction(async (tx) => applyAndFinalize(tx as unknown as Db)); } catch (error) { await db.update(connectionEventDeliveries).set({ status: "failed", @@ -459,31 +470,45 @@ export function githubConnectionEventService( }, "GitHub webhook activity publication failed after commit"); } } - return "processed" as const; + return applied ? "processed" as const : "duplicate" as const; } async function runPoll(): Promise { - const bindings = await activeBindings(); - const bots = await db.select().from(chatEndpoints).where(and(eq(chatEndpoints.provider, "github"), sql`${chatEndpoints.setup}->'github'->>'cloudRegistrationId' is not null`, sql`${chatEndpoints.status} <> 'archived'`)); - if (bindings.length === 0 && bots.length === 0) { - nextPollAt = now().getTime() + 5_000; - return { leased: 0, processed: 0, duplicate: 0, ignored: 0, failed: 0 }; - } - const config = options.connector ? null : paperclipCloudConnectorConfigFromEnv(options.env); - const connector = options.connector ?? (config ? createPaperclipCloudConnector({ config }) : null); if (!connector) { - nextPollAt = now().getTime() + 5 * 60_000; + nextPollAt = Number.POSITIVE_INFINITY; return { leased: 0, processed: 0, duplicate: 0, ignored: 0, failed: 0 }; } - const first = bindings[0] ?? { subject: "github-bot-events", companyId: bots[0]!.companyId }; - const lease = await connector.leaseEvents({ subject: first.subject, companyId: first.companyId }); + const bindings = await activeBindings(); + const bots = await db.select().from(chatEndpoints).where(and(eq(chatEndpoints.provider, "github"), sql`${chatEndpoints.setup}->'github'->>'cloudRegistrationId' is not null`, sql`${chatEndpoints.status} not in ('archived', 'disabled')`, sql`exists (select 1 from companies where id = ${chatEndpoints.companyId} and status <> 'archived')`)); + let identity = bindings[0] ?? (bots[0] ? { subject: "github-bot-events", companyId: bots[0].companyId } : null); + const hasConsumer = bindings.length > 0 || bots.length > 0; + if (!identity) { + const [history] = await db.select({ grant: connectionGrants, connection: toolConnections }).from(connectionGrants) + .innerJoin(toolConnections, and(eq(toolConnections.id, connectionGrants.connectionId), eq(toolConnections.companyId, connectionGrants.companyId))) + .where(and(sql`${toolConnections.config}->>'sourceTemplateKey' = 'github'`, sql`${toolConnections.config}->'oauth'->>'connectorProfile' = 'github.code'`)).limit(1); + const grant = history?.grant; + const subject = grant?.kind === "agent" && grant.subjectAgentId ? `agent:${grant.subjectAgentId}` + : grant?.kind === "user" ? grant.subjectUserId : null; + if (grant && subject) identity = { subject, companyId: grant.companyId }; + } + if (!identity) { + const [history] = await db.select().from(chatEndpoints).where(and(eq(chatEndpoints.provider, "github"), sql`${chatEndpoints.setup}->'github'->>'cloudRegistrationId' is not null`)).limit(1); + if (history) identity = { subject: "github-bot-events", companyId: history.companyId }; + } + if (!identity) { + nextPollAt = Number.POSITIVE_INFINITY; + return { leased: 0, processed: 0, duplicate: 0, ignored: 0, failed: 0 }; + } + const first = identity; + const lease = await connector.leaseEvents({ subject: first.subject, companyId: first.companyId, eventWakeVersion: forcePolling ? 0 : 1 }); if (!lease) { emptyPolls += 1; - nextPollAt = now().getTime() + Math.min(bots.length ? 15_000 : 5 * 60_000, 5_000 * (2 ** Math.min(emptyPolls, 6))); + nextPollAt = !hasConsumer || (!forcePolling && connector.eventWakeVersion?.() === 1) ? Number.POSITIVE_INFINITY + : now().getTime() + Math.min(bots.length ? 15_000 : 5 * 60_000, 5_000 * (2 ** Math.min(emptyPolls, 6))); return { leased: 0, processed: 0, duplicate: 0, ignored: 0, failed: 0 }; } emptyPolls = 0; - nextPollAt = now().getTime() + 5_000; + nextPollAt = !hasConsumer || (!forcePolling && connector.eventWakeVersion?.() === 1) ? Number.POSITIVE_INFINITY : now().getTime() + 5_000; const result: GitHubConnectionEventPollResult = { leased: lease.events.length + (lease.failedEventIds?.length ?? 0), processed: 0, @@ -512,17 +537,7 @@ export function githubConnectionEventService( result[status] += 1; } acknowledge.push(event.id); - if (event.event === "installation" && (event.action === "deleted" || event.action === "suspend")) { - await Promise.all(matched.map((binding) => connector.setWebhookBinding({ - subject: binding.subject, - companyId: binding.companyId, - id: binding.id, - installationId: binding.installationId, - connectionId: binding.connectionId, - grantId: binding.grantId, - active: false, - }))); - } + } catch { result.failed += 1; } @@ -543,13 +558,14 @@ export function githubConnectionEventService( } return { - pollOnce(input: { forSetup?: boolean } = {}): Promise { + nextPollAt: () => Number.isFinite(nextPollAt) ? nextPollAt : null, + pollOnce(input: { forSetup?: boolean; forDelivery?: boolean } = {}): Promise { // A manager actively awaiting a dedicated App's signed ping must not // inherit a personal connection's old empty-queue backoff. This is a // bounded readiness probe, not an additional background polling loop. if (inFlight) return inFlight; const time = now().getTime(); - if (time < (input.forSetup ? nextSetupPollAt : nextPollAt)) { + if (!input.forDelivery && time < (input.forSetup ? nextSetupPollAt : nextPollAt)) { return Promise.resolve({ leased: 0, processed: 0, duplicate: 0, ignored: 0, failed: 0 }); } nextSetupPollAt = time + 5_000; diff --git a/server/src/services/github-connection-notifications.ts b/server/src/services/github-connection-notifications.ts new file mode 100644 index 00000000000..79f3c067aba --- /dev/null +++ b/server/src/services/github-connection-notifications.ts @@ -0,0 +1,7 @@ +import type { toolConnections } from "@paperclipai/db"; +import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js"; +export async function notifyGitHubCredentialChange(transaction: object, connection: Pick) { + if (connection.config.sourceTemplateKey !== "github") return; + await notifyDeliveryWork(transaction, DELIVERY_QUEUES.connectionContinuity); + await notifyDeliveryWork(transaction, DELIVERY_QUEUES.connectionEvents); +} diff --git a/server/src/services/github-connection-work.ts b/server/src/services/github-connection-work.ts new file mode 100644 index 00000000000..7849cf3bda8 --- /dev/null +++ b/server/src/services/github-connection-work.ts @@ -0,0 +1,36 @@ +import { logger } from "../middleware/logger.js"; +import type { Db } from "@paperclipai/db"; +import type { createDeliveryWorkCoordinator } from "./delivery-work-coordinator.js"; +import { registerDeadlineWork } from "./deadline-work.js"; +import { DELIVERY_QUEUES, subscribeDeliveryWork } from "./delivery-work-notifications.js"; +import { registerConnectionEventWake } from "./connection-event-wake.js"; +import type { githubConnectionEventService } from "./github-connection-events.js"; +import type { createGitHubGrantContinuity } from "./tool-access/github-grant-continuity.js"; + +type Coordinator = ReturnType; +export function registerGitHubConnectionEventWork(db: Db, coordinator: Coordinator, + service: ReturnType, canRun: () => boolean) { + const worker = registerDeadlineWork(coordinator, DELIVERY_QUEUES.connectionEvents, { + retryMs: 5_000, canRun, + run: () => service.pollOnce({ forDelivery: true }), + next: async () => service.nextPollAt(), + }); + const stops = [registerConnectionEventWake(db, worker.wake), + subscribeDeliveryWork(db, DELIVERY_QUEUES.toolConnectionHealth, worker.wake), + subscribeDeliveryWork(db, DELIVERY_QUEUES.chatReceivers, worker.wake)]; + return { ...worker, stop: async () => { for (const stop of stops) stop(); await worker.stop(); } }; +} +export function registerGitHubConnectionContinuityWork(db: Db, coordinator: Coordinator, + service: Pick, "sweepGitHubConnectionContinuity" | "nextGitHubConnectionContinuityAt">, + canRun: () => boolean) { + const worker = registerDeadlineWork(coordinator, DELIVERY_QUEUES.connectionContinuity, { + retryMs: 30_000, canRun, + run: async () => { + const result = await service.sweepGitHubConnectionContinuity(); + if (result.failed) logger.warn(result, "GitHub credential continuity retained retry obligations"); + }, + next: () => service.nextGitHubConnectionContinuityAt(), + }); + const unsubscribe = subscribeDeliveryWork(db, DELIVERY_QUEUES.toolConnectionHealth, worker.wake); + return { ...worker, stop: async () => { unsubscribe(); await worker.stop(); } }; +} diff --git a/server/src/services/paperclip-cloud-connector.ts b/server/src/services/paperclip-cloud-connector.ts index b1461d3e14d..6159051c5d3 100644 --- a/server/src/services/paperclip-cloud-connector.ts +++ b/server/src/services/paperclip-cloud-connector.ts @@ -1,3 +1,4 @@ +import { providerRetryAt } from "./provider-retry-after.js"; import { SLACK_MANAGER_SCOPES, ASANA_CONNECTOR_SCOPES, isAsanaConnectorProfileId, type AsanaConnectorProfileId } from "@paperclipai/shared"; import { createDecipheriv, @@ -114,6 +115,7 @@ type ConnectorResponse = { events?: unknown; acknowledged?: unknown; githubApps?: { version: number }; + eventWakeVersion?: unknown; }; const ENDPOINTS: Record = { @@ -189,6 +191,7 @@ export class PaperclipCloudConnectorError extends Error { message: string, readonly code: string, readonly status?: number, + readonly retryAt?: number, ) { super(message); this.name = "PaperclipCloudConnectorError"; @@ -272,10 +275,11 @@ export function createPaperclipCloudConnector(input: { const now = input.now ?? Date.now; const signingKey = privateKey(config.signPrivateKey, "ed25519"); const sealKey = privateKey(config.sealPrivateKey, "x25519"); + let eventWakeVersion: 1 | null = null; async function call( operation: PaperclipCloudConnectorOperation, - claims: { subject: string; companyId: string; profile?: PaperclipCloudConnectorProfileId; scopes?: string[]; returnUri?: string; returnState?: string; claimId?: string; redemptionId?: string }, + claims: { subject: string; companyId: string; profile?: PaperclipCloudConnectorProfileId; scopes?: string[]; returnUri?: string; returnState?: string; claimId?: string; redemptionId?: string; eventWakeVersion?: 0 | 1 }, secret?: { field: "refreshToken" | "token" | "binding" | "acknowledgement"; value: string }, ): Promise { const endpoint = new URL(ENDPOINTS[operation], `${config.baseUrl}/`).toString(); @@ -291,6 +295,7 @@ export function createPaperclipCloudConnector(input: { exp: issuedAt + 60, jti: randomUUID(), }; + if (claims.eventWakeVersion !== undefined) payload.ewv = claims.eventWakeVersion; if (claims.returnUri !== undefined) payload.ruri = claims.returnUri; if (claims.returnState !== undefined) payload.rst = claims.returnState; if (claims.claimId !== undefined) payload.cl = claims.claimId; @@ -327,6 +332,7 @@ export function createPaperclipCloudConnector(input: { `Paperclip Cloud connector rejected the request (operation=${operation}, status=${response.status}, reason=${reason})`, response.status === 409 ? "REAUTHORIZATION_REQUIRED" : "CONNECTOR_REQUEST_FAILED", response.status, + providerRetryAt(response.headers.get("retry-after"), now()), ); } try { @@ -374,6 +380,7 @@ export function createPaperclipCloudConnector(input: { } return { + eventWakeVersion: () => eventWakeVersion, async githubAppsAvailable() { const response = await call("status", { subject: "instance-capabilities", companyId: "instance-capabilities" }); return response.active === true && response.githubApps?.version === 2; @@ -520,8 +527,9 @@ export function createPaperclipCloudConnector(input: { }); await call("webhook-bind", { ...values, profile: "github.code" }, { field: "binding", value: binding }); }, - async leaseEvents(values: { subject: string; companyId: string }): Promise<{ leaseId: string; events: SealedConnectorEvents["events"]; failedEventIds?: string[] } | null> { + async leaseEvents(values: { subject: string; companyId: string; eventWakeVersion?: 0 | 1 }): Promise<{ leaseId: string; events: SealedConnectorEvents["events"]; failedEventIds?: string[] } | null> { const response = await call("event-lease", values); + eventWakeVersion = response.eventWakeVersion === 1 && values.eventWakeVersion !== 0 ? 1 : null; if (Array.isArray(response.events) && response.events.length === 0) return null; if (typeof response.leaseId !== "string") { throw new PaperclipCloudConnectorError("Paperclip Cloud connector returned an invalid event lease", "CONNECTOR_BAD_RESPONSE"); diff --git a/server/src/services/provider-retry-after.test.ts b/server/src/services/provider-retry-after.test.ts new file mode 100644 index 00000000000..b10fa764f27 --- /dev/null +++ b/server/src/services/provider-retry-after.test.ts @@ -0,0 +1,26 @@ +import { describe, expect, it } from "vitest"; +import { providerRetryAt } from "./provider-retry-after.js"; +import { githubContinuityRevision, githubGrantContinuityDeadline } from "./tool-access/github-grant-continuity.js"; +const now = Date.parse("2026-01-01T00:00:00Z"); +describe("provider and credential deadlines", () => { + it("accepts bounded Retry-After seconds/dates and ignores malformed or expired advice", () => { + expect(providerRetryAt("120", now)).toBe(now + 120_000); + expect(providerRetryAt(new Date(now + 60_000).toUTCString(), now)).toBe(now + 60_000); + expect(providerRetryAt("999999999999", now)).toBe(now + 24 * 3600_000); + for (const value of [null, "", "unknown", "-3", new Date(now - 1).toUTCString()]) expect(providerRetryAt(value, now)).toBeUndefined(); + }); + it("reconstructs expiry, continuity and saved retries and preempts a retry when credentials change", () => { + const connection = { config: { sourceTemplateKey: "github", oauth: { connectorProfile: "github.code" } } }; + const github = { userId: "42", login: "owner", installationCount: 1, repositoryCount: 1, repositorySelection: "all" as const, + installationIds: ["101"], installationOwnerLogins: ["owner"], lastAccessRefreshAt: new Date(now).toISOString() }; + const grant = { providerTenant: { github, oauth: { accessTokenExpiresAt: new Date(now + 8 * 3600_000).toISOString(), refreshedAt: new Date(now).toISOString() } } }; + expect(githubGrantContinuityDeadline(connection, grant)).toBe(now + 7 * 3600_000); + const retried = { providerTenant: { ...grant.providerTenant, github: { ...github, + continuityRetry: { at: new Date(now + 10 * 3600_000).toISOString(), attempts: 2, revision: githubContinuityRevision(grant) } } } }; + expect(githubGrantContinuityDeadline(connection, retried)).toBe(now + 10 * 3600_000); + retried.providerTenant.oauth.accessTokenExpiresAt = new Date(now + 2 * 3600_000).toISOString(); + expect(githubGrantContinuityDeadline(connection, retried)).toBe(now + 3600_000); + expect(githubGrantContinuityDeadline(connection, { providerTenant: { github } })).toBe(now + 30 * 24 * 3600_000); + expect(githubGrantContinuityDeadline({ config: {} }, grant)).toBeNull(); + }); +}); diff --git a/server/src/services/provider-retry-after.ts b/server/src/services/provider-retry-after.ts new file mode 100644 index 00000000000..c90364ea7e6 --- /dev/null +++ b/server/src/services/provider-retry-after.ts @@ -0,0 +1,6 @@ +export function providerRetryAt(value: string | null, now = Date.now()): number | undefined { + if (!value?.trim()) return undefined; + const seconds = /^\d+$/.test(value.trim()) ? Number(value) : Number.NaN; + const deadline = Number.isFinite(seconds) ? now + seconds * 1000 : Date.parse(value); + return Number.isFinite(deadline) && deadline > now ? Math.min(deadline, now + 24 * 3600_000) : undefined; +} diff --git a/server/src/services/tool-access.ts b/server/src/services/tool-access.ts index 81b6f646c3c..7d4cfe8cf0d 100644 --- a/server/src/services/tool-access.ts +++ b/server/src/services/tool-access.ts @@ -787,7 +787,7 @@ export function toolAccessService( const refreshLeases = createOAuthRefreshLeases(db, now, getConnectionRow, getOAuthGrantRow); const maybeRefreshOAuthCredentials = createLegacyOAuthRefresh(db, { now, secrets, store: credentialStore, leases: refreshLeases, getConnectionRow, exchangeOAuthToken }); const refreshOAuthGrantCredentials = createOAuthGrantRefresh(db, { now, store: credentialStore, leases: refreshLeases, currentCloudConnector, getConnectionRow, exchangeOAuthToken }); - const { refreshManagedGitHubGrantAccess, sweepGitHubConnectionContinuity } = createGitHubGrantContinuity(db, { now, currentCloudConnector, refreshOAuthGrantCredentials, resolveOAuthGrantSecret }); + const { refreshManagedGitHubGrantAccess, syncGitHubGrantBindings, sweepGitHubConnectionContinuity, nextGitHubConnectionContinuityAt } = createGitHubGrantContinuity(db, { now, currentCloudConnector, refreshOAuthGrantCredentials, resolveOAuthGrantSecret }); const vercelCredentials = createVercelCredentialService(db, now, vercelConnect); const { resolveVercelCredentialHeaders, vercelGrantForConnection } = vercelCredentials; const credentialAccess = createConnectionCredentialAccess(db, { secrets, audit, vercel: vercelCredentials, resolveOAuthGrantSecret, maybeRefreshOAuthCredentials, refreshOAuthGrantCredentials }); @@ -811,7 +811,7 @@ export function toolAccessService( const finishOAuthCatalogWithRecommendedDefaults = createOAuthCatalogCompletion(db, finishGalleryAppConnection); const startOAuth = createOAuthAuthorizationStart(db, { now, getConnectionRow, setupPolicy, fixedPersonalIdentityForReconnect, currentCloudConnector, vercelConnect, ensureOAuthClient, oauthEndpointsForConnection, assertOAuthEndpointUrl }); const completeOAuthCallback = createDirectOAuthCallback(db, { now, getConnectionRow, state: oauthState, store: credentialStore, setupPolicy, refreshCatalog, finishOAuthCatalogWithRecommendedDefaults, checkConnectionHealth, ensureDefaultOrganizationGrant, oauthEndpointsForConnection, exchangeOAuthToken }); - const completePaperclipCloudConnectorCallback = createCloudOAuthCallback(db, { now, getConnectionRow, state: oauthState, store: credentialStore, refreshCatalog, finishOAuthCatalogWithRecommendedDefaults, currentCloudConnector }); + const completePaperclipCloudConnectorCallback = createCloudOAuthCallback(db, { now, getConnectionRow, state: oauthState, store: credentialStore, refreshCatalog, finishOAuthCatalogWithRecommendedDefaults, currentCloudConnector, syncGitHubGrantBindings }); const completeVercelConnectCallback = createVercelOAuthCallback(db, { now, getConnectionRow, state: oauthState, refreshCatalog, finishOAuthCatalogWithRecommendedDefaults, vercelConnect }); const finalizeOAuthAccess = createOAuthAccessFinalization(db, { getConnectionRow, secrets, store: credentialStore, assertAgentsInCompany, finishGalleryAppConnection }); const runtimeHealth = createToolRuntimeHealth(db, options, now, trustedRuntimeHost); @@ -4732,6 +4732,7 @@ export function toolAccessService( scheduledHealth: connectionHealthSchedule(db, checkConnectionHealth), sweepGitHubConnectionContinuity, + nextGitHubConnectionContinuityAt, ...catalogReader, ...toolActionReviewQueue, ...profileService, diff --git a/server/src/services/tool-access/connections/grants.ts b/server/src/services/tool-access/connections/grants.ts index 1836a6a52fc..572bb0e8856 100644 --- a/server/src/services/tool-access/connections/grants.ts +++ b/server/src/services/tool-access/connections/grants.ts @@ -1,3 +1,4 @@ +import { notifyGitHubCredentialChange } from "../../github-connection-notifications.js"; import type { ConnectionManagementContext } from "../../connection-management.js"; import { connectionActorUserId } from "../actor.js"; import { toConnectionGrant } from "./projection.js"; @@ -537,6 +538,7 @@ replaceConnectionGrantMembers: async ( providerRevocation = "local_only_shared_client"; } const grant = await db.transaction(async (tx) => { + await notifyGitHubCredentialChange(tx, connection); const [lockedGrant] = await tx.select({ id: connectionGrants.id }).from(connectionGrants) .where(and(eq(connectionGrants.id, grantId), eq(connectionGrants.companyId, connection.companyId), eq(connectionGrants.connectionId, connection.id))) .for("update"); diff --git a/server/src/services/tool-access/github-access-discovery.ts b/server/src/services/tool-access/github-access-discovery.ts index 4f0912b000e..2d45377ab6e 100644 --- a/server/src/services/tool-access/github-access-discovery.ts +++ b/server/src/services/tool-access/github-access-discovery.ts @@ -1,3 +1,4 @@ +import { providerRetryAt } from "../provider-retry-after.js"; import { randomUUID } from "node:crypto"; import { unprocessable } from "../../errors.js"; @@ -106,6 +107,7 @@ export async function loadGitHubGrantMetadata( throw unprocessable( "GitHub could not verify this account's installation access", { + retryAt: providerRetryAt(response.headers.get("retry-after")), code: response.status === 401 ? "oauth_reauthorization_required" diff --git a/server/src/services/tool-access/github-grant-continuity.ts b/server/src/services/tool-access/github-grant-continuity.ts index 568a0a00a99..0398c2a79be 100644 --- a/server/src/services/tool-access/github-grant-continuity.ts +++ b/server/src/services/tool-access/github-grant-continuity.ts @@ -1,12 +1,14 @@ +import { randomUUID } from "node:crypto"; +import { notifyGitHubCredentialChange } from "../github-connection-notifications.js"; import type { ActorInfo } from "./types.js"; import { loadGitHubGrantMetadata } from "./github-access-discovery.js"; import { measureRunPhase } from "../execution-preparation-timing.js"; import { measureRunTransaction } from "../execution-transaction-timing.js"; -import { and, eq } from "drizzle-orm"; +import { and, eq, sql } from "drizzle-orm"; import type { Db } from "@paperclipai/db"; import { connectionGrants, toolConnections } from "@paperclipai/db"; import { conflict, HttpError, notFound, unprocessable } from "../../errors.js"; -import { type PaperclipCloudConnector } from "../paperclip-cloud-connector.js"; +import { PaperclipCloudConnectorError, type PaperclipCloudConnector } from "../paperclip-cloud-connector.js"; import type { createOAuthGrantRefresh } from "./oauth/grant-refresh.js"; import type { OAuthCredentialStore } from "./oauth/credential-store.js"; @@ -24,7 +26,6 @@ type GitHubContinuityDependencies = { }; export function createGitHubGrantContinuity(db: Db, { now, currentCloudConnector, refreshOAuthGrantCredentials, resolveOAuthGrantSecret }: GitHubContinuityDependencies) { - let nextGitHubContinuitySweepAt = 0; async function refreshManagedGitHubGrantAccess( connection: typeof toolConnections.$inferSelect, initialGrant: typeof connectionGrants.$inferSelect, @@ -138,7 +139,8 @@ export function createGitHubGrantContinuity(db: Db, { now, currentCloudConnector throw retryError; } } - const { updated, previousGitHub } = await measureRunTransaction(db, "connectors.github.refresh", async (tx) => { + const { updated } = await measureRunTransaction(db, "connectors.github.refresh", async (tx) => { + await notifyGitHubCredentialChange(tx, connection); const [currentGrant] = await measureRunPhase("connectors.github.grant_lock", () => tx .select() .from(connectionGrants) @@ -170,6 +172,13 @@ export function createGitHubGrantContinuity(db: Db, { now, currentCloudConnector ...(currentGrant.providerTenant ?? {}), github: { ...metadata, + bindingSync: { + id: randomUUID(), + removeInstallationIds: [...new Set([ + ...(previousGitHub?.bindingSync?.removeInstallationIds ?? []), + ...(previousGitHub?.installationIds ?? []), + ])].filter(id => !metadata.installationIds.includes(id)), + }, ...(previousGitHub?.lastWebhookAt ? { lastWebhookAt: previousGitHub.lastWebhookAt } : {}), @@ -195,111 +204,161 @@ export function createGitHubGrantContinuity(db: Db, { now, currentCloudConnector return { updated, previousGitHub }; }); + return syncGitHubGrantBindings(connection, updated, actor, accessToken); + } + + async function syncGitHubGrantBindings( + connection: typeof toolConnections.$inferSelect, + initialGrant: typeof connectionGrants.$inferSelect, + actor?: ActorInfo, + accessToken?: string, + ) { + await actor?.managementContext?.reauthorize(); const cloudConnector = currentCloudConnector(); - const subject = - updated.kind === "agent" && updated.subjectAgentId - ? `agent:${updated.subjectAgentId}` - : updated.kind === "user" && updated.subjectUserId - ? updated.subjectUserId - : null; - if (cloudConnector && subject) { - const previous = new Set(previousGitHub?.installationIds ?? []); - const current = new Set(metadata.installationIds); - await measureRunPhase("connectors.github.webhooks", () => Promise.all([ - ...metadata.installationIds.map((installationId) => - cloudConnector.setWebhookBinding({ - subject, - companyId: connection.companyId, - id: `${updated.id}_${installationId}`, - installationId, - connectionId: connection.id, - grantId: updated.id, - active: true, - accessToken, - }), - ), - ...[...previous] - .filter((installationId) => !current.has(installationId)) - .map((installationId) => - cloudConnector.setWebhookBinding({ - subject, - companyId: connection.companyId, - id: `${updated.id}_${installationId}`, - installationId, - connectionId: connection.id, - grantId: updated.id, - active: false, - }), - ), - ])); - } - return updated; + if (!initialGrant.providerTenant?.github?.bindingSync) return initialGrant; + if (!cloudConnector) throw unprocessable("GitHub webhook delivery is not configured"); + const grant = accessToken || initialGrant.providerTenant.github.installationIds.length === 0 ? initialGrant : await refreshOAuthGrantCredentials({ + companyId: connection.companyId, connectionId: connection.id, grantId: initialGrant.id, actor, + }); + const accessRef = grant.credentialSecretRefs.find(ref => ref.configPath === "oauth.access_token"); + if (!accessRef && grant.providerTenant?.github?.installationIds.length) throw unprocessable("GitHub authorization must be reconnected", { code: "oauth_reauthorization_required" }); + const token = accessToken ?? (accessRef ? (await resolveOAuthGrantSecret(connection, grant, accessRef, actor, undefined)).value : undefined); + // Serialize external binding changes with grant/connection lifecycle writes. + // Requests occur only for saved debt; no idle lock or connection is retained. + return db.transaction(async tx => { + const [current] = await tx.select().from(connectionGrants).where(and( + eq(connectionGrants.id, grant.id), eq(connectionGrants.companyId, connection.companyId), + eq(connectionGrants.connectionId, connection.id), + )).for("update").limit(1); + if (!current || current.status !== "active") return current ?? grant; + const [liveConnection] = await tx.select().from(toolConnections).where(and( + eq(toolConnections.id, connection.id), eq(toolConnections.companyId, connection.companyId), + )).for("update").limit(1); + if (!liveConnection || liveConnection.status !== "active" || !liveConnection.enabled) return current; + const github = current.providerTenant?.github; + if (!github?.bindingSync || github.bindingSync.id !== grant.providerTenant?.github?.bindingSync?.id) return current; + const subject = current.kind === "agent" && current.subjectAgentId ? `agent:${current.subjectAgentId}` + : current.kind === "user" ? current.subjectUserId : null; + if (!subject) throw unprocessable("GitHub webhook subject is unavailable"); + for (const installationId of github.installationIds) await cloudConnector.setWebhookBinding({ + subject, companyId: connection.companyId, id: `${current.id}_${installationId}`, + installationId, connectionId: connection.id, grantId: current.id, active: true, accessToken: token!, + }); + for (const installationId of github.bindingSync.removeInstallationIds) await cloudConnector.setWebhookBinding({ + subject, companyId: connection.companyId, id: `${current.id}_${installationId}`, + installationId, connectionId: connection.id, grantId: current.id, active: false, + }); + await notifyGitHubCredentialChange(tx, connection); + const [settled] = await tx.update(connectionGrants).set({ + providerTenant: sql`${connectionGrants.providerTenant} #- '{github,bindingSync}' #- '{github,continuityRetry}'`, + updatedAt: now(), + }).where(and(eq(connectionGrants.id, current.id), eq(connectionGrants.companyId, connection.companyId))).returning(); + return settled!; + }); + } + + async function retainRetry(connection: typeof toolConnections.$inferSelect, grantId: string, error?: unknown, minimumMs = 30_000) { + await db.transaction(async tx => { + const [current] = await tx.select().from(connectionGrants).where(and( + eq(connectionGrants.id, grantId), eq(connectionGrants.companyId, connection.companyId), + eq(connectionGrants.connectionId, connection.id), eq(connectionGrants.status, "active"), + )).for("update").limit(1); + if (!current?.providerTenant?.github) return; + const attempts = Math.min(16, (current.providerTenant.github.continuityRetry?.attempts ?? 0) + 1); + const advised = error instanceof PaperclipCloudConnectorError ? error.retryAt + : error instanceof HttpError ? asRecord(error.details).retryAt : undefined; + const at = new Date(Math.max(now().getTime() + Math.max(minimumMs, Math.min(60 * 60_000, 30_000 * 2 ** Math.min(attempts - 1, 7))), + typeof advised === "number" ? advised : 0)).toISOString(); + await notifyGitHubCredentialChange(tx, connection); + await tx.update(connectionGrants).set({ + providerTenant: sql`jsonb_set(${connectionGrants.providerTenant}, '{github,continuityRetry}', ${JSON.stringify({ + at, attempts, revision: githubContinuityRevision(current), + })}::jsonb)`, + updatedAt: now(), + }).where(and(eq(connectionGrants.id, current.id), eq(connectionGrants.companyId, connection.companyId))); + }); + } + + async function continuityRows() { + return db + .select({ grant: connectionGrants, connection: toolConnections }) + .from(connectionGrants) + .innerJoin(toolConnections, and( + eq(toolConnections.id, connectionGrants.connectionId), + eq(toolConnections.companyId, connectionGrants.companyId), + )) + .where(and( + eq(connectionGrants.status, "active"), + eq(toolConnections.status, "active"), + eq(toolConnections.enabled, true), + sql`${toolConnections.config}->>'sourceTemplateKey' = 'github'`, + sql`${toolConnections.config}->'oauth'->>'connectorProfile' = 'github.code'`, + sql`${connectionGrants.providerTenant}->'github' is not null`, + sql`exists (select 1 from companies where id = ${connectionGrants.companyId} and status = 'active')`, + )); + } + + async function nextGitHubConnectionContinuityAt(): Promise { + const deadlines = (await continuityRows()).flatMap(({ grant, connection }) => { + const deadline = githubGrantContinuityDeadline(connection, grant); + return deadline === null ? [] : [deadline]; + }); + return deadlines.length ? Math.min(...deadlines) : null; } async function sweepGitHubConnectionContinuity() { - if (now().getTime() < nextGitHubContinuitySweepAt) { - return { checked: 0, due: 0, refreshed: 0, failed: 0 }; - } - nextGitHubContinuitySweepAt = now().getTime() + 60 * 60_000; - const cutoff = now().getTime() - 30 * 24 * 60 * 60_000; - const rows = await db - .select({ grant: connectionGrants, connection: toolConnections }) - .from(connectionGrants) - .innerJoin( - toolConnections, - and( - eq(toolConnections.id, connectionGrants.connectionId), - eq(toolConnections.companyId, connectionGrants.companyId), - ), - ) - .where( - and( - eq(connectionGrants.status, "active"), - eq(toolConnections.status, "active"), - eq(toolConnections.enabled, true), - ), - ); + const rows = await continuityRows(); const due = rows.filter(({ grant, connection }) => { - const config = asRecord(connection.config); - const oauth = asRecord(config.oauth); - if ( - config.sourceTemplateKey !== "github" || - oauth.connectorProfile !== "github.code" || - !grant.providerTenant?.github - ) - return false; - const expiresAt = grant.providerTenant.oauth?.accessTokenExpiresAt; - const refreshedAt = grant.providerTenant.oauth?.refreshedAt - ? Date.parse(grant.providerTenant.oauth.refreshedAt) - : Number.NaN; - const accessCheckedAt = grant.providerTenant.github.lastAccessRefreshAt - ? Date.parse(grant.providerTenant.github.lastAccessRefreshAt) - : Number.NaN; - if (typeof expiresAt === "string") { - const expiry = Date.parse(expiresAt); - return ( - !Number.isFinite(expiry) || - expiry <= now().getTime() + 60 * 60_000 || - !Number.isFinite(refreshedAt) || - refreshedAt <= cutoff - ); - } - return !Number.isFinite(accessCheckedAt) || accessCheckedAt <= cutoff; + const deadline = githubGrantContinuityDeadline(connection, grant); + return deadline !== null && deadline <= now().getTime(); }); let refreshed = 0; let failed = 0; for (const row of due) { try { - await refreshManagedGitHubGrantAccess(row.connection, row.grant, { - actorType: "system", - actorId: "github-connection-continuity", - }); + const actor: ActorInfo = { actorType: "system", actorId: "github-connection-continuity" }; + const updated = row.grant.providerTenant?.github?.bindingSync + ? await syncGitHubGrantBindings(row.connection, row.grant, actor) + : await refreshManagedGitHubGrantAccess(row.connection, row.grant, actor); + // A successful provider response can still leave an unchanged/short-lived + // credential due. Retain that obligation with bounded durable backoff. + const next = githubGrantContinuityDeadline(row.connection, updated); + if (next !== null && next <= now().getTime()) await retainRetry(row.connection, updated.id, undefined, 60 * 60_000); refreshed += 1; - } catch { + } catch (error) { + await retainRetry(row.connection, row.grant.id, error); failed += 1; } } return { checked: rows.length, due: due.length, refreshed, failed }; } - return { refreshManagedGitHubGrantAccess, sweepGitHubConnectionContinuity }; + return { refreshManagedGitHubGrantAccess, syncGitHubGrantBindings, sweepGitHubConnectionContinuity, nextGitHubConnectionContinuityAt }; +} + +/** Credential expiry and provider continuity are obligations, not a scan cadence. */ +export function githubGrantContinuityDeadline( + connection: Pick, + grant: Pick, +): number | null { + const config = asRecord(connection.config); + if (config.sourceTemplateKey !== "github" || asRecord(config.oauth).connectorProfile !== "github.code" + || !grant.providerTenant?.github) return null; + const oauth = grant.providerTenant.oauth; + const access = grant.providerTenant.github; + const parsed = (value: string | undefined) => { + const time = value ? Date.parse(value) : Number.NaN; + return Number.isFinite(time) ? time : 0; + }; + const continuityMs = 30 * 24 * 60 * 60_000; + const credentialDeadline = typeof oauth?.accessTokenExpiresAt === "string" + ? Math.min(parsed(oauth.accessTokenExpiresAt) - 60 * 60_000, oauth.refreshedAt ? parsed(oauth.refreshedAt) + continuityMs : 0) + : access.lastAccessRefreshAt ? parsed(access.lastAccessRefreshAt) + continuityMs : 0; + const deadline = access.bindingSync ? 0 : credentialDeadline; + return access.continuityRetry?.revision === githubContinuityRevision(grant) + ? Math.max(deadline, parsed(access.continuityRetry.at)) : deadline; +} + +export function githubContinuityRevision(grant: Pick): string { + return JSON.stringify([grant.providerTenant?.oauth?.accessTokenExpiresAt, grant.providerTenant?.oauth?.refreshedAt, + grant.providerTenant?.github?.lastAccessRefreshAt, grant.providerTenant?.github?.bindingSync?.id]); } diff --git a/server/src/services/tool-access/oauth/cloud-callback.ts b/server/src/services/tool-access/oauth/cloud-callback.ts index fac5bce24ba..7381c1687a4 100644 --- a/server/src/services/tool-access/oauth/cloud-callback.ts +++ b/server/src/services/tool-access/oauth/cloud-callback.ts @@ -1,3 +1,5 @@ +import type { createGitHubGrantContinuity } from "../github-grant-continuity.js"; +import { randomUUID } from "node:crypto"; import { notifyIssueReviewState } from "../../issue-review-state-notifications.js"; import { notifyToolConnectionHealth } from "../../tool-connection-health-notifications.js"; import { managedConnectorProfile } from "./client-settings.js"; @@ -32,6 +34,7 @@ type Dependencies = { finishOAuthCatalogWithRecommendedDefaults: OAuthCatalogCompletion; store: Pick; currentCloudConnector: () => PaperclipCloudConnector | null; + syncGitHubGrantBindings: ReturnType["syncGitHubGrantBindings"]; }; export function createCloudOAuthCallback(db: Db, dependencies: Dependencies) { @@ -301,6 +304,7 @@ export function createCloudOAuthCallback(db: Db, dependencies: Dependencies) { accessRef, ...(refreshRef ? [refreshRef] : []), ]; + await notifyToolConnectionHealth(tx, connection); const grantValues = { providerTenant: { name: githubMetadata?.login ?? providerName, @@ -319,7 +323,12 @@ export function createCloudOAuthCallback(db: Db, dependencies: Dependencies) { ? { refreshedAt: now().toISOString() } : {}), }, - ...(githubMetadata ? { github: githubMetadata } : {}), + ...(githubMetadata ? { github: { ...githubMetadata, bindingSync: { + id: randomUUID(), removeInstallationIds: [...new Set([ + ...(existingCredentialGrant?.providerTenant?.github?.bindingSync?.removeInstallationIds ?? []), + ...(existingCredentialGrant?.providerTenant?.github?.installationIds ?? []), + ])].filter(id => !githubMetadata.installationIds.includes(id)), + } } } : {}), }, credentialSecretRefs, status: "active" as const, @@ -358,7 +367,6 @@ export function createCloudOAuthCallback(db: Db, dependencies: Dependencies) { scopes: [...profile.scopes], }, }; - await notifyToolConnectionHealth(tx, connection); [connection] = await tx .update(toolConnections) .set({ @@ -438,42 +446,8 @@ export function createCloudOAuthCallback(db: Db, dependencies: Dependencies) { // Paperclip Cloud connector callback is the managed variant of an OAuth // callback completion. emitConnectionUpdated(connection, preCloudCallbackLifecycle, "oauth_callback"); - if (githubMetadata) { - const [githubGrant] = await db - .select({ id: connectionGrants.id }) - .from(connectionGrants) - .where( - and( - eq(connectionGrants.companyId, connection.companyId), - eq(connectionGrants.connectionId, connection.id), - subjectAgentId - ? and( - eq(connectionGrants.kind, "agent"), - eq(connectionGrants.subjectAgentId, subjectAgentId), - ) - : and( - eq(connectionGrants.kind, "user"), - eq(connectionGrants.subjectUserId, subjectUserId!), - ), - ), - ) - .limit(1); - if (!githubGrant) throw new Error("GitHub grant was not persisted"); - await Promise.all( - githubMetadata.installationIds.map((installationId) => - cloudConnector.setWebhookBinding({ - subject: connectorSubject, - companyId: connection.companyId, - id: `${githubGrant.id}_${installationId}`, - installationId, - connectionId: connection.id, - grantId: githubGrant.id, - active: true, - accessToken: credentials.accessToken, - }), - ), - ); - } + // Binding registration is saved above before Cloud I/O. The continuity + // worker drains it after activation, including after a callback/server crash. const refresh = await refreshCatalog(connection.id, input.actor, { enableAllByDefault: false, quarantineManagedOAuthDraft: shouldFinalizeManagedDefaults, @@ -494,6 +468,15 @@ export function createCloudOAuthCallback(db: Db, dependencies: Dependencies) { actor: input.actor, }) : null; + if (githubMetadata) { + const [grant] = await db.select().from(connectionGrants).where(and( + eq(connectionGrants.companyId, connection.companyId), eq(connectionGrants.connectionId, connection.id), + subjectAgentId ? and(eq(connectionGrants.kind, "agent"), eq(connectionGrants.subjectAgentId, subjectAgentId)) + : and(eq(connectionGrants.kind, "user"), eq(connectionGrants.subjectUserId, subjectUserId!)), + )).limit(1); + if (!grant) throw new Error("GitHub grant was not persisted"); + await dependencies.syncGitHubGrantBindings(await getConnectionRow(connection.id, connection.companyId), grant, input.actor, credentials.accessToken); + } const activatedCatalogEntryIds = new Set( shouldFinalizeManagedDefaults ? refresh.catalog diff --git a/server/src/services/tool-access/oauth/grant-refresh.ts b/server/src/services/tool-access/oauth/grant-refresh.ts index 48e09cf27b9..ce0ca102352 100644 --- a/server/src/services/tool-access/oauth/grant-refresh.ts +++ b/server/src/services/tool-access/oauth/grant-refresh.ts @@ -1,3 +1,4 @@ +import { notifyGitHubCredentialChange } from "../../github-connection-notifications.js"; import { writeToolConnectionHealthChange } from "../../tool-connection-health-notifications.js"; import type { ActorInfo } from "../types.js"; import { createOAuthClientProtocol, storedOAuthTokenEndpointAuthMethod } from "./client.js"; @@ -193,6 +194,7 @@ export function createOAuthGrantRefresh(db: Db, { now, store, leases, currentClo }, }; const updated = await db.transaction(async (tx) => { + await notifyGitHubCredentialChange(tx, connection); const [current] = await tx.select({ id: connectionGrants.id }).from(connectionGrants) .where(and(eq(connectionGrants.id, grant.id), eq(connectionGrants.companyId, grant.companyId), eq(connectionGrants.status, "active"), @@ -539,7 +541,7 @@ export function createOAuthGrantRefresh(db: Db, { now, store, leases, currentClo refreshedAt: now().toISOString(), }, }; - const [updated] = await db + const [updated] = await writeToolConnectionHealthChange(db, connection, tx => tx .update(connectionGrants) .set({ providerTenant, @@ -555,7 +557,7 @@ export function createOAuthGrantRefresh(db: Db, { now, store, leases, currentClo sql`${connectionGrants.providerTenant} -> 'oauth' -> 'refreshLease' ->> 'id' = ${lease.leaseId}`, ), ) - .returning(); + .returning()); if (!updated) { throw conflict( "OAuth credentials changed while refresh was in progress. Retry the request.", From a4dbaff604e89eef0d1537a27f65c45dde3fc1a3 Mon Sep 17 00:00:00 2001 From: Dotta Date: Fri, 30 Oct 2026 17:10:07 -0500 Subject: [PATCH 20/74] fix(github): narrow the event delivery identity contract Co-Authored-By: Paperclip --- server/src/services/github-connection-events.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/services/github-connection-events.ts b/server/src/services/github-connection-events.ts index 17fff9a993c..53834d6c4f5 100644 --- a/server/src/services/github-connection-events.ts +++ b/server/src/services/github-connection-events.ts @@ -480,7 +480,7 @@ export function githubConnectionEventService( } const bindings = await activeBindings(); const bots = await db.select().from(chatEndpoints).where(and(eq(chatEndpoints.provider, "github"), sql`${chatEndpoints.setup}->'github'->>'cloudRegistrationId' is not null`, sql`${chatEndpoints.status} not in ('archived', 'disabled')`, sql`exists (select 1 from companies where id = ${chatEndpoints.companyId} and status <> 'archived')`)); - let identity = bindings[0] ?? (bots[0] ? { subject: "github-bot-events", companyId: bots[0].companyId } : null); + let identity: { subject: string; companyId: string } | null = bindings[0] ?? (bots[0] ? { subject: "github-bot-events", companyId: bots[0].companyId } : null); const hasConsumer = bindings.length > 0 || bots.length > 0; if (!identity) { const [history] = await db.select({ grant: connectionGrants, connection: toolConnections }).from(connectionGrants) From 6fe6d6d29cbefd041435c6282d6b2dd7507140c9 Mon Sep 17 00:00:00 2001 From: Dotta Date: Fri, 30 Oct 2026 17:10:15 -0500 Subject: [PATCH 21/74] fix(server): bound retention reconstruction and preserve producer transactions Batch review and confirmation eligibility per company, scope decision and budget hints, and keep child blocker relation intents inside their owning transaction. Cover ordinary child, budget and execution producers plus full candidate-page query bounds; update startup/import fixtures for source-driven retention. Co-Authored-By: Paperclip --- .../2026-10-30-attention-retention-sleep.md | 4 +- .../attention-retention-work.test.ts | 75 +++++++++++- .../src/__tests__/company-portability.test.ts | 1 + .../decision-effect-recovery-work.test.ts | 4 +- .../src/__tests__/decisions-service.test.ts | 2 +- .../server-startup-feedback-export.test.ts | 18 ++- server/src/services/attention-retention.ts | 113 ++++++++++++------ .../services/attention-source-eligibility.ts | 4 + server/src/services/attention.ts | 7 +- server/src/services/budgets.ts | 21 ++-- server/src/services/decision-expiry.ts | 2 +- .../services/decision-work-notifications.ts | 14 ++- server/src/services/decisions.ts | 25 ++-- server/src/services/heartbeat-run-events.ts | 6 +- server/src/services/issues.ts | 47 ++++---- 15 files changed, 242 insertions(+), 101 deletions(-) diff --git a/doc/plans/2026-10-30-attention-retention-sleep.md b/doc/plans/2026-10-30-attention-retention-sleep.md index c0e4e168bd0..1b71dc71261 100644 --- a/doc/plans/2026-10-30-attention-retention-sleep.md +++ b/doc/plans/2026-10-30-attention-retention-sleep.md @@ -25,7 +25,7 @@ The feed and worker share confirmation collapse, newer-run probes, issue visibil ## Existing mutation boundaries reactivate work -Committed hints identify a company, source kind, and source ID where those values are known. Retention's own writes use exact source hints; blocker retention changes invalidate the company/source because multiple blocked roots can project one terminal subject. Reconstruction pages and due batches remain bounded. +Committed hints identify a company, source kind, and source ID where those values are known. Retention's own writes use exact source hints; blocker retention changes invalidate the company/source because multiple blocked roots can project one terminal subject. Reconstruction pages and due batches remain bounded. Review attention and interaction invokability are computed in company batches per candidate page; a full 100-candidate review page uses at most 20 statements and 100 confirmations use at most six. Ordinary execution, decision and budget mutations pass their affected company; exact decision sources are included. Producer coverage includes decision creation and changes; interaction creation, resolution, expiry and supersession; issue creation, updates, blocker relations, checkout, adoption and release; agent hire, configuration, lifecycle, error clearing and deletion; budget incidents and company pause/resume; exhausted-run events and retry-run creation; native status, review, finalization and reconstruction; external conversation transitions; and imported blocker relations. Existing service transactions retain outer commit ownership. Savepoint release does not publish work before outer commit. @@ -33,7 +33,7 @@ The writer contract follows the existing delivery workers: serving processes use ## Real PostgreSQL evidence proves quiet and reactivation -The new acceptance suite exercises all eight sources, including terminal blocker identity and date, newest confirmations, independent quality gates, unhealthy reporting chains, conversation visibility, maintained reviews and pending approvals, newer-run suppression, ordinary task and interaction producers, ordinary exhaustion-event ingress, and ordinary error clearing. +The new acceptance suite exercises all eight sources, including terminal blocker identity and date, newest confirmations, independent quality gates, unhealthy reporting chains, conversation visibility, maintained reviews and pending approvals, newer-run suppression, ordinary task and interaction producers, ordinary exhaustion-event ingress, and ordinary error clearing. Additional ordinary producer controls cover child creation with a parent-blocking relation, budget incident creation and resolution, and repeated company-scoped execution-control writes. It also checks keep/revive, moved and cancelled deadlines, company activation and isolation, retained terminal history, restart, nested outer commit and rollback, first writes during drain, retry across drain, atomic archive/outbox rollback, and notifications during reconstruction. A 225-item backlog uses three startup pages and source-specific self-invalidations. SQL is counted at the PostgreSQL driver boundary. diff --git a/server/src/__tests__/attention-retention-work.test.ts b/server/src/__tests__/attention-retention-work.test.ts index 22a966d5fef..889030d2c22 100644 --- a/server/src/__tests__/attention-retention-work.test.ts +++ b/server/src/__tests__/attention-retention-work.test.ts @@ -2,9 +2,12 @@ import { randomUUID } from "node:crypto"; import { and, eq, sql } from "drizzle-orm"; import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; import { authUsers, companyMemberships, agents, approvals, budgetIncidents, budgetPolicies, companies, createDb, applyPendingMigrations, decisionArchiveNotificationOutbox, - decisionRetention, decisions, heartbeatRunEvents, heartbeatRuns, issueApprovals, issueRecoveryActions, + decisionRetention, decisions, costEvents, heartbeatRunEvents, heartbeatRuns, issueApprovals, issueRecoveryActions, issueRelations, issueThreadInteractions, issues, type Db } from "@paperclipai/db"; import { attentionRetentionService, registerAttentionRetentionWork } from "../services/attention-retention.js"; +import { budgetService } from "../services/budgets.js"; +import { writeExecutionControl } from "../services/execution-control-notifications.js"; +import { DELIVERY_QUEUES } from "../services/delivery-work-notifications.js"; import { attentionService } from "../services/attention.js"; import { companyService } from "../services/companies.js"; import { createDeliveryWorkCoordinator } from "../services/delivery-work-coordinator.js"; @@ -135,6 +138,76 @@ const delay = DEFAULT_DECISION_ARCHIVE_DAYS * 86_400_000; await coordinator!.stop(); coordinator = undefined; await start(); await quiet(); }); + it("registers the ordinary createChild blocker path inside its relation transaction", async () => { + const parent = await issueService(db).create(companyId, { title: "Parent", status: "in_progress", assigneeAgentId: agentId, createdByAgentId: agentId }); + await start(); await quiet(); + const { issue: child, parentBlockerAdded } = await issueService(db).createChild(parent.id, { + title: "Child needs review", status: "in_review", assigneeUserId: "board", createdByAgentId: agentId, + blockParentUntilDone: true, + }); + expect(parentBlockerAdded).toBe(true); + expect(await db.select().from(issueRelations).where(and(eq(issueRelations.issueId, child.id), + eq(issueRelations.relatedIssueId, parent.id)))).toHaveLength(1); + await settle(); expect(coordinator!.nextWakeAt()).not.toBeNull(); + await vi.advanceTimersByTimeAsync(delay + 2000); await archived("review", child.id); await quiet(); + expect(await db.select().from(decisionArchiveNotificationOutbox).where(eq(decisionArchiveNotificationOutbox.sourceId, child.id))).toHaveLength(1); + }); + + it("batches a full page of maintained reviews and confirmations by company", async () => { + const rows = await db.insert(issues).values(Array.from({ length: 100 }, () => ({ companyId, + title: randomUUID(), status: "in_review", assigneeAgentId: agentId, updatedAt: old() }))).returning(); + await db.insert(issueThreadInteractions).values(rows.map(row => ({ companyId, issueId: row.id, + kind: "request_confirmation", addresseeAgentId: agentId, payload: { version: 1, prompt: "Agent verdict" } }))); + statements.length = 0; + const page = await attentionRetentionService(db).load({ companyId, sourceKind: "review" }); + expect(page.rows).toEqual([]); expect(page.complete).toBe(false); + expect(statements.length).toBeLessThanOrEqual(20); + expect(statements.filter(query => query.includes('from "agents"')).length).toBeLessThanOrEqual(1); + await db.update(agents).set({ status: "paused" }).where(eq(agents.id, agentId)); + statements.length = 0; + const interactions = await attentionRetentionService(db).load({ companyId, sourceKind: "issue_thread_interaction" }); + expect(interactions.rows).toHaveLength(100); expect(interactions.complete).toBe(false); + expect(statements.length).toBeLessThanOrEqual(6); + expect(statements.filter(query => query.includes('from "agents"'))).toHaveLength(1); + }); + + it("scopes ordinary execution-control writes to their company", async () => { + const otherCompany = randomUUID(); + await db.insert(companies).values({ id: otherCompany, name: "Other", issuePrefix: "OTHER" }); + await db.insert(issues).values(Array.from({ length: 125 }, () => ({ companyId: otherCompany, + title: randomUUID(), status: "in_review", assigneeUserId: "board", updatedAt: old() }))); + const [run] = await db.insert(heartbeatRuns).values({ companyId, agentId, status: "running" }).returning(); + const service = attentionRetentionService(db), load = vi.spyOn(service, "load"); + await start(service); await quiet(); load.mockClear(); + for (let count = 0; count < 3; count++) { + await writeExecutionControl(db, DELIVERY_QUEUES.executionFinalization, + tx => tx.update(heartbeatRuns).set({ updatedAt: new Date() }).where(eq(heartbeatRuns.id, run.id)), { companyId }); + await quiet(); + } + expect(load).toHaveBeenCalled(); + expect(load.mock.calls.every(([scope]) => scope?.companyId === companyId)).toBe(true); + }); + + it("registers and cancels budget deadlines through ordinary policy and incident producers", async () => { + const service = budgetService(db); + await service.upsertPolicy(companyId, { scopeType: "agent", scopeId: agentId, amount: 100, + warnPercent: 85, hardStopEnabled: false, notifyEnabled: true }, "board"); + const retention = attentionRetentionService(db), load = vi.spyOn(retention, "load"); + await start(retention); await quiet(); load.mockClear(); + const [event] = await db.insert(costEvents).values({ companyId, agentId, provider: "openai", biller: "openai", + billingType: "metered_api", model: "fixture", inputTokens: 100, outputTokens: 20, + costCents: 90, occurredAt: new Date() }).returning(); + await service.evaluateCostEvent(event); + await settle(); expect(coordinator!.nextWakeAt()).not.toBeNull(); + const [incident] = await db.select().from(budgetIncidents).where(eq(budgetIncidents.companyId, companyId)); + expect(incident).toMatchObject({ status: "open", thresholdType: "soft" }); + expect(await state("budget_alert", incident.id)).toBeNull(); + await service.upsertPolicy(companyId, { scopeType: "agent", scopeId: agentId, amount: 200 }, "board"); + await quiet(); + expect((await db.select().from(budgetIncidents).where(eq(budgetIncidents.id, incident.id)))[0].status).toBe("resolved"); + expect(load.mock.calls.every(([scope]) => scope?.companyId === companyId)).toBe(true); + }); + it("matches the existing feed across all eight remaining source kinds and exclusion variants", async () => { const origin = await insertIssue(); const review = await insertIssue("in_review", { assigneeUserId: "board" }); diff --git a/server/src/__tests__/company-portability.test.ts b/server/src/__tests__/company-portability.test.ts index fdf00b828af..ee7db993937 100644 --- a/server/src/__tests__/company-portability.test.ts +++ b/server/src/__tests__/company-portability.test.ts @@ -4184,6 +4184,7 @@ describe("company portability", () => { }), // eslint-disable-next-line @typescript-eslint/no-explicit-any } as any; + db.transaction = async (write: (tx: typeof db) => Promise) => write(db); return { db, insertedRelationValues, monitorUpdates }; } diff --git a/server/src/__tests__/decision-effect-recovery-work.test.ts b/server/src/__tests__/decision-effect-recovery-work.test.ts index 6c1a17d2b39..8a7d4e300c8 100644 --- a/server/src/__tests__/decision-effect-recovery-work.test.ts +++ b/server/src/__tests__/decision-effect-recovery-work.test.ts @@ -73,7 +73,7 @@ const support = externalUrl ? { supported: true } : await getEmbeddedPostgresTes const claim = (id: string, updatedAt = new Date(), owner = db) => writeDecisionChange(owner, async tx => tx.update(decisions).set({ status: "decided", executionStatus: "running", chosenOptionId: "yes", decidedByUserId: userId, updatedAt, inputValues: {}, metadata: { continuationPending: true }, - }).where(eq(decisions.id, id))); + }).where(eq(decisions.id, id)), { companyId }); async function start() { coordinator = createDeliveryWorkCoordinator({ owner: db, canRun: () => !draining, canReconcile: () => true, onError: error => errors.push(error) }); @@ -467,7 +467,7 @@ const support = externalUrl ? { supported: true } : await getEmbeddedPostgresTes } else { await writeDecisionChange(db, async tx => tx.update(decisions).set({ status: "decided", executionStatus: "running", chosenOptionId: "archive", decidedByUserId: userId, updatedAt: new Date(Date.now() - 120_000), - metadata: { ...row.metadata, continuationPending: true } }).where(eq(decisions.id, row.id))); + metadata: { ...row.metadata, continuationPending: true } }).where(eq(decisions.id, row.id)), { companyId }); } await start(); await settled(row.id, committed ? "failed" : "succeeded"); await quiet(); const [retained] = await db.select().from(decisionRetention); diff --git a/server/src/__tests__/decisions-service.test.ts b/server/src/__tests__/decisions-service.test.ts index 7fc2cc50c0d..999bde8d60e 100644 --- a/server/src/__tests__/decisions-service.test.ts +++ b/server/src/__tests__/decisions-service.test.ts @@ -793,7 +793,7 @@ describePg("decisionService", () => { expect((await service().get(id))?.metadata.continuationPending).toBe(false); } const markReady = (id: string, source = db) => writeDecisionChange(source, async tx => tx.update(decisions) - .set({ status: "cancelled", metadata: { continuationPending: true } }).where(eq(decisions.id, id))); + .set({ status: "cancelled", metadata: { continuationPending: true } }).where(eq(decisions.id, id)), { companyId }); it("coalesces sequential tenant writes without dropping pending recovery", async () => { const row = await createCommentDecision(); diff --git a/server/src/__tests__/server-startup-feedback-export.test.ts b/server/src/__tests__/server-startup-feedback-export.test.ts index bbb020c865d..bc72ad57f02 100644 --- a/server/src/__tests__/server-startup-feedback-export.test.ts +++ b/server/src/__tests__/server-startup-feedback-export.test.ts @@ -1,3 +1,4 @@ +import { attentionRetentionService } from "../services/attention-retention.js"; import { nativeHarnessBackupExpirationService } from "../services/native-runtime/native-harness-backup-expiration.js"; import { skillVersionRetentionService } from "../services/skill-version-retention.js"; import { approvalQueueSeedingService } from "../services/approval-queue-seeding-work.js"; @@ -377,6 +378,11 @@ vi.mock("../realtime/live-events-ws.js", () => ({ setupLiveEventsWebSocketServer: vi.fn(), })); +vi.mock("../services/attention-retention.js", async importOriginal => ({ + ...await importOriginal(), + attentionRetentionService: vi.fn(() => ({ load: vi.fn(async () => ({ rows: [], complete: true })), check: vi.fn() })), +})); + vi.mock("../services/join-request-retention.js", async importOriginal => ({ ...await importOriginal(), joinRequestRetentionService: vi.fn(() => ({ sweep: vi.fn(async () => ({ archived: 0 })), next: vi.fn(async () => null) })), @@ -974,7 +980,7 @@ describe("startServer feedback export wiring", () => { } }); - it.each([false, true])("registers archive delivery separately from retention sweeps (scheduler %s)", async enabled => { + it.each([false, true])("registers source retention separately from archive delivery without tick scans (scheduler %s)", async enabled => { loadConfigMock.mockReturnValue(buildTestConfig({ heartbeatSchedulerEnabled: enabled, heartbeatSchedulerIntervalMs: 30000 })); let tick: (() => void) | undefined; const timer = vi.spyOn(globalThis, "setInterval").mockImplementation(((callback: () => void, interval: number) => { @@ -986,12 +992,17 @@ describe("startServer feedback export wiring", () => { const registration = deliveryWorkRegisterMock.mock.calls.find(([queue]) => queue === DELIVERY_QUEUES.decisionArchiveDelivery); if (!enabled) { expect(registration).toBeUndefined(); + expect(attentionRetentionService).not.toHaveBeenCalled(); expect(decisionRetentionService).not.toHaveBeenCalled(); expect(tick).toBeDefined(); tick!(); await settleStartupHousekeeping(); expect(decisionRetentionService).not.toHaveBeenCalled(); return; } expect(registration).toBeDefined(); + expect(deliveryWorkRegisterMock.mock.calls.some(([queue]) => queue === DELIVERY_QUEUES.attentionRetention)).toBe(true); + expect(attentionRetentionService).toHaveBeenCalledOnce(); + const source = vi.mocked(attentionRetentionService).mock.results[0]!.value; + expect(source.load).toHaveBeenCalledOnce(); const retention = vi.mocked(decisionRetentionService).mock.results[0]!.value; expect(retention.deliverNotifications).toHaveBeenCalledOnce(); expect(retention.nextArchiveNotificationAt).toHaveBeenCalledOnce(); @@ -999,9 +1010,10 @@ describe("startServer feedback export wiring", () => { const companyReads = () => vi.mocked(db.select).mock.results.filter(({ value }) => vi.mocked(value.from).mock.calls.some(([table]) => table === companies)).length; const before = companyReads(); - expect(before).toBeGreaterThan(0); expect(tick).toBeDefined(); + expect(before).toBe(0); expect(tick).toBeDefined(); tick!(); await settleStartupHousekeeping(); - expect(companyReads()).toBeGreaterThan(before); + expect(companyReads()).toBe(before); + expect(source.load).toHaveBeenCalledOnce(); expect(retention.deliverNotifications).toHaveBeenCalledOnce(); await registration![1].run(new AbortController().signal); expect(retention.deliverNotifications).toHaveBeenCalledTimes(2); diff --git a/server/src/services/attention-retention.ts b/server/src/services/attention-retention.ts index 4622c4883bf..85e27a0d941 100644 --- a/server/src/services/attention-retention.ts +++ b/server/src/services/attention-retention.ts @@ -1,13 +1,14 @@ -import { and, eq, isNull, sql, type SQL } from "drizzle-orm"; +import { and, eq, inArray, isNull, sql, type SQL } from "drizzle-orm"; import { agents, decisionRetention, decisions, heartbeatRuns, issueApprovals, issueRecoveryActions, issueThreadInteractions, issues, approvals, type Db } from "@paperclipai/db"; import type { AttentionSourceKind } from "@paperclipai/shared"; import { evaluateAgentInvokability } from "./agent-invokability.js"; -import { collapsePendingConfirmationsToNewest, listFailedRunsWithNewerRun, readRunIssueId, requiresReviewAttention, terminalBlockerRequiresAttention } from "./attention-source-eligibility.js"; +import { HUMAN_RECOVERY_OWNER_TYPES, ATTENTION_BUDGET_WARNING_PERCENT, ATTENTION_AGENT_ERROR_STATUS, collapsePendingConfirmationsToNewest, listFailedRunsWithNewerRun, readRunIssueId, requiresReviewAttention, terminalBlockerRequiresAttention } from "./attention-source-eligibility.js"; import { decisionRetentionService, DEFAULT_DECISION_ARCHIVE_DAYS } from "./decision-retention.js"; import type { createDeliveryWorkCoordinator } from "./delivery-work-coordinator.js"; import { DELIVERY_QUEUES } from "./delivery-work-notifications.js"; import { attentionIssueCondition } from "./issue-visibility.js"; +import { RETRY_EXHAUSTION_MESSAGE_PREFIX } from "./heartbeat-run-events.js"; import { issueService } from "./issues.js"; import { parseIssueExecutionState } from "./issue-execution-policy.js"; import { isProspectiveBlockedTransition } from "./routable-blocked.js"; @@ -34,7 +35,7 @@ const sources: Record = { issue_thread_interaction: sql`select 'issue_thread_interaction', i.id::text, i.company_id, i.updated_at from issue_thread_interactions i where i.status = 'pending' and i.addressee_user_id is null`, recovery_action: sql`select 'recovery_action', r.id::text, r.company_id, r.updated_at - from issue_recovery_actions r where r.status in ('active', 'escalated') and r.owner_type in ('user', 'board')`, + from issue_recovery_actions r where r.status in ('active', 'escalated') and r.owner_type in (${sql.join(HUMAN_RECOVERY_OWNER_TYPES.map(owner => sql`${owner}`), sql`, `)})`, blocker_attention: sql`select 'blocker_attention', i.id::text, i.company_id, i.updated_at from issues i where i.status = 'blocked' and i.hidden_at is null and i.harness_kind is null`, review: sql`select 'review', i.id::text, i.company_id, i.updated_at @@ -42,11 +43,11 @@ const sources: Record = { failed_run: sql`select 'failed_run', r.id::text, r.company_id, coalesce(r.finished_at, r.updated_at, r.created_at) from heartbeat_runs r where r.status in ('failed', 'timed_out') and exists ( select 1 from heartbeat_run_events e where e.run_id = r.id and e.company_id = r.company_id - and e.event_type = 'lifecycle' and e.message like 'Bounded retry exhausted%')`, + and e.event_type = 'lifecycle' and e.message like ${`${RETRY_EXHAUSTION_MESSAGE_PREFIX}%`})`, budget_alert: sql`select 'budget_alert', b.id::text, b.company_id, b.updated_at from budget_incidents b where b.status = 'open' and (b.threshold_type = 'hard' or (b.amount_limit > 0 - and round(b.amount_observed::numeric / b.amount_limit * 10000) / 100 >= 85))`, - agent_error_alert: sql`select 'agent_error_alert', a.id::text, a.company_id, a.updated_at from agents a where a.status = 'error'`, + and round(b.amount_observed::numeric / b.amount_limit * 10000) / 100 >= ${ATTENTION_BUDGET_WARNING_PERCENT}))`, + agent_error_alert: sql`select 'agent_error_alert', a.id::text, a.company_id, a.updated_at from agents a where a.status = ${ATTENTION_AGENT_ERROR_STATUS}`, }; export function attentionRetentionService(db: Db) { @@ -74,7 +75,69 @@ export function attentionRetentionService(db: Db) { return row; } - async function eligible(database: Db, candidate: Candidate, variant?: "owner" | "terminal", lock = false): Promise { + async function interactionEligibility(database: Db, candidates: Candidate[]) { + const result = new Map(); + const byCompany = new Map(); + for (const candidate of candidates) { + if (candidate.sourceKind !== "issue_thread_interaction") continue; + const ids = byCompany.get(candidate.companyId) ?? []; + ids.push(candidate.sourceId); + byCompany.set(candidate.companyId, ids); + } + for (const [companyId, ids] of byCompany) { + const rows = await database.select().from(issueThreadInteractions).where(and( + eq(issueThreadInteractions.companyId, companyId), inArray(issueThreadInteractions.id, ids), + eq(issueThreadInteractions.status, "pending"), isNull(issueThreadInteractions.addresseeUserId))); + if (!rows.length) continue; + const visible = await database.select({ id: issues.id }).from(issues).where(and(eq(issues.companyId, companyId), + inArray(issues.id, rows.map(row => row.issueId)), attentionIssueCondition(true))); + const visibleIds = new Set(visible.map(row => row.id)); + const roster = await database.select().from(agents).where(eq(agents.companyId, companyId)); + const boardAddressed = (row: typeof issueThreadInteractions.$inferSelect) => !row.addresseeAgentId + || !evaluateAgentInvokability(roster.find(agent => agent.id === row.addresseeAgentId), roster).invokable; + const confirmationIssueIds = rows.filter(row => row.kind === "request_confirmation").map(row => row.issueId); + const siblings = confirmationIssueIds.length ? await database.select().from(issueThreadInteractions).where(and( + eq(issueThreadInteractions.companyId, companyId), inArray(issueThreadInteractions.issueId, confirmationIssueIds), + eq(issueThreadInteractions.status, "pending"), eq(issueThreadInteractions.kind, "request_confirmation"), + isNull(issueThreadInteractions.addresseeUserId))) : []; + const newest = new Set(collapsePendingConfirmationsToNewest(siblings.filter(boardAddressed)).map(row => row.id)); + for (const row of rows) result.set(row.id, visibleIds.has(row.issueId) && boardAddressed(row) + && (row.kind !== "request_confirmation" || newest.has(row.id))); + } + return result; + } + + async function reviewEligibility(database: Db, candidates: Candidate[]) { + const result = new Map(); + const byCompany = new Map(); + for (const candidate of candidates) { + if (candidate.sourceKind !== "review") continue; + const ids = byCompany.get(candidate.companyId) ?? []; + ids.push(candidate.sourceId); + byCompany.set(candidate.companyId, ids); + } + for (const [companyId, ids] of byCompany) { + const rows = await database.select().from(issues).where(and(eq(issues.companyId, companyId), + inArray(issues.id, ids), attentionIssueCondition(false))); + const pendingApprovals = await database.select({ issueId: issueApprovals.issueId }).from(issueApprovals) + .innerJoin(approvals, eq(approvals.id, issueApprovals.approvalId)).where(and( + eq(issueApprovals.companyId, companyId), eq(approvals.companyId, companyId), + inArray(issueApprovals.issueId, ids), eq(approvals.status, "pending"))); + const approved = new Set(pendingApprovals.map(row => row.issueId)); + const stalled = await issueService(database).listReviewAttention(companyId, rows); + for (const row of rows) { + const state = parseIssueExecutionState(row.executionState); + result.set(row.id, requiresReviewAttention({ + hasHumanParticipant: state?.status === "pending" && state.currentParticipant?.type === "user", + hasUserAssignee: Boolean(row.assigneeUserId), hasPendingApproval: approved.has(row.id), + stalled: stalled.get(row.id)?.state === "stalled", + })); + } + } + return result; + } + + async function eligible(database: Db, candidate: Candidate, variant?: "owner" | "terminal", lock = false, reviewAttention?: Map, interactionAttention?: Map): Promise { let sourceId = candidate.sourceId; let activityAt = candidate.activityAt; let dueAt = candidate.deadlineAt; @@ -83,38 +146,14 @@ export function attentionRetentionService(db: Db) { .where(and(eq(decisions.companyId, candidate.companyId), eq(decisions.id, sourceId), eq(decisions.status, "open"))); if (!row || row.originIssueId && !await visibleIssue(database, candidate.companyId, row.originIssueId)) return null; } else if (candidate.sourceKind === "issue_thread_interaction") { - const [row] = await database.select().from(issueThreadInteractions).where(and( - eq(issueThreadInteractions.companyId, candidate.companyId), eq(issueThreadInteractions.id, sourceId), - eq(issueThreadInteractions.status, "pending"), isNull(issueThreadInteractions.addresseeUserId))); - if (!row || !await visibleIssue(database, candidate.companyId, row.issueId, true)) return null; - const roster = row.addresseeAgentId ? await database.select().from(agents).where(eq(agents.companyId, candidate.companyId)) : []; - if (row.addresseeAgentId && evaluateAgentInvokability(roster.find(agent => agent.id === row.addresseeAgentId), roster).invokable) return null; - if (row.kind === "request_confirmation") { - const siblings = await database.select().from(issueThreadInteractions).where(and( - eq(issueThreadInteractions.companyId, candidate.companyId), eq(issueThreadInteractions.issueId, row.issueId), - eq(issueThreadInteractions.status, "pending"), eq(issueThreadInteractions.kind, "request_confirmation"), - isNull(issueThreadInteractions.addresseeUserId))); - const companyAgents = roster.length ? roster : await database.select().from(agents).where(eq(agents.companyId, candidate.companyId)); - const board = siblings.filter(sibling => !sibling.addresseeAgentId - || !evaluateAgentInvokability(companyAgents.find(agent => agent.id === sibling.addresseeAgentId), companyAgents).invokable); - if (!collapsePendingConfirmationsToNewest(board).some(sibling => sibling.id === sourceId)) return null; - } + const attention = interactionAttention ?? await interactionEligibility(database, [candidate]); + if (!attention.get(sourceId)) return null; } else if (candidate.sourceKind === "recovery_action") { const [row] = await database.select().from(issueRecoveryActions).where(and(eq(issueRecoveryActions.companyId, candidate.companyId), eq(issueRecoveryActions.id, sourceId))); if (!row || !await visibleIssue(database, candidate.companyId, row.sourceIssueId)) return null; } else if (candidate.sourceKind === "review") { - const row = await visibleIssue(database, candidate.companyId, sourceId); - if (!row) return null; - const state = parseIssueExecutionState(row.executionState); - const [approval] = await database.select({ id: approvals.id }).from(issueApprovals).innerJoin(approvals, eq(approvals.id, issueApprovals.approvalId)) - .where(and(eq(issueApprovals.companyId, candidate.companyId), eq(approvals.companyId, candidate.companyId), - eq(issueApprovals.issueId, sourceId), eq(approvals.status, "pending"))).limit(1); - if (!requiresReviewAttention({ hasHumanParticipant: state?.status === "pending" && state.currentParticipant?.type === "user", - hasUserAssignee: Boolean(row.assigneeUserId), hasPendingApproval: Boolean(approval), stalled: false })) { - const attention = await issueService(database).listReviewAttention(candidate.companyId, [row]); - if (!requiresReviewAttention({ hasHumanParticipant: false, hasUserAssignee: false, hasPendingApproval: false, - stalled: attention.get(sourceId)?.state === "stalled" })) return null; - } + const attention = reviewAttention ?? await reviewEligibility(database, [candidate]); + if (!attention.get(sourceId)) return null; } else if (candidate.sourceKind === "blocker_attention") { const row = await visibleIssue(database, candidate.companyId, sourceId); if (!row) return null; @@ -157,10 +196,12 @@ export function attentionRetentionService(db: Db) { const key = JSON.stringify(scope ?? {}); const rows = await candidates(db, scope, cursors.get(key)); const entries: Entry[] = []; + const reviews = await reviewEligibility(db, rows); + const interactions = await interactionEligibility(db, rows); for (const row of rows) { if (signal?.aborted) return { rows: null, complete: true }; for (const variant of row.sourceKind === "blocker_attention" ? ["owner", "terminal"] as const : [undefined]) { - const entry = await eligible(db, row, variant); + const entry = await eligible(db, row, variant, false, reviews, interactions); if (entry) entries.push(entry); } } diff --git a/server/src/services/attention-source-eligibility.ts b/server/src/services/attention-source-eligibility.ts index 6e3f484c03a..f129c522130 100644 --- a/server/src/services/attention-source-eligibility.ts +++ b/server/src/services/attention-source-eligibility.ts @@ -1,6 +1,10 @@ import { and, eq, gt, inArray, sql, type SQL } from "drizzle-orm"; import { heartbeatRuns, type Db } from "@paperclipai/db"; +export const HUMAN_RECOVERY_OWNER_TYPES = ["user", "board"] as const; +export const ATTENTION_BUDGET_WARNING_PERCENT = 85; +export const ATTENTION_AGENT_ERROR_STATUS = "error"; + export function collapsePendingConfirmationsToNewest = { const PENDING_INTERACTION_STATUSES = ["pending"] as const; const OPEN_RECOVERY_STATUSES = ["active", "escalated"] as const; -const HUMAN_RECOVERY_OWNER_TYPES = ["user", "board"] as const; const DETAIL_EXCERPT_LENGTH = 160; const DETAIL_IMAGE_LIMIT = 3; const OPEN_DECISION_DEFAULT_LIMIT = 500; @@ -1762,7 +1761,7 @@ export function attentionService(db: Db, serviceOptions: AttentionServiceOptions const budgetOverview = await budgetService(db).overview(companyId); for (const incident of budgetOverview.activeIncidents) { const observedPercent = budgetObservedPercent(incident.amountObserved, incident.amountLimit); - if (incident.thresholdType !== "hard" && observedPercent < 85) continue; + if (incident.thresholdType !== "hard" && observedPercent < ATTENTION_BUDGET_WARNING_PERCENT) continue; const dedupKey = `budget:${incident.policyId}:${toIso(incident.windowStart)}:${incident.thresholdType}`; add(createItem({ companyId, @@ -1825,7 +1824,7 @@ export function attentionService(db: Db, serviceOptions: AttentionServiceOptions updatedAt: agents.updatedAt, }) .from(agents) - .where(and(eq(agents.companyId, companyId), eq(agents.status, "error"))) + .where(and(eq(agents.companyId, companyId), eq(agents.status, ATTENTION_AGENT_ERROR_STATUS))) .orderBy(desc(agents.updatedAt), desc(agents.id)); for (const agent of erroredAgents) { diff --git a/server/src/services/budgets.ts b/server/src/services/budgets.ts index 93403926772..c9cedb7f7ee 100644 --- a/server/src/services/budgets.ts +++ b/server/src/services/budgets.ts @@ -505,8 +505,8 @@ export function budgetServiceInTransaction(db: Db, publications: ActivityPublica return incident ? { incident, created: true } : null; } - async function resolveOpenSoftIncidents(policyId: string) { - await notifyAttentionRetentionWork(db, { sourceKind: "budget_alert" }); + async function resolveOpenSoftIncidents(policyId: string, companyId: string) { + await notifyAttentionRetentionWork(db, { companyId, sourceKind: "budget_alert" }); await db .update(budgetIncidents) .set({ @@ -525,11 +525,12 @@ export function budgetServiceInTransaction(db: Db, publications: ActivityPublica async function resolveOpenIncidentsForPolicy( policyId: string, + companyId: string, approvalStatus: "approved" | "rejected" | null, decidedByUserId: string | null, thresholdType?: BudgetThresholdType, ) { - await notifyAttentionRetentionWork(db, { sourceKind: "budget_alert" }); + await notifyAttentionRetentionWork(db, { companyId, sourceKind: "budget_alert" }); const condition = and(eq(budgetIncidents.policyId, policyId), eq(budgetIncidents.status, "open"), thresholdType ? eq(budgetIncidents.thresholdType, thresholdType) : undefined); const openRows = await db @@ -618,9 +619,9 @@ export function budgetServiceInTransaction(db: Db, publications: ActivityPublica if (policy.isActive && policy.hardStopEnabled && policy.amount > 0 && observed.recoveringRunCount > 0) recoveringPolicy ??= policy; // A closed UTC window, inactive policy, or budget raise releases only // budget-owned pauses. Manual pauses remain operator decisions. - await resolveOpenIncidentsForPolicy(policy.id, "approved", "budget_service", "hard"); + await resolveOpenIncidentsForPolicy(policy.id, policy.companyId, "approved", "budget_service", "hard"); if (!policy.isActive || !policy.notifyEnabled || compareCents(observed.totalExact, Math.ceil(policy.amount * policy.warnPercent / 100)) < 0) { - await resolveOpenIncidentsForPolicy(policy.id, "approved", "budget_service", "soft"); + await resolveOpenIncidentsForPolicy(policy.id, policy.companyId, "approved", "budget_service", "soft"); } } // Agent status has its own admission gate and error code. Preserve a @@ -677,7 +678,7 @@ export function budgetServiceInTransaction(db: Db, publications: ActivityPublica } if (observedBlocks(policy, observed)) { - await resolveOpenSoftIncidents(policy.id); + await resolveOpenSoftIncidents(policy.id, policy.companyId); const hardIncident = await createIncidentIfNeeded(policy, "hard", observed); await pauseAndCancelScopeForBudget(policy); if (hardIncident?.created) { @@ -847,11 +848,11 @@ export function budgetServiceInTransaction(db: Db, publications: ActivityPublica const observed = await computeObservedSpend(db, row); if (!observedBlocks(row, observed)) { await resumeScopeFromBudget(row); - await resolveOpenIncidentsForPolicy(row.id, "approved", actorUserId ?? "budget_service", "hard"); + await resolveOpenIncidentsForPolicy(row.id, row.companyId, "approved", actorUserId ?? "budget_service", "hard"); if (row.notifyEnabled && compareCents(observed.totalExact, Math.ceil(row.amount * row.warnPercent / 100)) >= 0) { await createIncidentIfNeeded(row, "soft", observed); } else { - await resolveOpenIncidentsForPolicy(row.id, null, null, "soft"); + await resolveOpenIncidentsForPolicy(row.id, row.companyId, null, null, "soft"); } } else { const softThreshold = Math.ceil((row.amount * row.warnPercent) / 100); @@ -859,14 +860,14 @@ export function budgetServiceInTransaction(db: Db, publications: ActivityPublica await createIncidentIfNeeded(row, "soft", observed); } if (observedBlocks(row, observed)) { - await resolveOpenSoftIncidents(row.id); + await resolveOpenSoftIncidents(row.id, row.companyId); await createIncidentIfNeeded(row, "hard", observed); await pauseAndCancelScopeForBudget(row); } } } else { await resumeScopeFromBudget(row); - await resolveOpenIncidentsForPolicy(row.id, "approved", actorUserId ?? "budget_service"); + await resolveOpenIncidentsForPolicy(row.id, row.companyId, "approved", actorUserId ?? "budget_service"); } await recordActivity({ diff --git a/server/src/services/decision-expiry.ts b/server/src/services/decision-expiry.ts index ea5236a6ac0..796200128a7 100644 --- a/server/src/services/decision-expiry.ts +++ b/server/src/services/decision-expiry.ts @@ -23,7 +23,7 @@ export function decisionExpiryService(db: Db, deliverContinuation: (id: string) agentId: row.originAgentId, runId: row.originRunId, action: "decision.expired", entityType: "decision", entityId: row.id, details: { expiredReason: reason } }, publications); return row; - }); + }, { companyId: decision.companyId, sourceId: decision.id }); if (!updated) return 0; for (const publication of publications) publishActivity(publication); await deliverContinuation(updated.id); diff --git a/server/src/services/decision-work-notifications.ts b/server/src/services/decision-work-notifications.ts index 0abc8e95829..9e2ca840960 100644 --- a/server/src/services/decision-work-notifications.ts +++ b/server/src/services/decision-work-notifications.ts @@ -2,20 +2,22 @@ import { notifyAttentionRetentionWork } from "./retention-work-notifications.js" import type { Db } from "@paperclipai/db"; import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js"; -export async function notifyDecisionCreated(transaction: object) { - await notifyAttentionRetentionWork(transaction, { sourceKind: "decision" }); +type DecisionWorkScope = { companyId: string; sourceId?: string }; + +export async function notifyDecisionCreated(transaction: object, scope: DecisionWorkScope) { + await notifyAttentionRetentionWork(transaction, { ...scope, sourceKind: "decision" }); return notifyDeliveryWork(transaction, DELIVERY_QUEUES.decisionExpiry); } -export async function notifyDecisionChange(transaction: object) { - await notifyAttentionRetentionWork(transaction, { sourceKind: "decision" }); +export async function notifyDecisionChange(transaction: object, scope: DecisionWorkScope) { + await notifyAttentionRetentionWork(transaction, { ...scope, sourceKind: "decision" }); await notifyDeliveryWork(transaction, DELIVERY_QUEUES.decisionContinuation); await notifyDeliveryWork(transaction, DELIVERY_QUEUES.decisionExpiry); } -export function writeDecisionChange(db: Db, write: (transaction: Db) => Promise): Promise { +export function writeDecisionChange(db: Db, write: (transaction: Db) => Promise, scope: DecisionWorkScope): Promise { return db.transaction(async transaction => { - await notifyDecisionChange(transaction); + await notifyDecisionChange(transaction, scope); await notifyDeliveryWork(transaction, DELIVERY_QUEUES.decisionEffectRecovery); return write(transaction as unknown as Db); }); diff --git a/server/src/services/decisions.ts b/server/src/services/decisions.ts index 282327d433f..8a84743fcef 100644 --- a/server/src/services/decisions.ts +++ b/server/src/services/decisions.ts @@ -239,7 +239,7 @@ export function decisionService(db: Db, options: DecisionServiceOptions) { ...(input.additionalTargetSnapshots ?? {}), }; const id = randomUUID(); - await notifyDecisionCreated(dbOrTx); + await notifyDecisionCreated(dbOrTx, { companyId: input.companyId, sourceId: id }); await notifyDecisionTargetWork(dbOrTx, `decision:${id}`); const [created] = await dbOrTx.insert(decisions).values({ id, companyId: input.companyId, bundleId: input.bundleId ?? null, originAgentId: input.agentId, originIssueId: provenance.issueId, originRunId: input.runId, ruleKey: input.ruleKey ?? null, @@ -581,7 +581,8 @@ export function decisionService(db: Db, options: DecisionServiceOptions) { return attempt; } - async function settleInvalidEffectOutcome(id: string) { + async function settleInvalidEffectOutcome(decision: typeof decisions.$inferSelect) { + const id = decision.id; const publications: ActivityPublication[] = []; await writeDecisionChange(db, async tx => { const [current] = await tx.select().from(decisions).where(eq(decisions.id, id)).for("update"); @@ -601,14 +602,14 @@ export function decisionService(db: Db, options: DecisionServiceOptions) { action: "decision.recovery_failed", entityType: "decision", entityId: id, details: { reason, executionStatus, chosenOptionId: current.chosenOptionId, decidedByUserId: current.decidedByUserId }, }, publications); - }); + }, { companyId: decision.companyId, sourceId: id }); for (const publication of publications) publishActivity(publication); return outcome(id); } async function executeDecisionEffects(decision: typeof decisions.$inferSelect, userActor: AuthorizationActor) { const option = decision.options.find((item) => item.id === decision.chosenOptionId); - if (!option || !decision.decidedByUserId) return settleInvalidEffectOutcome(decision.id); + if (!option || !decision.decidedByUserId) return settleInvalidEffectOutcome(decision); const run = await db.select({ responsibleUserId: heartbeatRuns.responsibleUserId }).from(heartbeatRuns).where(eq(heartbeatRuns.id, decision.originRunId)).then((rows) => rows[0] ?? null); const metadata = decision.metadata as Record; if (metadata.kind === "attention_archive_proposal") { @@ -618,7 +619,7 @@ export function decisionService(db: Db, options: DecisionServiceOptions) { targetSnapshots: decision.targetSnapshots as Record, }), decision.signedSpec)) { await writeDecisionChange(db, async tx => tx.update(decisions).set({ executionStatus: "failed", updatedAt: new Date(), metadata: { ...metadata, archiveProposalError: "invalid_signature", ...(decision.continuationPolicy === "wake_origin_agent" ? { continuationPending: true } : {}) } }) - .where(and(eq(decisions.id, decision.id), eq(decisions.executionStatus, "running")))); + .where(and(eq(decisions.id, decision.id), eq(decisions.executionStatus, "running"))), { companyId: decision.companyId, sourceId: decision.id }); return outcome(decision.id); } if (option.id === "archive") { @@ -629,7 +630,7 @@ export function decisionService(db: Db, options: DecisionServiceOptions) { const manifestHash = manifest ? hashAttentionArchiveManifest(manifest) : null; if (!manifest || manifestHash !== metadata.manifestHash) { await writeDecisionChange(db, async tx => tx.update(decisions).set({ executionStatus: "failed", updatedAt: new Date(), metadata: { ...metadata, archiveProposalError: "manifest_mismatch", ...(decision.continuationPolicy === "wake_origin_agent" ? { continuationPending: true } : {}) } }) - .where(and(eq(decisions.id, decision.id), eq(decisions.executionStatus, "running")))); + .where(and(eq(decisions.id, decision.id), eq(decisions.executionStatus, "running"))), { companyId: decision.companyId, sourceId: decision.id }); return outcome(decision.id); } const originActor: AuthorizationActor = { @@ -651,7 +652,7 @@ export function decisionService(db: Db, options: DecisionServiceOptions) { } catch (error) { const message = error instanceof Error ? error.message : "archive_proposal_failed"; await writeDecisionChange(db, async tx => tx.update(decisions).set({ executionStatus: "failed", updatedAt: new Date(), metadata: { ...metadata, archiveProposalError: message, ...(decision.continuationPolicy === "wake_origin_agent" ? { continuationPending: true } : {}) } }) - .where(and(eq(decisions.id, decision.id), eq(decisions.executionStatus, "running")))); + .where(and(eq(decisions.id, decision.id), eq(decisions.executionStatus, "running"))), { companyId: decision.companyId, sourceId: decision.id }); return outcome(decision.id); } } @@ -659,7 +660,7 @@ export function decisionService(db: Db, options: DecisionServiceOptions) { executionStatus: "succeeded", updatedAt: new Date(), metadata: { ...metadata, ...(decision.continuationPolicy === "wake_origin_agent" ? { continuationPending: true } : {}) }, - }).where(and(eq(decisions.id, decision.id), eq(decisions.executionStatus, "running")))); + }).where(and(eq(decisions.id, decision.id), eq(decisions.executionStatus, "running"))), { companyId: decision.companyId, sourceId: decision.id }); return outcome(decision.id); } for (let index = 0; index < option.effects.length; index += 1) await executeEffect(decision, option.effects[index]!, index, userActor, decision.decidedByUserId, run?.responsibleUserId ?? null); @@ -667,7 +668,7 @@ export function decisionService(db: Db, options: DecisionServiceOptions) { const successful = rows.filter((row) => row.status === "executed").length; const status = rows.every((row) => row.status === "executed") ? "succeeded" : successful ? "partial" : "failed"; await writeDecisionChange(db, async tx => tx.update(decisions).set({ executionStatus: status, updatedAt: new Date(), metadata: { ...decision.metadata, - ...(decision.continuationPolicy === "wake_origin_agent" ? { continuationPending: true } : {}) } }).where(and(eq(decisions.id, decision.id), eq(decisions.executionStatus, "running")))); + ...(decision.continuationPolicy === "wake_origin_agent" ? { continuationPending: true } : {}) } }).where(and(eq(decisions.id, decision.id), eq(decisions.executionStatus, "running"))), { companyId: decision.companyId, sourceId: decision.id }); return outcome(decision.id); } @@ -706,7 +707,7 @@ export function decisionService(db: Db, options: DecisionServiceOptions) { decideIdempotencyKey: input.idempotencyKey ?? null, ...(current.continuationPolicy === "wake_origin_agent" ? { continuationPending: true } : {}), ...(input.dismissed ? { dismissed: true, dismissReason: input.dismissReason ?? null } : {}) } }) - .where(and(eq(decisions.id, current.id), eq(decisions.status, "open"))).returning()); + .where(and(eq(decisions.id, current.id), eq(decisions.status, "open"))).returning(), { companyId: current.companyId, sourceId: current.id }); if (!claimed) throw conflict("decision_already_resolved", { code: "decision_already_resolved" }); const run = await db.select({ responsibleUserId: heartbeatRuns.responsibleUserId }).from(heartbeatRuns).where(eq(heartbeatRuns.id, claimed.originRunId)).then((rows) => rows[0] ?? null); await logActivity(db, { companyId: claimed.companyId, actorType: "system", actorId: "decision-executor", agentId: claimed.originAgentId, runId: claimed.originRunId, @@ -724,7 +725,7 @@ export function decisionService(db: Db, options: DecisionServiceOptions) { return (await get(id))!; } const [updated] = await writeDecisionChange(db, async tx => tx.update(decisions).set({ status: "cancelled", updatedAt: new Date(), metadata: { ...current.metadata, - ...(current.continuationPolicy === "wake_origin_agent" ? { continuationPending: true } : {}) } }).where(and(eq(decisions.id, id), eq(decisions.status, "open"))).returning()); + ...(current.continuationPolicy === "wake_origin_agent" ? { continuationPending: true } : {}) } }).where(and(eq(decisions.id, id), eq(decisions.status, "open"))).returning(), { companyId: current.companyId, sourceId: id }); if (!updated) throw conflict("decision_already_resolved", { code: "decision_already_resolved" }); await logActivity(db, { companyId: updated.companyId, actorType: actor.actorType, actorId: actor.actorId, runId: actor.runId, action: "decision.cancelled", entityType: "decision", entityId: id }); await continuations.deliver(updated.id); @@ -738,7 +739,7 @@ export function decisionService(db: Db, options: DecisionServiceOptions) { if (empty) return decide({ id, optionId: empty.id, decidedByUserId: userId, userActor, dismissed: true, dismissReason: reason }); const [updated] = await writeDecisionChange(db, async tx => tx.update(decisions).set({ status: "decided", executionStatus: "succeeded", chosenOptionId: "dismissed", decidedByUserId: userId, decidedAt: new Date(), updatedAt: new Date(), metadata: { ...current.metadata, dismissed: true, dismissReason: reason ?? null, - ...(current.continuationPolicy === "wake_origin_agent" ? { continuationPending: true } : {}) } }).where(and(eq(decisions.id, id), eq(decisions.status, "open"))).returning()); + ...(current.continuationPolicy === "wake_origin_agent" ? { continuationPending: true } : {}) } }).where(and(eq(decisions.id, id), eq(decisions.status, "open"))).returning(), { companyId: current.companyId, sourceId: id }); if (!updated) throw conflict("decision_already_resolved", { code: "decision_already_resolved" }); await logActivity(db, { companyId: updated.companyId, actorType: "system", actorId: "decision-executor", agentId: updated.originAgentId, runId: updated.originRunId, responsibleUserIdOverride: userId, action: "decision.dismissed", entityType: "decision", entityId: updated.id, diff --git a/server/src/services/heartbeat-run-events.ts b/server/src/services/heartbeat-run-events.ts index 22219811668..ced43f5376f 100644 --- a/server/src/services/heartbeat-run-events.ts +++ b/server/src/services/heartbeat-run-events.ts @@ -10,6 +10,8 @@ import type { Db } from "@paperclipai/db"; import { heartbeatRunEvents, heartbeatRuns } from "@paperclipai/db"; import { nativeSha256 } from "./native-runtime/canonical.js"; +export const RETRY_EXHAUSTION_MESSAGE_PREFIX = "Bounded retry exhausted"; + export interface AppendHeartbeatRunEventInput { companyId: string; runId: string; @@ -75,7 +77,7 @@ export async function appendHeartbeatRunEvent( db: Db, input: AppendHeartbeatRunEventInput, ): Promise { - const projectsRetention = input.eventType === "lifecycle" && Boolean(input.message?.startsWith("Bounded retry exhausted")); + const projectsRetention = input.eventType === "lifecycle" && Boolean(input.message?.startsWith(RETRY_EXHAUSTION_MESSAGE_PREFIX)); const projectsReassessment = readNativeEventEvidenceOutcome(input.payload?.prpEvent) !== null; const projectsProgress = isSafeNativeChatProgressEventType(input.eventType); const projectsResponse = isNativeResponseEvidence(input.eventType, input.payload?.prpEvent); @@ -131,7 +133,7 @@ export async function appendHeartbeatRunEvent( eq(heartbeatRunEvents.runId, input.runId), eq(heartbeatRunEvents.agentId, input.agentId), eq(heartbeatRunEvents.eventType, "lifecycle"), - sql`${heartbeatRunEvents.message} like 'Bounded retry exhausted%'`, + sql`${heartbeatRunEvents.message} like ${`${RETRY_EXHAUSTION_MESSAGE_PREFIX}%`}`, sql`${heartbeatRunEvents.payload} @> ${JSON.stringify(input.retryExhaustion)}::jsonb`, )) .limit(1) diff --git a/server/src/services/issues.ts b/server/src/services/issues.ts index 4ee7e681907..7239075de99 100644 --- a/server/src/services/issues.ts +++ b/server/src/services/issues.ts @@ -7598,7 +7598,7 @@ export function issueService(db: Db) { companyId: string, blockedByIssueIds: string[], actor: { agentId?: string | null; userId?: string | null } = {}, - dbOrTx: any = db, + dbOrTx: any, ) { const deduped = [...new Set(blockedByIssueIds)]; if (deduped.some((candidate) => candidate === issueId)) { @@ -9676,27 +9676,32 @@ export function issueService(db: Db) { }); if (blockParentUntilDone) { - const existingBlockers = await db - .select({ blockerIssueId: issueRelations.issueId }) - .from(issueRelations) - .where( - and( - eq(issueRelations.companyId, parent.companyId), - eq(issueRelations.relatedIssueId, parent.id), - eq(issueRelations.type, "blocks"), - ), + await db.transaction(async tx => { + await tx.select({ id: issues.id }).from(issues).where(and( + eq(issues.companyId, parent.companyId), eq(issues.id, parent.id))).for("update"); + const existingBlockers = await tx + .select({ blockerIssueId: issueRelations.issueId }) + .from(issueRelations) + .where( + and( + eq(issueRelations.companyId, parent.companyId), + eq(issueRelations.relatedIssueId, parent.id), + eq(issueRelations.type, "blocks"), + ), + ); + await syncBlockedByIssueIds( + parent.id, + parent.companyId, + [ + ...new Set([ + ...existingBlockers.map((row) => row.blockerIssueId), + child.id, + ]), + ], + { agentId: actorAgentId ?? null, userId: actorUserId ?? null }, + tx, ); - await syncBlockedByIssueIds( - parent.id, - parent.companyId, - [ - ...new Set([ - ...existingBlockers.map((row) => row.blockerIssueId), - child.id, - ]), - ], - { agentId: actorAgentId ?? null, userId: actorUserId ?? null }, - ); + }); [child] = await withIssueRelationSummaries( parent.companyId, [child], From 30730dd61983ebcd1500170bef66df85c296871c Mon Sep 17 00:00:00 2001 From: Dotta Date: Fri, 30 Oct 2026 16:54:51 -0500 Subject: [PATCH 22/74] test(skills): verify Cloud CLI reads through shared object storage Use the real S3 provider with a controlled HTTP fixture and a stable test key. Preserve read-only/no-seeding behavior and verify corrupt, missing, and restored object outcomes under managed Cloud policy. Co-Authored-By: Paperclip --- .../__tests__/managed-skill-content.test.ts | 90 ++++++++++++++++--- 1 file changed, 77 insertions(+), 13 deletions(-) diff --git a/server/src/__tests__/managed-skill-content.test.ts b/server/src/__tests__/managed-skill-content.test.ts index cd4ddb8135f..59566d743e3 100644 --- a/server/src/__tests__/managed-skill-content.test.ts +++ b/server/src/__tests__/managed-skill-content.test.ts @@ -1,3 +1,5 @@ +import { createServer } from "node:http"; +import { createS3StorageProvider } from "../storage/s3-provider.js"; import { skillInstallationMaintenance } from "../services/skill-installation-maintenance.js"; import { removeRuntimeAssetTree } from "../services/native-runtime/runtime-asset-retention.js"; import { eq } from "drizzle-orm"; @@ -79,26 +81,88 @@ describe.skipIf(!support.supported)("managed skills use committed versions", () it("loads persisted Cloud configuration for read-only maintenance without seeding it", async () => { const companyId = await company(); - await companySkillService(db).createLocalSkill(companyId, { name: "Cloud CLI", markdown: "# Cloud CLI" }); + const objects = new Map(); + const server = createServer((request, response) => { + void (async () => { + const key = new URL(request.url!, "http://fixture").pathname; + if (request.method === "PUT") { + const chunks: Buffer[] = []; + for await (const chunk of request) chunks.push(Buffer.from(chunk)); + let body = Buffer.concat(chunks); + if (request.headers["content-encoding"]?.includes("aws-chunked")) { + const decoded: Buffer[] = []; + let offset = 0; + while (offset < body.length) { + const end = body.indexOf("\r\n", offset); + const size = Number.parseInt(body.subarray(offset, end).toString().split(";")[0]!, 16); + if (size === 0) break; + decoded.push(body.subarray(end + 2, end + 2 + size)); offset = end + 2 + size + 2; + } + body = Buffer.concat(decoded); + } + objects.set(key, body); response.writeHead(200, { etag: '"fixture"' }).end(); return; + } + const body = objects.get(key); + if (!body) { response.writeHead(404).end(); return; } + response.writeHead(200, { "content-length": body.length, "content-type": "application/octet-stream", etag: '"fixture"' }); + response.end(request.method === "HEAD" ? undefined : body); + })().catch(() => { response.writeHead(500).end(); }); + }); + await new Promise(resolve => { server.listen(0, "127.0.0.1", resolve); }); + const address = server.address(); if (!address || typeof address === "string") throw new Error("Expected fixture address"); + const endpoint = `http://127.0.0.1:${address.port}`; + const credentials = { AWS_ACCESS_KEY_ID: "fixture-access-key", AWS_SECRET_ACCESS_KEY: "fixture-secret-key", AWS_SESSION_TOKEN: "" }; + const previousCredentials = Object.fromEntries(Object.keys(credentials).map(key => [key, process.env[key]])); + Object.assign(process.env, credentials); + vi.mocked(storageModule.getStorageService).mockReturnValue(createStorageService(createS3StorageProvider({ + bucket: "managed-skill-fixture", region: "us-east-1", endpoint, forcePathStyle: true, + }))); const key = "cloud-runtime-config:v1"; - const run = (phase: string) => promisify(execFile)(process.execPath, [ - "--import", fileURLToPath(import.meta.resolve("tsx")), - fileURLToPath(new URL("../../scripts/migrate-skill-packages.ts", import.meta.url)), - `--phase=${phase}`, `--company=${companyId}`, "--limit=50", - ], { env: { ...process.env, DATABASE_URL: database.connectionString, - PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN: "disposable-cloud-cli-fixture", - PAPERCLIP_STORAGE_PROVIDER: "local_disk", PAPERCLIP_STORAGE_LOCAL_DIR: path.join(root, "objects") }, - timeout: 60_000, maxBuffer: 1024 * 1024 }); - await expect(run("eligibility")).rejects.toThrow("Initialize Cloud runtime configuration"); - expect(await db.select().from(instanceSettings).where(eq(instanceSettings.singletonKey, key))).toHaveLength(0); - await db.insert(instanceSettings).values({ singletonKey: key, general: { version: 1, revision: 0, config: null } }); try { + await companySkillService(db).createLocalSkill(companyId, { name: "Cloud CLI", markdown: "# Cloud CLI" }); + const run = (phase: string) => promisify(execFile)(process.execPath, [ + "--import", fileURLToPath(import.meta.resolve("tsx")), + fileURLToPath(new URL("../../scripts/migrate-skill-packages.ts", import.meta.url)), + `--phase=${phase}`, `--company=${companyId}`, "--limit=50", + ], { env: { ...process.env, ...credentials, DATABASE_URL: database.connectionString, + PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN: "disposable-cloud-cli-fixture", + PAPERCLIP_SECRETS_MASTER_KEY: "a".repeat(64), PAPERCLIP_STORAGE_PROVIDER: "s3", + PAPERCLIP_STORAGE_S3_BUCKET: "managed-skill-fixture", PAPERCLIP_STORAGE_S3_REGION: "us-east-1", + PAPERCLIP_STORAGE_S3_ENDPOINT: endpoint, PAPERCLIP_STORAGE_S3_FORCE_PATH_STYLE: "true", PAPERCLIP_STORAGE_S3_PREFIX: "" }, + timeout: 60_000, maxBuffer: 1024 * 1024 }); + await expect(run("eligibility")).rejects.toThrow("Initialize Cloud runtime configuration"); + expect(await db.select().from(instanceSettings).where(eq(instanceSettings.singletonKey, key))).toHaveLength(0); + await db.insert(instanceSettings).values({ singletonKey: key, general: { version: 1, revision: 0, config: null } }); const eligibility = JSON.parse((await run("eligibility")).stdout.trim().split("\n").at(-1)!); expect(eligibility.results).toEqual([expect.objectContaining({ status: "version_backed" })]); + const before = [...objects.entries()].map(([key, body]) => [key, Buffer.from(body)] as const); const verification = JSON.parse((await run("verification")).stdout.trim().split("\n").at(-1)!); - expect(verification.results).toEqual([expect.objectContaining({ status: "local_storage" })]); + expect(verification.results).toEqual([expect.objectContaining({ status: "remote_storage" })]); + expect([...objects.entries()]).toEqual(before); + const archiveKey = [...objects.keys()].find(key => key.endsWith("/files.tar.gz"))!; + const archive = objects.get(archiveKey)!; + const corrupted = Buffer.from(archive); corrupted[0] = corrupted[0]! ^ 1; + objects.set(archiveKey, corrupted); + const corrupt = JSON.parse((await run("verification")).stdout.trim().split("\n").at(-1)!); + expect(corrupt.results).toEqual([expect.objectContaining({ status: "corrupt_package" })]); + objects.set(archiveKey, archive); + const manifestKey = [...objects.keys()].find(key => key.endsWith("/manifest.json"))!; + const manifest = objects.get(manifestKey)!; objects.delete(manifestKey); + const unavailable = JSON.parse((await run("verification")).stdout.trim().split("\n").at(-1)!); + expect(unavailable.results).toEqual([expect.objectContaining({ status: "unavailable_or_corrupt_package" })]); + objects.set(manifestKey, manifest); + const recovered = JSON.parse((await run("verification")).stdout.trim().split("\n").at(-1)!); + expect(recovered.results).toEqual([expect.objectContaining({ status: "remote_storage" })]); + expect([...objects.entries()].sort()).toEqual([...before].sort()); + expect(await db.select().from(instanceSettings).where(eq(instanceSettings.singletonKey, key))) + .toEqual([expect.objectContaining({ general: { version: 1, revision: 0, config: null } })]); } finally { await db.delete(instanceSettings).where(eq(instanceSettings.singletonKey, key)); + vi.mocked(storageModule.getStorageService).mockReturnValue(createStorageService(createLocalDiskStorageProvider(path.join(root, "objects")))); + for (const [key, value] of Object.entries(previousCredentials)) { + if (value === undefined) delete process.env[key]; else process.env[key] = value; + } + server.closeAllConnections(); await new Promise(resolve => { server.close(() => resolve()); }); } }, 180_000); From 2fb56dba9ee09b5a299db2014a6bcb7530338339 Mon Sep 17 00:00:00 2001 From: Dotta Date: Fri, 30 Oct 2026 17:16:47 -0500 Subject: [PATCH 23/74] fix(github): serialize binding delivery with company admission Bound binding calls and wait for each in-flight batch before releasing lifecycle locks. Keep archived companies from dispatching saved binding changes. Verify company isolation and the startup history reconciliation contract. Co-Authored-By: Paperclip --- .../__tests__/github-connection-work.test.ts | 15 +++++++++-- .../tool-access/github-grant-continuity.ts | 26 ++++++++++++------- 2 files changed, 30 insertions(+), 11 deletions(-) diff --git a/server/src/__tests__/github-connection-work.test.ts b/server/src/__tests__/github-connection-work.test.ts index 4e789eedb2b..ceeb033d5ad 100644 --- a/server/src/__tests__/github-connection-work.test.ts +++ b/server/src/__tests__/github-connection-work.test.ts @@ -99,9 +99,10 @@ describeDatabase("GitHub coordinator with PostgreSQL and controlled Cloud HTTP ( const ew = registerGitHubConnectionEventWork(db, coordinator, events, () => true); const cw = registerGitHubConnectionContinuityWork(db, coordinator, f.service, () => true); try { - await Promise.all([ew.ready, cw.ready]); expect(leases).toBe(0); + await Promise.all([ew.ready, cw.ready]); expect(leases).toBe(1); + const startupLeases = leases; await f.service.refreshManagedGitHubGrantAccess(f.connection, f.grant, { actorType: "system", actorId: "test" }); - await vi.waitFor(() => expect(leases).toBeGreaterThan(0)); + await vi.waitFor(() => expect(leases).toBeGreaterThan(startupLeases)); await vi.advanceTimersByTimeAsync(31_000); await settled(coordinator); const [saved] = await db.select().from(connectionGrants).where(eq(connectionGrants.id, f.grant.id)); expect(saved!.providerTenant!.github!.installationIds).toEqual(["101"]); @@ -142,6 +143,16 @@ describeDatabase("GitHub coordinator with PostgreSQL and controlled Cloud HTTP ( expect(await restarted.nextGitHubConnectionContinuityAt()).toBeNull(); }); + it("settles only the company and grant that own saved binding debt", async () => { + const f = await fixture(["101"]), other = await fixture(["202"]); + await db.update(connectionGrants).set({ providerTenant: { ...f.grant.providerTenant, github: { ...f.grant.providerTenant!.github!, + bindingSync: { id: randomUUID(), removeInstallationIds: [] } } } }).where(eq(connectionGrants.id, f.grant.id)); + await f.service.sweepGitHubConnectionContinuity(); + expect(bindings).toHaveLength(1); expect(bindings[0]).toMatchObject({ grantId: f.grant.id, installationId: "101" }); + const [untouched] = await db.select().from(connectionGrants).where(eq(connectionGrants.id, other.grant.id)); + expect(untouched!.providerTenant).toEqual(other.grant.providerTenant); + }); + it("retains a real deadline when successful renewal leaves a short-lived credential unchanged", async () => { const f = await fixture(["101"]); fakeClock(); vi.spyOn(globalThis, "fetch").mockImplementation((url, init) => String(url).startsWith("https://api.github.com") diff --git a/server/src/services/tool-access/github-grant-continuity.ts b/server/src/services/tool-access/github-grant-continuity.ts index 0398c2a79be..1096d5f08e6 100644 --- a/server/src/services/tool-access/github-grant-continuity.ts +++ b/server/src/services/tool-access/github-grant-continuity.ts @@ -6,7 +6,7 @@ import { measureRunPhase } from "../execution-preparation-timing.js"; import { measureRunTransaction } from "../execution-transaction-timing.js"; import { and, eq, sql } from "drizzle-orm"; import type { Db } from "@paperclipai/db"; -import { connectionGrants, toolConnections } from "@paperclipai/db"; +import { companies, connectionGrants, toolConnections } from "@paperclipai/db"; import { conflict, HttpError, notFound, unprocessable } from "../../errors.js"; import { PaperclipCloudConnectorError, type PaperclipCloudConnector } from "../paperclip-cloud-connector.js"; @@ -235,19 +235,27 @@ export function createGitHubGrantContinuity(db: Db, { now, currentCloudConnector eq(toolConnections.id, connection.id), eq(toolConnections.companyId, connection.companyId), )).for("update").limit(1); if (!liveConnection || liveConnection.status !== "active" || !liveConnection.enabled) return current; + const [company] = await tx.select({ status: companies.status }).from(companies) + .where(eq(companies.id, connection.companyId)).for("share").limit(1); + if (company?.status !== "active") return current; const github = current.providerTenant?.github; if (!github?.bindingSync || github.bindingSync.id !== grant.providerTenant?.github?.bindingSync?.id) return current; const subject = current.kind === "agent" && current.subjectAgentId ? `agent:${current.subjectAgentId}` : current.kind === "user" ? current.subjectUserId : null; if (!subject) throw unprocessable("GitHub webhook subject is unavailable"); - for (const installationId of github.installationIds) await cloudConnector.setWebhookBinding({ - subject, companyId: connection.companyId, id: `${current.id}_${installationId}`, - installationId, connectionId: connection.id, grantId: current.id, active: true, accessToken: token!, - }); - for (const installationId of github.bindingSync.removeInstallationIds) await cloudConnector.setWebhookBinding({ - subject, companyId: connection.companyId, id: `${current.id}_${installationId}`, - installationId, connectionId: connection.id, grantId: current.id, active: false, - }); + const changes = [ + ...github.installationIds.map(installationId => ({ installationId, active: true as const })), + ...github.bindingSync.removeInstallationIds.map(installationId => ({ installationId, active: false as const })), + ]; + for (let offset = 0; offset < changes.length; offset += 4) { + const outcomes = await Promise.allSettled(changes.slice(offset, offset + 4).map(change => cloudConnector.setWebhookBinding({ + subject, companyId: connection.companyId, id: `${current.id}_${change.installationId}`, + installationId: change.installationId, connectionId: connection.id, grantId: current.id, + ...change.active ? { active: true, accessToken: token! } : { active: false }, + }))); + const failure = outcomes.find(outcome => outcome.status === "rejected"); + if (failure?.status === "rejected") throw failure.reason; + } await notifyGitHubCredentialChange(tx, connection); const [settled] = await tx.update(connectionGrants).set({ providerTenant: sql`${connectionGrants.providerTenant} #- '{github,bindingSync}' #- '{github,continuityRetry}'`, From b45b6a225d8a9983ace61cca426f0c0c949e2ab7 Mon Sep 17 00:00:00 2001 From: Dotta Date: Fri, 30 Oct 2026 17:18:38 -0500 Subject: [PATCH 24/74] fix(external-objects): bound refresh lifecycle and activation scope Reuse the provider-owned request cancellation contract from merge-confirmation work. Keep host signals out of plugin RPC payloads, preserve refresh debt on shutdown, and target identities or known objects for source writes. Co-Authored-By: Paperclip --- .../2026-10-30-external-objects-sleep.md | 26 +++++- .../external-object-refresh-work.test.ts | 79 +++++++++++++++++-- .../server-startup-feedback-export.test.ts | 22 +++++- .../external-object-refresh-notifications.ts | 6 +- server/src/services/external-objects.ts | 60 ++++++++++---- .../src/services/github-connection-events.ts | 3 +- .../github-external-object-provider.ts | 18 +++-- 7 files changed, 179 insertions(+), 35 deletions(-) diff --git a/doc/plans/2026-10-30-external-objects-sleep.md b/doc/plans/2026-10-30-external-objects-sleep.md index ab566999917..61c7d8dded1 100644 --- a/doc/plans/2026-10-30-external-objects-sleep.md +++ b/doc/plans/2026-10-30-external-objects-sleep.md @@ -24,7 +24,9 @@ probe before their terminal verdict is definitive. A null deadline retains its existing never-refresh meaning. Registration does not change stored timestamps. The worker reconstructs eligible deadlines in pages of at most 250 rows. A -company notification reads that company only. A due callback reads only its +company notification reads that company only. Ordinary ingestion uses object IDs +or stable provider/object/external identity hints, including concurrent upserts, +so one mention write does not reconstruct all company history. A due callback reads only its object. Pages select scheduling columns and retain compact deadlines in memory. Dispatch uses fair batches of at most 50 objects. The deadline is the later of `next_refresh_at` and the current refresh lease expiry. Lease renewals and atomic @@ -46,7 +48,7 @@ and fences automatic result persistence. It does not modify that draft PR. | Mutation | Owner and activation | | --- | --- | -| Issue title/description, comment and document references | Existing sync methods register company intent before mention replacement and object upsert. Routes, runtime tools, adapters and plugin paths reuse these methods. | +| Issue title/description, comment and document references | Existing sync methods register object intent before mention replacement and stable identity intent before object upsert. Routes, runtime tools, adapters and plugin paths reuse these methods. | | Comment/document disappearance | Sync derives the retained mention owners and registers only those companies before deleting obsolete mentions. | | Last mention removal and reattachment | Mention replacement invalidates company deadlines. Retained objects stay available for reattachment. | | Issue deletion | Issue service registers its company before cascading mention deletion. | @@ -90,3 +92,23 @@ The conservative external-object host-sleep blocker remains. An in-process deadline cannot wake stopped compute. This change proves worker quiet and reactivation in the tested process; it does not establish Cloud alarm coverage, host/database suspension or deployed savings. + + +## Host cancellation and explicit legacy entry points + +The resolver contract accepts an optional host `AbortSignal`. The worker passes +its coordinator signal through refresh to the resolver. This reuses the narrow +provider-owned capability introduced with merge-confirmation work: GitHub combines +host cancellation with a ten-second request budget for real fetch, response-body +reads and visibility probes. Plugin RPC payloads remain unchanged. The host +observes cancellation while retaining handlers for any eventual plugin/custom +result; it releases only its own claim and cannot publish that late result. +Startup does not await provider work. Shutdown closes the active GitHub HTTP +request, and its saved due row can resume immediately after restart. + +The two explicit `refreshDueObjects*` helpers remain manual diagnostic/test entry +points. They have no production scheduler caller. Their regression fixtures also +preserve the prior selection/lease semantics. The scheduled check retains its +post-refresh scoped read to revalidate current company, reference and resolver +eligibility and recover an authoritative lease after a competing writer; all such +reads target one object rather than rebuilding a company. diff --git a/server/src/__tests__/external-object-refresh-work.test.ts b/server/src/__tests__/external-object-refresh-work.test.ts index f429c31396b..754725c7d80 100644 --- a/server/src/__tests__/external-object-refresh-work.test.ts +++ b/server/src/__tests__/external-object-refresh-work.test.ts @@ -1,8 +1,9 @@ +import { createServer } from "node:http"; import { randomUUID } from "node:crypto"; import { setTimeout as delay } from "node:timers/promises"; import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest"; import { eq, sql } from "drizzle-orm"; -import { activityLog, approvals, applyPendingMigrations, budgetIncidents, budgetPolicies, companies, costEvents, createDb, externalObjects, issues, issueComments, plugins } from "@paperclipai/db"; +import { activityLog, approvals, applyPendingMigrations, databaseWorkPending, budgetIncidents, budgetPolicies, companies, costEvents, createDb, externalObjects, externalObjectMentions, issues, issueComments, plugins } from "@paperclipai/db"; import { externalObjectService, type ExternalObjectResolveResult } from "../services/external-objects.js"; import type { PaperclipPluginManifestV1 } from "@paperclipai/shared"; import type { PluginWorkerManager } from "../services/plugin-worker-manager.js"; @@ -12,6 +13,7 @@ import { pluginRegistryService } from "../services/plugin-registry.js"; import { companyService } from "../services/companies.js"; import { instanceSettingsService } from "../services/instance-settings.js"; import { notifyExternalObjectRefresh } from "../services/external-object-refresh-notifications.js"; +import { DELIVERY_QUEUES } from "../services/delivery-work-notifications.js"; import { createDeliveryWorkCoordinator } from "../services/delivery-work-coordinator.js"; import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js"; import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; @@ -295,8 +297,12 @@ const active: ExternalObjectResolveResult = { ok: true, snapshot: { key: "notes", format: "markdown", body: `https://example.com/${a.companyId}` }); id = result.document.id; await svc.syncDocument(id); } - await settle(async () => (await row(a.companyId))?.lastResolvedAt !== null && (await row(b.companyId))?.lastResolvedAt !== null); - await settle(() => coordinator!.nextWakeAt() !== null); + await settle(async () => { + const left = await row(a.companyId), right = await row(b.companyId); + return Boolean(left?.lastResolvedAt && right?.lastResolvedAt); + }); + const expectedDeadline = Math.min((await row(a.companyId)).nextRefreshAt!.getTime(), (await row(b.companyId)).nextRefreshAt!.getTime()); + await settle(() => coordinator!.nextWakeAt() === expectedDeadline && !databaseWorkPending(db, DELIVERY_QUEUES.externalObjectRefresh)); const reads: Array<{ query: string; params: unknown[] }> = []; const debug = db.$client.options.debug; db.$client.options.debug = (_connection, query, params) => { @@ -344,7 +350,7 @@ const active: ExternalObjectResolveResult = { ok: true, snapshot: { finish(terminal); await settle(async () => (await row(f.companyId)).isTerminal); await quiet(); }); - it("uses durable provider retry deadlines and waits for in-flight shutdown without overlap", async () => { + it("uses durable provider retry deadlines and cancels the observed result on shutdown without overlap", async () => { clock(); const svc = service(); await start(svc); provider.mockResolvedValueOnce({ ok: false, liveness: "rate_limited", errorCode: "slow_down", retryAfterSeconds: 5 }); const f = await fixture(); await svc.syncIssue(f.issueId); @@ -360,13 +366,74 @@ const active: ExternalObjectResolveResult = { ok: true, snapshot: { await svc.syncIssue(f.issueId); await vi.advanceTimersByTimeAsync(0); expect(provider).toHaveBeenCalledTimes(2); let stopped = false; const stop = coordinator!.stop().then(() => { stopped = true; }); - await delay(10); expect(stopped).toBe(false); - finish(terminal); await stop; coordinator = undefined; + await stop; expect(stopped).toBe(true); coordinator = undefined; + const cancelled = await row(f.companyId); expect(cancelled.refreshToken).toBeNull(); expect(cancelled.isTerminal).toBe(false); + finish(terminal); await delay(10); + expect((await row(f.companyId)).isTerminal).toBe(false); const selects = vi.spyOn(db, "select"); await vi.advanceTimersByTimeAsync(600_000); expect(selects).not.toHaveBeenCalled(); expect(provider).toHaveBeenCalledTimes(2); }); + it("cancels an actual stalled GitHub HTTP body, retains due debt and resumes after restart", async () => { + clock(); let stall = true, started = false, closed = false; + const server = createServer((request, response) => { + if (request.url?.includes("/pulls/")) { + if (stall) { + started = true; response.on("close", () => { closed = true; }); + response.writeHead(200, { "content-type": "application/json" }); response.write('{"state":'); + } else response.end(JSON.stringify({ state: "closed", merged: true, title: "Done" })); + } else response.end(JSON.stringify({ private: false })); + }); + await new Promise(resolve => server.listen(0, "127.0.0.1", resolve)); + const address = server.address(); if (!address || typeof address === "string") throw new Error("Missing HTTP test port"); + const svc = externalObjectService(db, { github: { + fetch: (url, init) => fetch(`http://127.0.0.1:${address.port}${new URL(url).pathname}`, init), + credentialProvider: () => ({ token: "simulated-company-token", source: "company_secret" }), + } }); + try { + await start(svc); const f = await fixture(); + await db.update(issues).set({ description: "https://github.com/acme/app/pull/42" }).where(eq(issues.id, f.issueId)); + await svc.syncIssue(f.issueId); await settle(() => started); + await coordinator!.stop(); coordinator = undefined; + await settle(() => closed); + const cancelled = await row(f.companyId); + expect(cancelled.refreshToken).toBeNull(); expect(cancelled.lastResolvedAt).toBeNull(); + expect(cancelled.nextRefreshAt!.getTime()).toBeLessThanOrEqual(Date.now()); + stall = false; await start(svc); + await settle(async () => Boolean((await row(f.companyId))?.isTerminal)); await quiet(); + } finally { await coordinator?.stop(); coordinator = undefined; server.closeAllConnections(); await new Promise(resolve => server.close(() => resolve())); } + }); + + it("loads only the changed identity and objects after ingestion into ten thousand future objects", async () => { + clock(); const svc = service(); const f = await fixture(); + const future = new Date(Date.now() + 86_400_000); + const ids = Array.from({ length: 10_000 }, () => randomUUID()); + for (let offset = 0; offset < ids.length; offset += 500) { + const batch = ids.slice(offset, offset + 500); + await db.insert(externalObjects).values(batch.map(id => ({ id, companyId: f.companyId, + providerKey: "url", objectType: "link", externalId: id, nextRefreshAt: future }))); + await db.insert(externalObjectMentions).values(batch.map(objectId => ({ companyId: f.companyId, + sourceIssueId: f.issueId, sourceKind: "description" as const, objectId }))); + } + await start(svc); await settle(() => coordinator!.nextWakeAt() === future.getTime()); + const [incoming] = await db.insert(issues).values({ companyId: f.companyId, title: "Incoming", + description: "https://example.com/new-object" }).returning(); + const reads: Array<{ query: string; params: unknown[] }> = []; + const debug = db.$client.options.debug; + db.$client.options.debug = (_connection, query, params) => { + if (/^select/i.test(query) && query.includes('from "external_objects"') && query.includes('inner join "companies"')) + reads.push({ query, params: [...params] }); + }; + try { + await svc.syncIssue(incoming!.id); + await settle(() => provider.mock.calls.length === 1); + await settle(() => coordinator!.nextWakeAt() === future.getTime() && !databaseWorkPending(db, DELIVERY_QUEUES.externalObjectRefresh)); + expect(reads.length).toBeGreaterThan(0); expect(reads.length).toBeLessThanOrEqual(6); + for (const read of reads) expect(read.query).toMatch(/"external_objects"\."(?:id|external_id)" =/); + } finally { db.$client.options.debug = debug; } + }); + it("reconstructs paginated multi-company work and never invokes inactive or never-refresh objects", async () => { clock(); const svc = service(); const a = await fixture(); const b = await fixture(); const inactive = await fixture("paused"); for (const companyId of [b.companyId, inactive.companyId]) { diff --git a/server/src/__tests__/server-startup-feedback-export.test.ts b/server/src/__tests__/server-startup-feedback-export.test.ts index bbb020c865d..5002dc44a50 100644 --- a/server/src/__tests__/server-startup-feedback-export.test.ts +++ b/server/src/__tests__/server-startup-feedback-export.test.ts @@ -207,6 +207,8 @@ const { }; const executionWorkspaceServiceFactoryMock = vi.fn(() => executionWorkspaceServiceMock); const externalObjectsServiceMock = { + registerRefreshWork: vi.fn((coordinator: { register: typeof deliveryWorkRegisterMock }, _canRun: () => boolean) => + coordinator.register("external-object-refresh", { run: async () => undefined, hasPending: async () => false })), refreshDueObjectsForActiveCompanies: vi.fn(async () => ({ companies: 0, checked: 0, refreshed: 0 })), }; const externalObjectsServiceFactoryMock = vi.fn(() => externalObjectsServiceMock); @@ -1399,7 +1401,10 @@ describe("startServer feedback export wiring", () => { await Promise.resolve(); expect(heartbeatServiceMock.tickAgentTimers).toHaveBeenCalledTimes(1); - expect(externalObjectsServiceMock.refreshDueObjectsForActiveCompanies).toHaveBeenCalledTimes(1); + expect(externalObjectsServiceMock.registerRefreshWork).toHaveBeenCalledTimes(1); + expect(externalObjectsServiceMock.registerRefreshWork).toHaveBeenCalledWith(expect.any(Object), expect.any(Function)); + expect(deliveryWorkRegisterMock.mock.calls.some(([queue]) => queue === DELIVERY_QUEUES.externalObjectRefresh)).toBe(true); + expect(externalObjectsServiceMock.refreshDueObjectsForActiveCompanies).not.toHaveBeenCalled(); expect(issueThreadInteractionServiceMock.sweepMergedPullRequestConfirmations).toHaveBeenCalledTimes(1); expect(executionWorkspaceServiceMock.sweepTerminalWorkspaces).toHaveBeenCalledTimes(1); expect(routineServiceMock.tickScheduledTriggers).toHaveBeenCalledTimes(1); @@ -1422,6 +1427,16 @@ describe("startServer feedback export wiring", () => { } }); + it.each([true, false])("does not wait for external-object provider work before readiness (scheduler=%s)", async enabled => { + loadConfigMock.mockReturnValue(buildTestConfig({ heartbeatSchedulerEnabled: enabled })); + externalObjectsServiceMock.registerRefreshWork.mockReturnValueOnce({ + ready: new Promise(() => {}), wake: vi.fn(), stop: vi.fn(async () => {}), + }); + await startServer(); + expect(externalObjectsServiceMock.registerRefreshWork).toHaveBeenCalledOnce(); + expect(externalObjectsServiceMock.refreshDueObjectsForActiveCompanies).not.toHaveBeenCalled(); + }); + it("keeps external object refresh active when heartbeat scheduling is disabled", async () => { loadConfigMock.mockReturnValue(buildTestConfig({ heartbeatSchedulerEnabled: false, @@ -1449,7 +1464,10 @@ describe("startServer feedback export wiring", () => { await Promise.resolve(); await Promise.resolve(); - expect(externalObjectsServiceMock.refreshDueObjectsForActiveCompanies).toHaveBeenCalledTimes(1); + expect(externalObjectsServiceMock.registerRefreshWork).toHaveBeenCalledTimes(1); + expect(externalObjectsServiceMock.registerRefreshWork).toHaveBeenCalledWith(expect.any(Object), expect.any(Function)); + expect(deliveryWorkRegisterMock.mock.calls.some(([queue]) => queue === DELIVERY_QUEUES.externalObjectRefresh)).toBe(true); + expect(externalObjectsServiceMock.refreshDueObjectsForActiveCompanies).not.toHaveBeenCalled(); expect(routineServiceMock.tickScheduledTriggers).not.toHaveBeenCalled(); expect(heartbeatServiceMock.reconcileCostAccounting).toHaveBeenCalledTimes(1); expect(environmentCustomImagesServiceMock.cleanupExpiredSetupSessions).not.toHaveBeenCalled(); diff --git a/server/src/services/external-object-refresh-notifications.ts b/server/src/services/external-object-refresh-notifications.ts index 95833421a8a..271d172e463 100644 --- a/server/src/services/external-object-refresh-notifications.ts +++ b/server/src/services/external-object-refresh-notifications.ts @@ -1,7 +1,9 @@ import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js"; -export async function notifyExternalObjectRefresh(tx: object, companyId?: string, objectId?: string) { +export type ExternalObjectRefreshIdentity = { providerKey: string; objectType: string; externalId: string; ignoreExternalIdCase?: boolean }; + +export async function notifyExternalObjectRefresh(tx: object, companyId?: string, object?: string | ExternalObjectRefreshIdentity) { await notifyDeliveryWork(tx, DELIVERY_QUEUES.externalObjectRefresh, - companyId ? JSON.stringify({ companyId, ...(objectId ? { objectId } : {}) }) : "*"); + companyId ? JSON.stringify({ companyId, ...(typeof object === "string" ? { objectId: object } : object ? { identity: object } : {}) }) : "*"); } diff --git a/server/src/services/external-objects.ts b/server/src/services/external-objects.ts index d524b473e1a..0208cdab7b6 100644 --- a/server/src/services/external-objects.ts +++ b/server/src/services/external-objects.ts @@ -18,7 +18,7 @@ import { notFound } from "../errors.js"; import { logger } from "../middleware/logger.js"; import { logActivity, type LogActivityInput } from "./activity-log.js"; import { createGitHubExternalObjectProvider, type GitHubExternalObjectProviderOptions } from "./github-external-object-provider.js"; -import { notifyExternalObjectRefresh } from "./external-object-refresh-notifications.js"; +import { notifyExternalObjectRefresh, type ExternalObjectRefreshIdentity } from "./external-object-refresh-notifications.js"; import { DELIVERY_QUEUES } from "./delivery-work-notifications.js"; import type { createDeliveryWorkCoordinator } from "./delivery-work-coordinator.js"; import { registerScopedDeadlineWork } from "./scoped-deadline-work.js"; @@ -88,6 +88,7 @@ export interface ExternalObjectResolver { resolve(input: { companyId: string; object: ExternalObjectRecord; + signal?: AbortSignal; }): Promise; } @@ -99,8 +100,8 @@ const DEFAULT_RETRY_AFTER_SECONDS = 300; const DEFAULT_REFRESH_LEASE_SECONDS = 300; const REFRESH_LEASE_RENEW_INTERVAL_MS = 60_000; -type RefreshScope = { companyId: string; objectId?: string }; -type RefreshDeadline = { id: string; companyId: string; dueAt: number }; +type RefreshScope = { companyId: string; objectId?: string; identity?: ExternalObjectRefreshIdentity }; +type RefreshDeadline = { id: string; companyId: string; dueAt: number; providerKey: string; objectType: string; externalId: string }; function refreshableObjectWhere() { @@ -471,7 +472,7 @@ export function externalObjectService( nextRefreshAt: now, updatedAt: now, }; - await notifyExternalObjectRefresh(dbOrTx, companyId); + await notifyExternalObjectRefresh(dbOrTx, companyId, { providerKey: detection.providerKey, objectType: detection.objectType, externalId: detection.externalId }); const inserted = await dbOrTx .insert(externalObjects) .values(values) @@ -501,7 +502,9 @@ export function externalObjectService( dbOrTx: any = db, ) { const urls = extractExternalObjectCanonicalUrls(input.text ?? ""); - await notifyExternalObjectRefresh(dbOrTx, input.companyId); + const previous: Array<{ objectId: string | null }> = await dbOrTx.selectDistinct({ objectId: externalObjectMentions.objectId }) + .from(externalObjectMentions).where(sourceWhere(input)); + for (const row of previous) if (row.objectId) await notifyExternalObjectRefresh(dbOrTx, input.companyId, row.objectId); await dbOrTx.delete(externalObjectMentions).where(sourceWhere(input)); if (urls.length === 0) return; @@ -520,6 +523,7 @@ export function externalObjectService( if (seen.has(sourceKey)) continue; seen.add(sourceKey); const object = await upsertObjectFromDetection(input.companyId, detection, dbOrTx); + await notifyExternalObjectRefresh(dbOrTx, input.companyId, object.id); values.push({ companyId: input.companyId, sourceIssueId: input.sourceIssueId, @@ -573,9 +577,9 @@ export function externalObjectService( async function deleteMissingSourceMentions(kind: "comment" | "document", recordId: string, tx: any) { const source = and(eq(externalObjectMentions.sourceKind, kind), eq(externalObjectMentions.sourceRecordId, recordId)); - const owners: Array<{ companyId: string }> = await tx.selectDistinct({ companyId: externalObjectMentions.companyId }) + const owners: Array<{ companyId: string; objectId: string | null }> = await tx.selectDistinct({ companyId: externalObjectMentions.companyId, objectId: externalObjectMentions.objectId }) .from(externalObjectMentions).where(source); - for (const owner of owners) await notifyExternalObjectRefresh(tx, owner.companyId); + for (const owner of owners) if (owner.objectId) await notifyExternalObjectRefresh(tx, owner.companyId, owner.objectId); await tx.delete(externalObjectMentions).where(source); } @@ -836,6 +840,7 @@ export function externalObjectService( actor?: Pick; force?: boolean; scheduled?: boolean; + signal?: AbortSignal; now?: Date; }; @@ -875,7 +880,8 @@ export function externalObjectService( now: Date, refreshToken: string, ) { - const pluginResult = await resolveViaPluginProvider(db, opts.pluginWorkerManager, object); + input.signal?.throwIfAborted(); + const pluginResult = await observeRefreshCancellation(resolveViaPluginProvider(db, opts.pluginWorkerManager, object), input.signal); const resolver = pluginResult ? null : resolverRegistry.find(object); if (!pluginResult && !resolver) { const [updated] = await persistRefresh(object, refreshToken, { @@ -889,7 +895,9 @@ export function externalObjectService( return { object: toObjectPayload(updated ?? object, now), refreshed: false, reason: "no_resolver" as const }; } - const result = pluginResult ?? await resolver!.resolve({ companyId: object.companyId, object }); + const result = pluginResult ?? await observeRefreshCancellation( + resolver!.resolve({ companyId: object.companyId, object, signal: input.signal }), input.signal); + input.signal?.throwIfAborted(); if (!result.ok) { // A failed refresh means the object's status is currently unknown, not // terminal. Clear a previous terminal verdict so the object keeps being @@ -1027,6 +1035,16 @@ export function externalObjectService( }; } + function observeRefreshCancellation(work: Promise, signal?: AbortSignal): Promise { + if (!signal) return work; + return new Promise((resolve, reject) => { + const abort = () => reject(signal.reason); + work.then(value => { signal.removeEventListener("abort", abort); resolve(value); }, + error => { signal.removeEventListener("abort", abort); reject(error); }); + if (signal.aborted) abort(); else signal.addEventListener("abort", abort, { once: true }); + }); + } + const objectRefreshesInFlight = new Map>>>(); async function refreshObject( @@ -1046,7 +1064,7 @@ export function externalObjectService( const refreshKey = `${object.companyId}:${object.id}`; const existingRefresh = objectRefreshesInFlight.get(refreshKey); - if (existingRefresh) return existingRefresh; + if (existingRefresh) return observeRefreshCancellation(existingRefresh, input.signal); const claimed = await claimObjectRefresh(object, input, now); if (!claimed) { @@ -1063,7 +1081,10 @@ export function externalObjectService( } const stopRenewingRefreshLease = startRefreshLeaseRenewal(object, claimed.refreshToken); - const refresh = resolveObjectRefresh(object, input, now, claimed.refreshToken).finally(() => { + const refresh = resolveObjectRefresh(object, input, now, claimed.refreshToken).catch(async error => { + if (input.signal?.aborted) await persistRefresh(object, claimed.refreshToken!, { refreshStartedAt: null, refreshToken: null }); + throw error; + }).finally(() => { stopRenewingRefreshLease(); objectRefreshesInFlight.delete(refreshKey); }); @@ -1149,12 +1170,17 @@ export function externalObjectService( const key = scopeKey(scope); const cursor = cursors.get(key); const rows = await db.select({ id: externalObjects.id, companyId: externalObjects.companyId, - nextRefreshAt: externalObjects.nextRefreshAt, refreshStartedAt: externalObjects.refreshStartedAt }).from(externalObjects) + nextRefreshAt: externalObjects.nextRefreshAt, refreshStartedAt: externalObjects.refreshStartedAt, + providerKey: externalObjects.providerKey, objectType: externalObjects.objectType, externalId: externalObjects.externalId }).from(externalObjects) .innerJoin(companies, eq(companies.id, externalObjects.companyId)) .where(and( eq(companies.status, "active"), automaticRefreshWhere(), scope ? eq(externalObjects.companyId, scope.companyId) : undefined, scope?.objectId ? eq(externalObjects.id, scope.objectId) : undefined, + scope?.identity ? and(eq(externalObjects.providerKey, scope.identity.providerKey), + eq(externalObjects.objectType, scope.identity.objectType), scope.identity.ignoreExternalIdCase + ? sql`lower(${externalObjects.externalId}) = lower(${scope.identity.externalId})` + : eq(externalObjects.externalId, scope.identity.externalId)) : undefined, cursor ? gt(externalObjects.id, cursor) : undefined, )).orderBy(asc(externalObjects.id)).limit(250); const entries = rows.map(deadline); @@ -1162,8 +1188,8 @@ export function externalObjectService( else cursors.delete(key); return { rows: entries, complete: rows.length < 250 }; } - function deadline(object: Pick): RefreshDeadline { - return { id: object.id, companyId: object.companyId, + function deadline(object: Pick): RefreshDeadline { + return { id: object.id, companyId: object.companyId, providerKey: object.providerKey, objectType: object.objectType, externalId: object.externalId, dueAt: Math.max(object.nextRefreshAt!.getTime(), object.refreshStartedAt ? object.refreshStartedAt.getTime() + DEFAULT_REFRESH_LEASE_SECONDS * 1000 : 0) }; } @@ -1173,7 +1199,10 @@ export function externalObjectService( minimumPassIntervalMs: 0, decodeScope: hint => hint === "*" ? undefined : JSON.parse(hint) as RefreshScope, matches: (entry, scope) => !scope || (entry.companyId === scope.companyId - && (!scope.objectId || entry.id === scope.objectId)), + && (!scope.objectId || entry.id === scope.objectId) + && (!scope.identity || (entry.providerKey === scope.identity.providerKey && entry.objectType === scope.identity.objectType + && (scope.identity.ignoreExternalIdCase ? entry.externalId.toLowerCase() === scope.identity.externalId.toLowerCase() + : entry.externalId === scope.identity.externalId)))), load, check: async (entry, signal) => { if (signal?.aborted || !canRun()) return entry; @@ -1183,6 +1212,7 @@ export function externalObjectService( await refreshObject(entry.id, { companyId: entry.companyId, scheduled: true, + signal, actor: { actorType: "system", actorId: "external-object-resolver", agentId: null, runId: null }, }); const updated = await load({ companyId: entry.companyId, objectId: entry.id }); diff --git a/server/src/services/github-connection-events.ts b/server/src/services/github-connection-events.ts index 76755ce30c3..643bc557b93 100644 --- a/server/src/services/github-connection-events.ts +++ b/server/src/services/github-connection-events.ts @@ -246,7 +246,8 @@ export function githubConnectionEventService( if (!snapshot) return; const appliedAt = now(); await db.transaction(async tx => { - await notifyExternalObjectRefresh(tx, companyId); + await notifyExternalObjectRefresh(tx, companyId, { providerKey: "github", objectType: "pull_request", + externalId: snapshot.externalId, ignoreExternalIdCase: true }); await tx.update(externalObjects).set({ statusKey: snapshot.statusKey, statusLabel: snapshot.statusLabel, diff --git a/server/src/services/github-external-object-provider.ts b/server/src/services/github-external-object-provider.ts index 6dedac97efb..83b47354035 100644 --- a/server/src/services/github-external-object-provider.ts +++ b/server/src/services/github-external-object-provider.ts @@ -15,6 +15,7 @@ type FetchLike = (url: string, init?: RequestInit) => Promise; export interface GitHubExternalObjectProviderOptions { fetch?: FetchLike; + requestTimeoutMs?: number; tokenProvider?: (companyId: string) => Promise | string | null; /** * Resolve the credential and its source. Prefer this over `tokenProvider` when the @@ -243,12 +244,12 @@ type RepoProbeOutcome = { async function probeRepoVisibility( identity: GitHubObjectIdentity, - opts: { fetch: FetchLike; headers?: Record | undefined }, + opts: { fetch: FetchLike; headers?: Record | undefined; signal?: AbortSignal }, ): Promise { try { const probe = await opts.fetch( `${gitHubApiBase(identity.host)}/repos/${encodeURIComponent(identity.owner)}/${encodeURIComponent(identity.repo)}`, - { headers: opts.headers }, + { headers: opts.headers, signal: opts.signal }, ); if (probe.ok) { const body = await safeJson(probe); @@ -427,12 +428,13 @@ export function createGitHubExternalObjectProvider( companyId: string, identity: GitHubObjectIdentity, headers: Record, + signal: AbortSignal, ): Promise { const credentialFingerprint = createHash("sha256").update(headers.authorization ?? "anonymous").digest("hex"); const key = `${companyId}\u0000${credentialFingerprint}\u0000${identity.owner.toLowerCase()}\u0000${identity.repo.toLowerCase()}`; const cached = probeCache.get(key); if (cached && cached.expiresAt > Date.now()) return cached.outcome; - const outcome = await probeRepoVisibility(identity, { fetch: fetchImpl, headers }); + const outcome = await probeRepoVisibility(identity, { fetch: fetchImpl, headers, signal }); if (outcome.visibility === "visible" || outcome.visibility === "hidden") { probeCache.set(key, { outcome, expiresAt: Date.now() + GITHUB_OBJECT_TTL_SECONDS * 1000 }); } @@ -464,7 +466,9 @@ export function createGitHubExternalObjectProvider( return { providerKey: "github", objectType, - async resolve({ companyId, object }) { + async resolve({ companyId, object, signal: ownerSignal }) { + const deadline = AbortSignal.timeout(opts.requestTimeoutMs ?? 10_000); + const signal = ownerSignal ? AbortSignal.any([ownerSignal, deadline]) : deadline; const identity = parseGitHubObject(object); if (!identity || identity.objectType !== objectType) { return { @@ -509,7 +513,7 @@ export function createGitHubExternalObjectProvider( // "public" verdict must not authorize a later read of its private contents. // The cache is still used below to classify a 404, where a stale verdict can // only make the object look deleted — it can never expose private data. - const probe = await probeRepoVisibility(identity, { fetch: fetchImpl, headers }); + const probe = await probeRepoVisibility(identity, { fetch: fetchImpl, headers, signal }); if (probe.visibility === "hidden") { return { ok: false, @@ -554,7 +558,7 @@ export function createGitHubExternalObjectProvider( let response: Response; try { - response = await fetchImpl(url, { headers }); + response = await fetchImpl(url, { headers, signal }); } catch { return { ok: false, @@ -573,7 +577,7 @@ export function createGitHubExternalObjectProvider( // Probe the repository metadata to classify: if the repo itself is visible // the object truly is gone; if it is not, surface credentials-needed // instead of permanently freezing a reference as "Not found". - const probe = await cachedProbeRepoVisibility(companyId, identity, headers); + const probe = await cachedProbeRepoVisibility(companyId, identity, headers, signal); if (probe.visibility === "visible") { return { ok: true, snapshot: notFoundSnapshot(identity, etag) }; } From 15fe6a7e42438db174aa5f4d80592619819e1d96 Mon Sep 17 00:00:00 2001 From: Dotta Date: Fri, 30 Oct 2026 17:19:42 -0500 Subject: [PATCH 25/74] test(github): qualify polling handoff and isolated company fixtures Co-Authored-By: Paperclip --- server/src/__tests__/github-connection-work.test.ts | 2 +- .../src/services/github-connection-events.test.ts | 13 +++++++++++-- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/server/src/__tests__/github-connection-work.test.ts b/server/src/__tests__/github-connection-work.test.ts index ceeb033d5ad..41ba2724079 100644 --- a/server/src/__tests__/github-connection-work.test.ts +++ b/server/src/__tests__/github-connection-work.test.ts @@ -62,7 +62,7 @@ describeDatabase("GitHub coordinator with PostgreSQL and controlled Cloud HTTP ( async function fixture(ids: string[] = []) { bindings = []; leases = 0; failBindings = false; githubRequests = 0; const companyId = randomUUID(); companyIds.push(companyId); - const [company] = await db.insert(companies).values({ id: companyId, name: "GitHub work test", issuePrefix: "GWT" }).returning(); + const [company] = await db.insert(companies).values({ id: companyId, name: "GitHub work test", issuePrefix: `GW${companyId.replaceAll("-", "").slice(0, 6).toUpperCase()}` }).returning(); const [app] = await db.insert(toolApplications).values({ companyId, applicationKey: randomUUID(), name: "GitHub", type: "mcp_server", status: "active" }).returning(); const [connection] = await db.insert(toolConnections).values({ companyId, applicationId: app!.id, uid: randomUUID(), name: "GitHub", status: "active", enabled: true, transport: "mcp_remote", authKind: "oauth", credentialPolicy: "per_user", config: { sourceTemplateKey: "github", oauth: { connectorProfile: "github.code" } }, transportConfig: {} }).returning(); diff --git a/server/src/services/github-connection-events.test.ts b/server/src/services/github-connection-events.test.ts index 2a7a15dbffc..35ebfd07d5a 100644 --- a/server/src/services/github-connection-events.test.ts +++ b/server/src/services/github-connection-events.test.ts @@ -3,7 +3,7 @@ import { chatEndpoints, type Db } from "@paperclipai/db"; import { githubConnectionEventService } from "./github-connection-events.js"; import type { PaperclipCloudConnector } from "./paperclip-cloud-connector.js"; -function fixture(push = false) { +function fixture(push = false, forcePolling = false) { let time = Date.parse("2026-09-04T12:00:00Z"); const bindings = [{ grant: { id: "grant", companyId: "company", kind: "user", subjectUserId: "owner", status: "active", @@ -19,7 +19,7 @@ function fixture(push = false) { : { innerJoin: () => ({ where: () => result(bindings) }) } })); const leaseEvents = vi.fn().mockResolvedValue(null); const connector = { leaseEvents, eventWakeVersion: () => push ? 1 : null } as unknown as PaperclipCloudConnector; - const service = githubConnectionEventService({ select } as unknown as Db, { connector, now: () => new Date(time) }); + const service = githubConnectionEventService({ select } as unknown as Db, { connector, env: forcePolling ? { PAPERCLIP_GITHUB_EVENT_POLLING: "1" } : {}, now: () => new Date(time) }); return { service, leaseEvents, select, bots, bindings, advance: (ms: number) => { time += ms; } }; } @@ -90,6 +90,15 @@ describe("GitHub external delivery activation", () => { await f.service.pollOnce({ forDelivery: true }); expect(f.leaseEvents).toHaveBeenCalledTimes(2); expect(f.service.nextPollAt()).toBeNull(); }); + it("arms forced fallback before a failed Cloud enrollment revocation", async () => { + const f = fixture(true, true); + f.leaseEvents.mockRejectedValueOnce(new Error("handoff unavailable")); + await expect(f.service.pollOnce()).rejects.toThrow("handoff unavailable"); + expect(f.service.nextPollAt()).not.toBeNull(); + expect(f.leaseEvents.mock.calls[0]![0].eventWakeVersion).toBe(0); + f.advance(5_000); await f.service.pollOnce(); + expect(f.leaseEvents).toHaveBeenCalledTimes(2); expect(f.service.nextPollAt()).not.toBeNull(); + }); it("retains old-Cloud fallback and settles once all bindings are disabled", async () => { const f = fixture(); await f.service.pollOnce(); expect(f.service.nextPollAt()).not.toBeNull(); f.bindings.length = 0; f.advance(10_000); await f.service.pollOnce(); From bd19f69e957040a57f8d0d5bcf366eaaf9d037ce Mon Sep 17 00:00:00 2001 From: Dotta Date: Fri, 30 Oct 2026 17:20:05 -0500 Subject: [PATCH 26/74] test(github): distinguish failed removal attempts from settled replay Co-Authored-By: Paperclip --- server/src/__tests__/github-connection-work.test.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/server/src/__tests__/github-connection-work.test.ts b/server/src/__tests__/github-connection-work.test.ts index 41ba2724079..1ab9882772a 100644 --- a/server/src/__tests__/github-connection-work.test.ts +++ b/server/src/__tests__/github-connection-work.test.ts @@ -125,6 +125,7 @@ describeDatabase("GitHub coordinator with PostgreSQL and controlled Cloud HTTP ( const [failed] = await db.select().from(connectionGrants).where(eq(connectionGrants.id, f.grant.id)); expect(failed!.providerTenant!.github!.bindingSync!.removeInstallationIds).toEqual(["202"]); const retryAt = Date.parse(failed!.providerTenant!.github!.continuityRetry!.at); + const attemptedRemovals = bindings.filter(binding => binding.installationId === "202").length; const restarted = createGitHubGrantContinuity(db, f.dependencies); expect(await restarted.nextGitHubConnectionContinuityAt()).toBe(retryAt); await vi.advanceTimersByTimeAsync(29_000); @@ -133,7 +134,7 @@ describeDatabase("GitHub coordinator with PostgreSQL and controlled Cloud HTTP ( await Promise.all([restarted.sweepGitHubConnectionContinuity(), restarted.sweepGitHubConnectionContinuity()]); const [saved] = await db.select().from(connectionGrants).where(eq(connectionGrants.id, f.grant.id)); expect(saved!.providerTenant!.github!.bindingSync).toBeUndefined(); expect(saved!.providerTenant!.github!.continuityRetry).toBeUndefined(); - expect(bindings.filter(binding => binding.installationId === "202")).toHaveLength(1); + expect(bindings.filter(binding => binding.installationId === "202")).toHaveLength(attemptedRemovals + 1); expect(githubRequests).toBe(0); expect(await restarted.nextGitHubConnectionContinuityAt()).toBeGreaterThan(Date.now()); await db.update(toolConnections).set({ enabled: false }).where(eq(toolConnections.id, f.connection.id)); From bb34039d0b698c762e0f19ef47ec3f4f3c080f64 Mon Sep 17 00:00:00 2001 From: Dotta Date: Fri, 30 Oct 2026 17:20:52 -0500 Subject: [PATCH 27/74] test(github): run worker persistence acceptance on supported CI PostgreSQL Use the existing embedded PostgreSQL harness when an external disposable database is not supplied. Local acceptance retains its exclusive external database and never retries exhausted host bootstrap. Co-Authored-By: Paperclip --- .../src/__tests__/github-connection-work.test.ts | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/server/src/__tests__/github-connection-work.test.ts b/server/src/__tests__/github-connection-work.test.ts index 1ab9882772a..f92cefd5ac1 100644 --- a/server/src/__tests__/github-connection-work.test.ts +++ b/server/src/__tests__/github-connection-work.test.ts @@ -1,9 +1,10 @@ +import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; import { randomUUID, generateKeyPairSync, verify, createHash } from "node:crypto"; import { createServer, type Server } from "node:http"; import { once } from "node:events"; import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest"; import { eq } from "drizzle-orm"; -import { applyPendingMigrations, companies, connectionGrants, createDb, toolApplications, toolConnections } from "@paperclipai/db"; +import { applyPendingMigrations, companies, connectionGrants, createDb, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS, toolApplications, toolConnections } from "@paperclipai/db"; import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js"; import { createGitHubGrantContinuity, githubGrantContinuityDeadline } from "../services/tool-access/github-grant-continuity.js"; import { createPaperclipCloudConnector } from "../services/paperclip-cloud-connector.js"; @@ -14,18 +15,21 @@ import { notifyGitHubCredentialChange } from "../services/github-connection-noti import { idleWorkSnapshot } from "../services/task-admission.js"; const databaseUrl = resolveExternalTestDatabaseUrl("PAPERCLIP_GITHUB_WORK_TEST_DATABASE_URL"); -const describeDatabase = databaseUrl ? describe : describe.skip; +const embeddedSupport = databaseUrl ? { supported: true } : await getEmbeddedPostgresTestSupport(); +const describeDatabase = embeddedSupport.supported ? describe : describe.skip; describeDatabase("GitHub coordinator with PostgreSQL and controlled Cloud HTTP (simulation)", () => { let db: ReturnType; + let temporaryDatabase: Awaited> | undefined; let server: Server, origin: string; let bindings: Array>, leases: number, failBindings: boolean, githubRequests: number; const signing = generateKeyPairSync("ed25519"), sealing = generateKeyPairSync("x25519"); const realFetch = globalThis.fetch; let companyIds: string[] = []; beforeAll(async () => { - await applyPendingMigrations(databaseUrl!); - db = createDb(databaseUrl!, { maxConnections: 4 }); + if (databaseUrl) await applyPendingMigrations(databaseUrl); + else temporaryDatabase = await startEmbeddedPostgresTestDatabase("paperclip-github-work-"); + db = createDb(databaseUrl ?? temporaryDatabase!.connectionString, { maxConnections: 4 }); server = createServer(async (req, res) => { const chunks: Buffer[] = []; for await (const chunk of req) chunks.push(Buffer.from(chunk)); const raw = Buffer.concat(chunks).toString(); @@ -52,12 +56,12 @@ describeDatabase("GitHub coordinator with PostgreSQL and controlled Cloud HTTP ( server.listen(0, "127.0.0.1"); await once(server, "listening"); const address = server.address(); if (!address || typeof address === "string") throw new Error("HTTP fixture did not start"); origin = `http://127.0.0.1:${address.port}`; - }); + }, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS); afterEach(async () => { vi.restoreAllMocks(); vi.useRealTimers(); for (const id of companyIds) await db.delete(companies).where(eq(companies.id, id)); companyIds = []; }); - afterAll(async () => { server.close(); await once(server, "close"); await db.$client.end({ timeout: 1 }); }); + afterAll(async () => { server.close(); await once(server, "close"); await db.$client.end({ timeout: 1 }); await temporaryDatabase?.cleanup(); }); async function fixture(ids: string[] = []) { bindings = []; leases = 0; failBindings = false; githubRequests = 0; From dbd400ce17d85b88c4b51ff9860e779dc169555b Mon Sep 17 00:00:00 2001 From: Dotta Date: Fri, 30 Oct 2026 17:24:55 -0500 Subject: [PATCH 28/74] fix: back off expired deadlines during domain drain Keep a future in-memory resume wake when domain dispatch is paused independently of coordinator admission. Preserve the saved obligation without rescheduling its expired deadline into an immediate skipped-attempt loop. Co-Authored-By: Paperclip --- server/src/__tests__/deadline-work.test.ts | 47 ++++++++++++++++++++++ server/src/services/deadline-work.ts | 2 +- 2 files changed, 48 insertions(+), 1 deletion(-) diff --git a/server/src/__tests__/deadline-work.test.ts b/server/src/__tests__/deadline-work.test.ts index 7657a7e9d99..c00daeb2f00 100644 --- a/server/src/__tests__/deadline-work.test.ts +++ b/server/src/__tests__/deadline-work.test.ts @@ -122,3 +122,50 @@ it("does not release an ambiguous database intent while dispatch is drained", as draining = false; await vi.advanceTimersByTimeAsync(30_000); expect(run).toHaveBeenCalledTimes(2); expect(coordinator.nextWakeAt()).toBeNull(); }); + +it("backs off an expired deadline when only the domain is drained and resumes promptly", async () => { + vi.useFakeTimers(); + let paused = false; + const owner = installDatabaseWorkSignals({ execute: vi.fn(async () => []) }); + const run = vi.fn(async () => {}); + const domainCanRun = vi.fn(() => !paused); + const deadline = Date.now() + 1_000; + const next = vi.fn(async () => run.mock.calls.length === 1 ? deadline : null); + coordinator = createDeliveryWorkCoordinator({ owner, canRun: () => true, onError: vi.fn() }); + const worker = registerDeadlineWork(coordinator, DELIVERY_QUEUES.agentTimers, { + run, next, canRun: domainCanRun, retryMs: 30_000, + }); + await worker.ready; + paused = true; + const before = idleWorkSnapshot(); + await vi.advanceTimersByTimeAsync(1_000); + expect(coordinator.nextWakeAt()).toBe(Date.now() + 30_000); + const attempts = domainCanRun.mock.calls.length; + await vi.advanceTimersByTimeAsync(90_000); + expect(domainCanRun.mock.calls.length - attempts).toBeLessThanOrEqual(12); + // Four attempts (expiry plus three retries), each begins and ends tracked work. + expect(idleWorkSnapshot().generation - before.generation).toBe(8); + expect(idleWorkSnapshot().active).toBe(before.active); + expect(run).toHaveBeenCalledOnce(); expect(next).toHaveBeenCalledOnce(); expect(owner.execute).not.toHaveBeenCalled(); + paused = false; + await vi.advanceTimersByTimeAsync(30_000); + expect(run).toHaveBeenCalledTimes(2); expect(coordinator.nextWakeAt()).toBeNull(); +}); + +it("retains a prompt paused wake when a new request preempts a far-future deadline", async () => { + vi.useFakeTimers(); + let paused = false; + const owner = installDatabaseWorkSignals({ execute: vi.fn(async () => []) }); + const run = vi.fn(async () => {}); + const deadline = Date.now() + 86_400_000; + coordinator = createDeliveryWorkCoordinator({ owner, canRun: () => true, onError: vi.fn() }); + const worker = registerDeadlineWork(coordinator, DELIVERY_QUEUES.agentTimers, { + run, next: async () => run.mock.calls.length === 1 ? deadline : null, canRun: () => !paused, retryMs: 30_000, + }); + await worker.ready; paused = true; worker.wake(); + await vi.advanceTimersByTimeAsync(0); + expect(coordinator.nextWakeAt()).toBe(Date.now() + 30_000); + expect(owner.execute).not.toHaveBeenCalled(); + paused = false; await vi.advanceTimersByTimeAsync(30_000); + expect(run).toHaveBeenCalledTimes(2); expect(coordinator.nextWakeAt()).toBeNull(); +}); diff --git a/server/src/services/deadline-work.ts b/server/src/services/deadline-work.ts index 15ba76abdae..208b208e051 100644 --- a/server/src/services/deadline-work.ts +++ b/server/src/services/deadline-work.ts @@ -52,7 +52,7 @@ export function registerDeadlineWork( // the domain to probe its database while dispatch is closed. hasPending: async () => task.canRun() && requested !== reconciled, nextRunAt: () => !task.canRun() && requested !== reconciled - ? Math.min(next ?? Infinity, Date.now() + task.retryMs) + ? Date.now() + task.retryMs : next, }); return { ...worker, wake: () => { requested++; worker.wake(); } }; From 4af412eaea9e780e2d867bd930e77865a1c4672b Mon Sep 17 00:00:00 2001 From: Dotta Date: Fri, 30 Oct 2026 17:25:18 -0500 Subject: [PATCH 29/74] refactor: drive status-card evaluation from committed work Move generation sequencing into a feature-owned deadline worker and retain persisted evaluation, eligibility and relative-search obligations. Add transactional producer hints and initial PostgreSQL acceptance coverage; follow-up completes the producer audit and remaining lifecycle gates. Co-Authored-By: Paperclip --- doc/plans/2026-10-30-status-cards-sleep.md | 33 ++++ .../status-card-evaluation-work.test.ts | 180 ++++++++++++++++++ .../status-card-update-engine.test.ts | 23 +++ server/src/__tests__/status-cards.test.ts | 9 +- server/src/index.ts | 35 +--- .../active-run-watchdog/adapters/postgres.ts | 2 + server/src/services/access.ts | 2 + server/src/services/agent-deletion.ts | 2 + server/src/services/companies.ts | 3 + server/src/services/company-deletion.ts | 2 + server/src/services/company-search.ts | 10 +- server/src/services/company-skills.ts | 2 + .../services/delivery-work-notifications.ts | 1 + .../execution-control-notifications.ts | 3 + server/src/services/execution-workspaces.ts | 4 + server/src/services/heartbeat/retries.ts | 2 + server/src/services/instance-settings.ts | 2 + server/src/services/issue-comment-writes.ts | 2 + server/src/services/issue-prefix.ts | 2 + .../issue-review-state-notifications.ts | 2 + server/src/services/issue-title.ts | 2 + server/src/services/issue-tree-control.ts | 2 + server/src/services/issues.ts | 6 + .../native-finalization-reconciler.ts | 2 + .../native-review-participant.ts | 2 + .../native-workspace-export-recovery.ts | 2 + .../native-workspace-export-retry.ts | 2 + .../native-workspace-reconstruction.ts | 2 + .../paperclip-runner-tool-authority.ts | 2 + server/src/services/plugin-workspaces.ts | 2 + .../src/services/provider-fresh-workspace.ts | 2 + server/src/services/routines.ts | 2 + .../services/slack-conversation-lifecycle.ts | 2 + .../src/services/slack-conversation-state.ts | 2 + server/src/services/status-card-evaluation.ts | 90 +++++++++ .../src/services/status-card-finalization.ts | 2 + .../src/services/status-card-notifications.ts | 10 + .../src/services/status-card-update-engine.ts | 40 +++- server/src/services/status-cards.ts | 159 ++++++++++------ 39 files changed, 561 insertions(+), 93 deletions(-) create mode 100644 doc/plans/2026-10-30-status-cards-sleep.md create mode 100644 server/src/__tests__/status-card-evaluation-work.test.ts create mode 100644 server/src/services/status-card-evaluation.ts create mode 100644 server/src/services/status-card-notifications.ts diff --git a/doc/plans/2026-10-30-status-cards-sleep.md b/doc/plans/2026-10-30-status-cards-sleep.md new file mode 100644 index 00000000000..dcd6c5ca226 --- /dev/null +++ b/doc/plans/2026-10-30-status-cards-sleep.md @@ -0,0 +1,33 @@ +# Status-card evaluation deadlines + +Baseline: `76fde891ccc29283b3f5c8dcdce089c5b3d937d1`. + +Status-card ownership covers evaluation policy, card claims, generation, ordinary assignment wakeups, and failed-assignment cancellation. `index.ts` registers a typed capability. Shared delivery/deadline infrastructure owns transaction settlement, coalescing, admission/drain, retries and shutdown. Existing cards, updates and issues remain durable truth; no timer table or sleep-safety exemption is introduced. + +Manual cards have no automatic obligation. Interval cards retain their requested evaluation cadence. Reactive cards evaluate committed source changes, retain the actual debounce/rate/budget/active-hours deadline while relevant changes remain, and disarm once unchanged. Saved relative-search windows retain their actual membership expiry; the duration interpretation stays with company search. Startup reconstructs existing work without advancing business timestamps. + +Scheduled reactive source events can preempt a future evaluation. A feature-owned transaction advisory lock serializes scheduler evaluation across connections while existing card/issue services perform their own committed writes. The lock holder has no card row lock, avoiding self-deadlock. Existing next-evaluation CAS, generating-issue ownership and issue idempotency keys remain required, including for manual refreshes and older server instances that do not use the advisory lock. The persisted five-minute claim remains the failure retry after a scheduled refresh cannot finish. Internal timers do not establish a Cloud wake alarm. + +## Producer audit (in progress) + +| Source | Owning committed boundary | State | +| --- | --- | --- | +| Card create/config/archive/delete, compile, query/summary writeback, evaluation | status-card mutation transactions | wired; ordinary lifecycle exercised | +| Experimental feature flag | instance settings update transaction | wired; disable exercised | +| Company pause/archive/reactivate/delete | company owning transactions | wired; lifecycle acceptance remains | +| Ordinary issue create/update/remove | issue service transactions | wired; create/change exercised | +| Comments create/delete/tombstone | common insert and issue mutation transactions | wired; comment acceptance remains | +| Review state / chat-run milestones | issue-review-state notification boundary | wired; exact caller coverage audit remains | +| Execution-control writes/run settlement | execution-control transaction helpers | wired; exact caller coverage audit remains | +| Agent deletion/membership reassignment, issue tree transitions, title/prefix | owner transaction | wired; representative acceptance remains | +| Workspace selection/binding, plugin/provider workspace, native recovery/reclaim, active watchdog, retry | owner transactions where uncovered by shared notifications | partially wired; remaining per-write audit required | +| Raw workspace cleanup/routes, conversation activation/settlement, queue paths | direct writes and adjacent owner notifications | exact coverage incomplete; not acceptance-ready | +| Imports/independent writers/replicas | supported service writes and startup reconstruction | full ingress contract audit remains | + +Process-local notifications cover writes using the installed database owner. They do not notify an unrelated SQL writer or another server process; those supported paths need explicit ingress or reconstruction coverage. No claim of cross-replica delivery, host suspension, or fleet savings is made by the current tests. + +## Acceptance evidence and remaining gates + +The original status-card PostgreSQL suite passed25/25. New real PostgreSQL tests exercise ordinary card compilation/writeback, source-change debounce, persisted assignment at a controlled provider boundary, result persistence, quiet SQL after completion, earlier/cancelled deadlines, nested outer commit/rollback and drain/resume. The provider boundary is simulated; no model runs. Additional race, restart, failure, active-hour/rate/budget persistence, multi-company and producer-class coverage remain before PR delivery. + +Shared deadline regressions reproduced and corrected a settled-write admission hold during drain. The correction releases only settled holds, retains an in-memory resume wake, and leaves ambiguous transaction intent fenced by the database coordinator. Unit coverage includes first write during drain, failure entering drain, preserved future deadlines, warm standby and ambiguous commit settlement. diff --git a/server/src/__tests__/status-card-evaluation-work.test.ts b/server/src/__tests__/status-card-evaluation-work.test.ts new file mode 100644 index 00000000000..427ad3385eb --- /dev/null +++ b/server/src/__tests__/status-card-evaluation-work.test.ts @@ -0,0 +1,180 @@ +import { randomUUID } from "node:crypto"; +import { eq, sql } from "drizzle-orm"; +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; +import { activityLog, agentWakeupRequests, agents, companies, companyMemberships, createDb, documents, + heartbeatRuns, instanceSettings, issueComments, issues, statusCards, statusCardUpdates, documentRevisions } from "@paperclipai/db"; +import { companySearchQuerySchema, statusCardRefreshPolicySchema } from "@paperclipai/shared"; +import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js"; +import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; +import { createDeliveryWorkCoordinator } from "../services/delivery-work-coordinator.js"; +import { registerStatusCardEvaluationWork, statusCardEvaluationService } from "../services/status-card-evaluation.js"; +import { statusCardService } from "../services/status-cards.js"; +import { issueService } from "../services/issues.js"; +import { instanceSettingsService } from "../services/instance-settings.js"; +import { idleWorkSnapshot } from "../services/task-admission.js"; + +const external = resolveExternalTestDatabaseUrl("PAPERCLIP_STATUS_CARD_EVALUATION_TEST_DATABASE_URL"); +const support = external ? { supported: true } : await getEmbeddedPostgresTestSupport(); +(support.supported ? describe : describe.skip)("status-card evaluation work", () => { + let temporary: Awaited> | undefined; + let db: ReturnType; + let coordinator: ReturnType | undefined; + let draining = false; + let companyId: string, agentId: string; + const errors: unknown[] = []; + const actor = { agentId: null, userId: "board-user" }; + const policy = (mode: "manual" | "reactive" | "interval" = "reactive") => statusCardRefreshPolicySchema.parse({ + mode, debounceSeconds: 60, intervalMinutes: 15, + }); + // Controlled provider boundary: assignment persists a queued wake, and tests + // explicitly run the authorized summarizer writeback. No model/provider runs. + const wakeup = vi.fn(async (id: string, options: { payload?: Record | null; reason?: string | null }) => { + return db.insert(agentWakeupRequests).values({ companyId, agentId: id, source: "assignment", + reason: options.reason, payload: options.payload, status: "queued" }).returning(); + }); + + beforeAll(async () => { + if (!external) temporary = await startEmbeddedPostgresTestDatabase("status-card-evaluation-"); + db = createDb(external ?? temporary!.connectionString, { maxConnections: 6, idleTimeoutSeconds: 0, maxLifetimeSeconds: 0 }); + }, 30_000); + beforeEach(async () => { + companyId = randomUUID(); agentId = randomUUID(); draining = false; errors.length = 0; wakeup.mockClear(); + await db.insert(companies).values({ id: companyId, name: "Status", issuePrefix: `S${companyId.slice(0, 6)}` }); + await db.insert(agents).values({ id: agentId, companyId, name: "Summarizer", role: "general", status: "idle", adapterType: "process", adapterConfig: {} }); + await instanceSettingsService(db).updateExperimental({ enableStatusCards: true }); + await Promise.all(Array.from({ length: 6 }, () => db.execute(sql`select pg_sleep(0.01)`))); + vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"], shouldAdvanceTime: true }); + }); + afterEach(async () => { + await coordinator?.stop(); coordinator = undefined; + vi.useRealTimers(); vi.restoreAllMocks(); + await db.delete(agentWakeupRequests); await db.delete(statusCardUpdates); await db.delete(statusCards); + await db.delete(documentRevisions); await db.delete(documents); await db.delete(activityLog); + await db.delete(issueComments); await db.delete(issues); await db.delete(heartbeatRuns); + await db.delete(instanceSettings); await db.delete(agents); await db.delete(companyMemberships); await db.delete(companies); + }); + afterAll(async () => { await db?.$client.end({ timeout: 1 }); await temporary?.cleanup(); }); + async function start() { + coordinator = createDeliveryWorkCoordinator({ owner: db, canRun: () => !draining, canReconcile: () => true, + onError: error => errors.push(error) }); + await registerStatusCardEvaluationWork(coordinator, statusCardEvaluationService(db, { wakeup }), () => !draining).ready; + } + async function settled() { + await vi.waitFor(async () => { + await vi.advanceTimersByTimeAsync(30_000); + expect(idleWorkSnapshot().active).toBe(0); + }, { timeout: 5000 }); + } + async function quiet() { + await settled(); + expect(coordinator!.nextWakeAt()).toBeNull(); + const reads = vi.spyOn(db, "select"), writes = vi.spyOn(db, "update"), executes = vi.spyOn(db, "execute"); + const calls = wakeup.mock.calls.length; + await vi.advanceTimersByTimeAsync(10 * 60_000); + expect(reads).not.toHaveBeenCalled(); expect(writes).not.toHaveBeenCalled(); expect(executes).not.toHaveBeenCalled(); + expect(wakeup).toHaveBeenCalledTimes(calls); + reads.mockRestore(); writes.mockRestore(); executes.mockRestore(); + } + async function writeback(cardId: string, generationIssueId: string, compile = false) { + const [run] = await db.insert(heartbeatRuns).values({ companyId, agentId, status: "running" }).returning(); + await db.update(issues).set({ checkoutRunId: run!.id }).where(eq(issues.id, generationIssueId)); + const writer = { agentId, runId: run!.id }; + const cards = statusCardService(db); + if (compile) await cards.writeQuery(cardId, { queries: [companySearchQuerySchema.parse({ scope: "issues", status: ["blocked"] })], + title: "Blockers", changeSummary: "Compiled", generationIssueId }, writer); + await cards.writeSummary(cardId, { markdown: "Current blockers", changeSummary: "Updated", generationIssueId }, writer); + await issueService(db).update(generationIssueId, { status: "done" }); + } + async function card(mode: "manual" | "reactive" | "interval" = "reactive") { + const cards = statusCardService(db); + const created = await cards.create(companyId, { interestPrompt: "Blockers", titlePinned: false, agentId, refreshPolicy: policy(mode) }, actor); + const compile = await cards.requestCompile(created.id, actor); + await writeback(created.id, compile.generatingIssue.id, true); + return cards.getById(created.id).then(row => row!); + } + + it("settles empty, disabled, configured manual and unchanged reactive history without recurring SQL", async () => { + await start(); await quiet(); + await card("manual"); await card("reactive"); await quiet(); + await instanceSettingsService(db).updateExperimental({ enableStatusCards: false }); await quiet(); + expect(errors).toEqual([]); + }); + + it("ordinary issue change schedules debounce, assigns one durable wake, writes a result and becomes quiet", async () => { + await start(); const created = await card(); await quiet(); + const changed = await issueService(db).create(companyId, { title: "New blocker", status: "blocked", priority: "medium", createdByUserId: "board-user" }); + await vi.waitFor(async () => { + expect((await statusCardService(db).getById(created.id))?.pendingChangeCount).toBeGreaterThan(0); + }, { timeout: 5000 }); + const pending = await statusCardService(db).getById(created.id); + expect(pending?.pendingChangeCount).toBeGreaterThan(0); + expect(pending?.nextEvalAt).toBeInstanceOf(Date); + expect(wakeup).not.toHaveBeenCalled(); + await vi.advanceTimersByTimeAsync(60_000); await settled(); + const generating = await statusCardService(db).getById(created.id); + expect(generating?.generatingIssueId).toBeTruthy(); + const durable = await db.select().from(agentWakeupRequests); + expect(durable).toHaveLength(1); expect(durable[0]?.payload?.issueId).toBe(generating?.generatingIssueId); + expect(changed.companyId).toBe(companyId); + await writeback(created.id, generating!.generatingIssueId!); await quiet(); + expect(await db.select().from(statusCardUpdates).then(rows => rows.filter(row => row.generationIssueId === generating?.generatingIssueId))) + .toMatchObject([{ status: "ok", finishedAt: expect.any(Date) }]); + expect(errors).toEqual([]); + }); + + it("cancels archived or disabled future work and replaces an earlier interval deadline", async () => { + await start(); let created = await card("interval"); await settled(); + const old = coordinator!.nextWakeAt(); + created = await statusCardService(db).update(created, { refreshPolicy: { ...policy("interval"), intervalMinutes: 1 } }, actor); + await settled(); expect(coordinator!.nextWakeAt()).toBeLessThan(old!); + await statusCardService(db).update(created, { archived: true }, actor); await quiet(); + expect(wakeup).not.toHaveBeenCalled(); expect(errors).toEqual([]); + }); + + it("does not react to rolled-back or nested uncommitted card configuration", async () => { + await start(); const created = await card("manual"); await quiet(); + await expect(db.transaction(async tx => { + await statusCardService(tx as unknown as typeof db).update(created, { refreshPolicy: policy("interval") }, actor); + await vi.advanceTimersByTimeAsync(30_000); expect(wakeup).not.toHaveBeenCalled(); + throw new Error("rollback"); + })).rejects.toThrow("rollback"); + await quiet(); expect((await statusCardService(db).getById(created.id))?.nextEvalAt).toBeNull(); + await db.transaction(async tx => { + await tx.transaction(async nested => { + await statusCardService(nested as unknown as typeof db).update(created, { refreshPolicy: policy("interval") }, actor); + }); + expect(coordinator!.nextWakeAt()).not.toBeNull(); + expect((await statusCardService(db).getById(created.id))?.nextEvalAt).toBeNull(); + }); + await settled(); expect(coordinator!.nextWakeAt()).not.toBeNull(); expect(errors).toEqual([]); + }); + + it("serializes competing connections when one source event preempts a future evaluation", async () => { + const created = await card(); + await db.update(statusCards).set({ nextEvalAt: new Date(Date.now() + 3_600_000) }).where(eq(statusCards.id, created.id)); + await issueService(db).create(companyId, { title: "Concurrent blocker", status: "blocked", priority: "medium", createdByUserId: "board-user" }); + const first = statusCardEvaluationService(db, { wakeup }); + const second = statusCardEvaluationService(db, { wakeup }); + first.changed(new Set([`changes:${companyId}`])); second.changed(new Set([`changes:${companyId}`])); + await Promise.all([first.run(new AbortController().signal), second.run(new AbortController().signal)]); + const pending = await statusCardService(db).getById(created.id); + expect(pending?.pendingChangeCount).toBeGreaterThan(0); + expect(pending?.nextEvalAt?.getTime()).toBeLessThan(Date.now() + 3_600_000); + expect(wakeup).not.toHaveBeenCalled(); + await vi.advanceTimersByTimeAsync(60_000); + await Promise.all([first.run(new AbortController().signal), second.run(new AbortController().signal)]); + const updated = await statusCardService(db).getById(created.id); + expect(updated?.generatingIssueId).toBeTruthy(); expect(wakeup).toHaveBeenCalledOnce(); + expect(await db.select().from(statusCardUpdates).then(rows => rows.filter(row => row.generationIssueId === updated?.generatingIssueId))) + .toHaveLength(1); + }); + + it("preserves the first write during drain until resume", async () => { + await start(); const created = await card("manual"); await quiet(); + draining = true; + await statusCardService(db).update(created, { refreshPolicy: policy("interval") }, actor); + await settled(); expect(wakeup).not.toHaveBeenCalled(); + draining = false; await vi.advanceTimersByTimeAsync(30_000); await settled(); + expect(coordinator!.nextWakeAt()).not.toBeNull(); expect(errors).toEqual([]); + }); +}); diff --git a/server/src/__tests__/status-card-update-engine.test.ts b/server/src/__tests__/status-card-update-engine.test.ts index 09a660e8741..b8309415770 100644 --- a/server/src/__tests__/status-card-update-engine.test.ts +++ b/server/src/__tests__/status-card-update-engine.test.ts @@ -8,6 +8,7 @@ import { filterStatusCardChanges, isWithinStatusCardActiveHours, nextStatusCardEvaluationAt, + nextStatusCardPolicyEligibility, statusCardChangesHash, } from "../services/status-card-update-engine.js"; @@ -78,6 +79,28 @@ describe("status card update engine", () => { expect(evaluateStatusCardPolicy({ policy: hours, now: new Date("2026-07-23T18:00:00.000Z"), lastChangeAt: null, updatesLastHour: 0, tokensToday: 0, manual: false }).action).toBe("pause_hours"); }); + it("disarms unchanged reactive evaluation while retaining exact pending eligibility", () => { + const policy = statusCardRefreshPolicySchema.parse({ mode: "reactive", debounceSeconds: 90, maxUpdatesPerHour: 2, + activeHours: { start: "09:00", end: "17:00", timezone: "UTC" } }); + const now = new Date("2026-07-23T16:59:30Z"); + expect(nextStatusCardEvaluationAt(policy, now)).toBeNull(); + expect(nextStatusCardPolicyEligibility({ policy, now, lastChangeAt: now, completedUpdateTimes: [], tokensToday: 0 })) + .toEqual(new Date("2026-07-24T09:00:00Z")); + const open = new Date("2026-07-23T14:00:00Z"); + expect(nextStatusCardPolicyEligibility({ policy, now: open, lastChangeAt: new Date("2026-07-23T13:00:00Z"), + completedUpdateTimes: [new Date("2026-07-23T13:30:00Z"), new Date("2026-07-23T13:45:00Z")], tokensToday: 0 })) + .toEqual(new Date("2026-07-23T14:30:00.001Z")); + expect(nextStatusCardPolicyEligibility({ policy, now: open, lastChangeAt: open, completedUpdateTimes: [], tokensToday: 100_000 })) + .toEqual(new Date("2026-07-24T09:00:00Z")); + }); + + it("retains empty-window pending changes without a recurring eligibility scan", () => { + const policy = statusCardRefreshPolicySchema.parse({ mode: "reactive", debounceSeconds: 1, + activeHours: { start: "09:00", end: "09:00", timezone: "UTC" } }); + const now = new Date("2026-07-23T14:00:00Z"); + expect(nextStatusCardPolicyEligibility({ policy, now, lastChangeAt: now, completedUpdateTimes: [], tokensToday: 0 })).toBeNull(); + }); + it("selects full rebuilds for bounded drift rules and incremental otherwise", () => { const base = { hasDocument: true, changeCount: 2, queryVersion: 3, lastUpdateQueryVersion: 3, incrementalCount: 2, configurationChanged: false }; expect(chooseStatusCardUpdateKind(base)).toBe("incremental"); diff --git a/server/src/__tests__/status-cards.test.ts b/server/src/__tests__/status-cards.test.ts index e1436666316..0c24b05dd70 100644 --- a/server/src/__tests__/status-cards.test.ts +++ b/server/src/__tests__/status-cards.test.ts @@ -1,3 +1,4 @@ +import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js"; import { randomUUID } from "node:crypto"; import express from "express"; import request from "supertest"; @@ -37,7 +38,8 @@ import { startEmbeddedPostgresTestDatabase, } from "./helpers/embedded-postgres.js"; -const embeddedPostgresSupport = await getEmbeddedPostgresTestSupport(); +const externalDatabaseUrl = resolveExternalTestDatabaseUrl("PAPERCLIP_STATUS_CARDS_TEST_DATABASE_URL"); +const embeddedPostgresSupport = externalDatabaseUrl ? { supported: true } : await getEmbeddedPostgresTestSupport(); const describeEmbeddedPostgres = embeddedPostgresSupport.supported ? describe : describe.skip; type Db = ReturnType; @@ -79,8 +81,8 @@ describeEmbeddedPostgres("status card routes", () => { let tempDb: Awaited> | null = null; beforeAll(async () => { - tempDb = await startEmbeddedPostgresTestDatabase("paperclip-status-cards-"); - db = createDb(tempDb.connectionString); + if (!externalDatabaseUrl) tempDb = await startEmbeddedPostgresTestDatabase("paperclip-status-cards-"); + db = createDb(externalDatabaseUrl ?? tempDb!.connectionString, { maxConnections: 6 }); }, 20_000); afterEach(async () => { @@ -100,6 +102,7 @@ describeEmbeddedPostgres("status card routes", () => { }); afterAll(async () => { + await db?.$client.end({ timeout: 1 }); await tempDb?.cleanup(); }); diff --git a/server/src/index.ts b/server/src/index.ts index ae229f03518..6fb78f1612d 100644 --- a/server/src/index.ts +++ b/server/src/index.ts @@ -145,13 +145,12 @@ import { reconcileCodexLocalManagedHomesOnStartup, reconcilePersistedRuntimeServicesOnStartup, routineService, - statusCardService, toolAccessService, workspaceOperationService, } from "./services/index.js"; import { questionResponseDeliveryService } from "./services/question-response-delivery.js"; import { deliverNativeQuestionResponse } from "./services/native-runtime/native-question-bridge.js"; -import { queueIssueAssignmentWakeup } from "./services/issue-assignment-wakeup.js"; +import { registerStatusCardEvaluationWork, statusCardEvaluationService } from "./services/status-card-evaluation.js"; import { createSecretProposalsService } from "./services/secret-proposals.js"; import { environmentRuntimeService } from "./services/environment-runtime.js"; import { historicalCleanupVerificationService } from "./services/historical-cleanup-verification.js"; @@ -1569,7 +1568,7 @@ async function startServerWithDatabaseTeardown( if (result.settled > 0) logger.info(result, "reconciled routine execution issue status"); }).catch(() => logger.warn("routine execution status reconciliation deferred"))); }; - const statusCards = statusCardService(db as any); + const statusCardEvaluation = statusCardEvaluationService(db as any, heartbeat); const issues = issueService(db as any); const mergedPullRequestConfirmations = issueThreadInteractionService(db as any, { wakeup: heartbeat.wakeup, @@ -1980,6 +1979,8 @@ async function startServerWithDatabaseTeardown( () => !isIdleTaskDrainActive() && !isWarmStandby() && !heartbeatSchedulerStopped).ready); await step("join_request_retention", () => registerJoinRequestRetentionWork(deliveryWork, joinRequestRetentionService(db as any), () => !isIdleTaskDrainActive() && !isWarmStandby() && !heartbeatSchedulerStopped).ready); + await step("status_card_evaluation", () => registerStatusCardEvaluationWork(deliveryWork, statusCardEvaluation, + () => !isIdleTaskDrainActive() && !isWarmStandby() && !heartbeatSchedulerStopped).ready); await step("retention_sweep", runRetentionSweep); }; @@ -2010,33 +2011,7 @@ async function startServerWithDatabaseTeardown( scheduleGitHubConnectionContinuitySweep(); if (isIdleTaskDrainActive() || heartbeatSchedulerStopped) return; - trackHeartbeatSchedulerWork((async () => { - const experimental = await instanceSettingsService(db).getExperimental(); - if (experimental.enableStatusCards !== true) return; - const result = await statusCards.tickDueStatusCards(new Date()); - await Promise.all(result.enqueued.map(async ({ cardId, generatingIssue }) => { - try { - await queueIssueAssignmentWakeup({ - heartbeat, - issue: generatingIssue, - reason: "status_card_update_assigned", - mutation: "status_card.scheduler_update_requested", - contextSource: "status_card_scheduler", - requestedByActorType: "system", - taskKey: `status-card:${cardId}`, - rethrowOnError: true, - }); - } catch (err) { - await issues.update(generatingIssue.id, { status: "cancelled" }); - throw err; - } - })); - if (result.evaluated > 0 || result.enqueued.length > 0) { - logger.info({ evaluated: result.evaluated, enqueued: result.enqueued.length }, "status-card scheduler tick complete"); - } - })().catch((err) => { - logger.error({ err }, "status-card scheduler tick failed"); - })); + if (isIdleTaskDrainActive() || heartbeatSchedulerStopped) return; if (!(await heartbeat.resolveSchedulingSuppression()).suppressed) { diff --git a/server/src/modules/active-run-watchdog/adapters/postgres.ts b/server/src/modules/active-run-watchdog/adapters/postgres.ts index 9531c449260..002415c03a6 100644 --- a/server/src/modules/active-run-watchdog/adapters/postgres.ts +++ b/server/src/modules/active-run-watchdog/adapters/postgres.ts @@ -1,3 +1,4 @@ +import { notifyStatusCardIssueChange } from "../../../services/status-card-notifications.js"; import { DELIVERY_QUEUES, notifyDeliveryWork } from "../../../services/delivery-work-notifications.js"; import { notifyExternalAgentSettlement } from "../../external-agents/index.js"; import { notifyNativeStatusReassessment } from "../../../services/native-status-reassessment-notifications.js"; @@ -385,6 +386,7 @@ export function createPostgresWatchdogAdapter(db: Db): WatchdogRunReader & Watch )); } + await notifyStatusCardIssueChange(tx, companyId); await tx .update(issues) .set({ diff --git a/server/src/services/access.ts b/server/src/services/access.ts index aa0f9dcc828..477c0434c50 100644 --- a/server/src/services/access.ts +++ b/server/src/services/access.ts @@ -1,3 +1,4 @@ +import { notifyStatusCardIssueChange } from "./status-card-notifications.js"; import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js"; import { notifyNativeStatusReassessment } from "./native-status-reassessment-notifications.js"; import { notifyUnsafeWorkspaceExportRecovery } from "./unsafe-workspace-export-notifications.js"; @@ -745,6 +746,7 @@ export function accessService(db: Db) { await notifyNativeStatusReassessment(tx, { companyId }); await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors); await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers); + await notifyStatusCardIssueChange(tx, companyId); const resetInProgress = await tx .update(issues) .set({ diff --git a/server/src/services/agent-deletion.ts b/server/src/services/agent-deletion.ts index 3650c40f1a6..5907d222e1d 100644 --- a/server/src/services/agent-deletion.ts +++ b/server/src/services/agent-deletion.ts @@ -1,3 +1,4 @@ +import { notifyStatusCardIssueChange } from "./status-card-notifications.js"; import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js"; import { notifyExternalConversationSettlement } from "./external-conversation-settlement-notifications.js"; import { notifyUnsafeWorkspaceExportRecovery } from "./unsafe-workspace-export-notifications.js"; @@ -38,6 +39,7 @@ export async function deleteAgentDependencies(tx: Db, companyId: string, id: str await issueThreadInteractionService(tx).cancelPendingForDeletedAddressee(companyId, id); await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors); await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers); + await notifyStatusCardIssueChange(tx, companyId); await tx.update(agents).set({ reportsTo: null }).where(eq(agents.reportsTo, id)); await tx .update(issues) diff --git a/server/src/services/companies.ts b/server/src/services/companies.ts index 0f2ed547a79..8efd6a269fb 100644 --- a/server/src/services/companies.ts +++ b/server/src/services/companies.ts @@ -1,3 +1,4 @@ +import { notifyStatusCardIssueChange } from "./status-card-notifications.js"; import { notifyChatInboundWakeNotices } from "./chat-inbound-wake-notice-notifications.js"; import { notifyCompanyRetentionWork } from "./retention-work-notifications.js"; import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js"; @@ -336,6 +337,7 @@ export function companyService(db: Db, _budgetHooks: BudgetServiceHooks = {}) { await notifyDeliveryWork(tx, DELIVERY_QUEUES.skillSourceSync); await notifyDeliveryWork(tx, DELIVERY_QUEUES.routineSchedules); await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors); + await notifyStatusCardIssueChange(tx, id); await notifyCompanyRetentionWork(tx, id); await notifyChatInboundWakeNotices(tx, { companyId: id }); await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentLifecycle, id); @@ -492,6 +494,7 @@ export function companyService(db: Db, _budgetHooks: BudgetServiceHooks = {}) { await notifyDeliveryWork(tx, DELIVERY_QUEUES.skillSourceSync); await notifyDeliveryWork(tx, DELIVERY_QUEUES.routineSchedules); await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors); + await notifyStatusCardIssueChange(tx, id); await notifyCompanyRetentionWork(tx, id); await notifyChatInboundWakeNotices(tx, { companyId: id }); await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentLifecycle, id); diff --git a/server/src/services/company-deletion.ts b/server/src/services/company-deletion.ts index 7ab88aadfa9..0379638968b 100644 --- a/server/src/services/company-deletion.ts +++ b/server/src/services/company-deletion.ts @@ -1,3 +1,4 @@ +import { notifyStatusCardEvaluation } from "./status-card-notifications.js"; import { notifyUnsafeWorkspaceExportRecovery } from "./unsafe-workspace-export-notifications.js"; import { notifyCancelledWorkspacePreservation } from "./cancelled-workspace-preservation-notifications.js"; import { notifyInstructionCollection } from "./instruction-collection-notifications.js"; @@ -77,6 +78,7 @@ export async function deleteCompany(db: Db, id: string) { await notifyDeliveryWork(tx, DELIVERY_QUEUES.skillPackages); await notifyDeliveryWork(tx, DELIVERY_QUEUES.routineSchedules); await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors); + await notifyStatusCardEvaluation(tx, id); await notifyDeliveryWork(tx, DELIVERY_QUEUES.secretProposalExpiry, id); await notifyToolReviewChange(tx, id); await agentObservabilityService(tx as unknown as Db).removeForCompanyDeletion(id); diff --git a/server/src/services/company-search.ts b/server/src/services/company-search.ts index 220967e362d..bcf2a550d3f 100644 --- a/server/src/services/company-search.ts +++ b/server/src/services/company-search.ts @@ -207,14 +207,20 @@ function priorityRank(priority: string | null | undefined) { return index >= 0 ? index : ISSUE_PRIORITIES.length; } -function updatedWithinStart(value: string | undefined, now = new Date()): Date | null { +/** Duration used by saved search membership and its deadline consumers. */ +export function companySearchUpdatedWithinMs(value: string | undefined): number | null { if (!value) return null; const match = /^(\d+)(h|d|w|m)$/.exec(value); if (!match) return null; const amount = Number.parseInt(match[1]!, 10); const unit = match[2]; const hours = unit === "h" ? amount : unit === "d" ? amount * 24 : unit === "w" ? amount * 24 * 7 : amount * 24 * 30; - return new Date(now.getTime() - hours * 60 * 60 * 1000); + return hours * 60 * 60 * 1000; +} + +function updatedWithinStart(value: string | undefined, now = new Date()): Date | null { + const duration = companySearchUpdatedWithinMs(value); + return duration === null ? null : new Date(now.getTime() - duration); } function issueOnlyFiltersActive(query: CompanySearchQuery) { diff --git a/server/src/services/company-skills.ts b/server/src/services/company-skills.ts index 7ba950039e9..faf54c410a9 100644 --- a/server/src/services/company-skills.ts +++ b/server/src/services/company-skills.ts @@ -1,3 +1,4 @@ +import { notifyStatusCardIssueChange } from "./status-card-notifications.js"; import { notifySkillVersionRetention } from "./skill-version-retention-notifications.js"; import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js"; import { verifyFileManifest } from "./file-manifest-cache.js"; @@ -7450,6 +7451,7 @@ export function companySkillService(db: Db, options: { actor?: AuthorizationActo for (const row of rows) { await db.transaction(async (tx) => { await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers); + await notifyStatusCardIssueChange(tx, companyId); await tx .update(issues) .set({ hiddenAt: now, updatedAt: now }) diff --git a/server/src/services/delivery-work-notifications.ts b/server/src/services/delivery-work-notifications.ts index 9195f1fcb12..fdc4aaadc20 100644 --- a/server/src/services/delivery-work-notifications.ts +++ b/server/src/services/delivery-work-notifications.ts @@ -2,6 +2,7 @@ import { signalDatabaseWork, subscribeDatabaseWork } from "@paperclipai/db"; /** Topics for existing durable queues; register intent before their writes. */ export const DELIVERY_QUEUES = { + statusCardEvaluation: "status-card-evaluation", runEvidenceMigration: "run-evidence-migration", providerTraceRetention: "provider-trace-retention", nativeRunnerEvidenceRetention: "native-runner-evidence-retention", diff --git a/server/src/services/execution-control-notifications.ts b/server/src/services/execution-control-notifications.ts index 22c8913bbcd..c7b26a8bd72 100644 --- a/server/src/services/execution-control-notifications.ts +++ b/server/src/services/execution-control-notifications.ts @@ -1,7 +1,9 @@ +import { notifyStatusCardIssueChange } from "./status-card-notifications.js"; import type { Db } from "@paperclipai/db"; import { DELIVERY_QUEUES, notifyDeliveryWork, type DeliveryQueue } from "./delivery-work-notifications.js"; export async function notifyExecutionRunWork(transaction: object) { + await notifyStatusCardIssueChange(transaction); for (const queue of [DELIVERY_QUEUES.executionStatus, DELIVERY_QUEUES.executionFinalization, DELIVERY_QUEUES.executionReplacement, DELIVERY_QUEUES.executionDisposition, DELIVERY_QUEUES.issueMonitors]) { await notifyDeliveryWork(transaction, queue); @@ -14,6 +16,7 @@ export function writeExecutionControl( write: (tx: Db) => PromiseLike, ): Promise { return db.transaction(async transaction => { + await notifyStatusCardIssueChange(transaction); await notifyDeliveryWork(transaction, DELIVERY_QUEUES.issueMonitors); for (const topic of typeof queue === "string" ? [queue] : queue) await notifyDeliveryWork(transaction, topic); return write(transaction as unknown as Db); diff --git a/server/src/services/execution-workspaces.ts b/server/src/services/execution-workspaces.ts index 4b716916638..caaae186ad8 100644 --- a/server/src/services/execution-workspaces.ts +++ b/server/src/services/execution-workspaces.ts @@ -1,3 +1,4 @@ +import { notifyStatusCardIssueChange } from "./status-card-notifications.js"; import { effectiveProjectWorkspacePolicy } from "./project-workspace-provider.js"; import { inspectPluginWorkspace, removePluginWorkspace, restorePluginWorkspace, type WorkspaceWorkers } from "./plugin-workspaces.js"; import { acquireExecutionWorkspaceLifecycleLock, withExecutionWorkspaceLifecycleLock } from "./execution-workspace-lifecycle-lock.js"; @@ -1590,6 +1591,7 @@ export function executionWorkspaceService(db: Db, opts: ExecutionWorkspaceServic await validateSelection(input, tx); const intent: TaskWorkspaceIntent = { version: 1, request: { key: input.requestKey, expectedBindingRevision: input.expectedBindingRevision }, selection: input.selection, source: input.source ?? "explicit" }; const nextRevision = task.workspaceBindingRevision + 1; + await notifyStatusCardIssueChange(tx, task.companyId); await tx.update(issues).set({ workspaceBindingRevision: nextRevision, workspacePendingSelection: { version: 1, requestKey: input.requestKey, expectedBindingRevision: nextRevision, intent }, updatedAt: new Date() }).where(eq(issues.id, task.id)); @@ -1633,6 +1635,7 @@ export function executionWorkspaceService(db: Db, opts: ExecutionWorkspaceServic } } } + await notifyStatusCardIssueChange(tx, task.companyId); await tx.update(issues).set({ ...patch, assigneeAdapterOverrides, workspaceSelection: pending.intent, workspacePendingSelection: null, workspaceBindingRevision: task.workspaceBindingRevision + 1, updatedAt: new Date() }).where(eq(issues.id, task.id)); return true; @@ -1651,6 +1654,7 @@ export function executionWorkspaceService(db: Db, opts: ExecutionWorkspaceServic const pendingAfterFirstBinding = !task.executionWorkspaceId && pending?.expectedBindingRevision === task.workspaceBindingRevision ? { ...pending, expectedBindingRevision: nextRevision } : pending; + await notifyStatusCardIssueChange(tx, companyId); const [bound] = await tx.update(issues).set({ ...patch, executionWorkspaceId: workspaceId, workspaceBindingRevision: nextRevision, workspacePendingSelection: pendingAfterFirstBinding, updatedAt: new Date() }) .where(eq(issues.id, issueId)).returning(); diff --git a/server/src/services/heartbeat/retries.ts b/server/src/services/heartbeat/retries.ts index 946f5e309f8..de03aebeca0 100644 --- a/server/src/services/heartbeat/retries.ts +++ b/server/src/services/heartbeat/retries.ts @@ -1,3 +1,4 @@ +import { notifyStatusCardIssueChange } from "../status-card-notifications.js"; import { deriveTaskKeyWithHeartbeatFallback } from "./run-state.js"; import { infrastructureRetryDelayMs } from "../system-recovery-state.js"; import { @@ -1178,6 +1179,7 @@ export function createHeartbeatRetries(db: Db, dependencies: HeartbeatRetryDepen } if (issueId) { + await notifyStatusCardIssueChange(tx, run.companyId); await tx .update(issues) .set({ diff --git a/server/src/services/instance-settings.ts b/server/src/services/instance-settings.ts index 0208653a86f..35eddc10660 100644 --- a/server/src/services/instance-settings.ts +++ b/server/src/services/instance-settings.ts @@ -1,3 +1,4 @@ +import { notifyStatusCardEvaluation } from "./status-card-notifications.js"; import { notifySkillVersionRetentionReconstruction } from "./skill-version-retention-notifications.js"; import { notifyNativeEnvironmentSettlement } from "./native-environment-settlement-notifications.js"; import { notifyExternalAgentSettlement } from "../modules/external-agents/index.js"; @@ -625,6 +626,7 @@ export function instanceSettingsService(db: Db, options: InstanceSettingsService } const now = new Date(); const [updated] = await db.transaction(async tx => { + if (patch.enableStatusCards !== undefined) await notifyStatusCardEvaluation(tx); await notifyDeliveryWork(tx, DELIVERY_QUEUES.mcpEvents, "*"); await notifyDeliveryWork(tx, DELIVERY_QUEUES.dotEvents, "*"); await notifyDeliveryWork(tx, DELIVERY_QUEUES.grokDispatch); diff --git a/server/src/services/issue-comment-writes.ts b/server/src/services/issue-comment-writes.ts index e24e78e9855..4d4cf374e5b 100644 --- a/server/src/services/issue-comment-writes.ts +++ b/server/src/services/issue-comment-writes.ts @@ -1,3 +1,4 @@ +import { notifyStatusCardIssueChange } from "./status-card-notifications.js"; import type { PgInsertValue } from "drizzle-orm/pg-core"; import { issueComments, type Db } from "@paperclipai/db"; import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js"; @@ -6,6 +7,7 @@ type WriteDb = Db | Parameters[0]>[0]; export function insertIssueComment(db: WriteDb, input: PgInsertValue & { companyId: string }) { return db.transaction(async tx => { + await notifyStatusCardIssueChange(tx, input.companyId); await notifyDeliveryWork(tx, DELIVERY_QUEUES.dotEvents, `company:${input.companyId}`); return tx.insert(issueComments).values(input).returning(); }); diff --git a/server/src/services/issue-prefix.ts b/server/src/services/issue-prefix.ts index 5e31f9bd7ff..649a4aaae9b 100644 --- a/server/src/services/issue-prefix.ts +++ b/server/src/services/issue-prefix.ts @@ -1,3 +1,4 @@ +import { notifyStatusCardIssueChange } from "./status-card-notifications.js"; import { and, eq, sql } from "drizzle-orm"; import type { PgColumn } from "drizzle-orm/pg-core"; import type { Db } from "@paperclipai/db"; @@ -127,6 +128,7 @@ export async function rekeyCompanyIssueIdentifiers( // overload, which returns NULL for a non-pattern argument. const tailStart = sql`${fromPrefix.length + 1}::int`; + await notifyStatusCardIssueChange(tx, companyId); const rekeyedIssues = await tx .update(issues) .set({ identifier: sql`${toPrefix} || substring(${issues.identifier} from ${tailStart})` }) diff --git a/server/src/services/issue-review-state-notifications.ts b/server/src/services/issue-review-state-notifications.ts index 679b7e37475..d66ca96eb0e 100644 --- a/server/src/services/issue-review-state-notifications.ts +++ b/server/src/services/issue-review-state-notifications.ts @@ -1,9 +1,11 @@ +import { notifyStatusCardIssueChange } from "./status-card-notifications.js"; import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js"; import { notifyExternalConversationSettlement } from "./external-conversation-settlement-notifications.js"; import { notifyNativeReviewResponse } from "./native-review-response-notifications.js"; import { notifyNativeStatusReassessment } from "./native-status-reassessment-notifications.js"; export async function notifyIssueReviewState(tx: object, scope: { companyId: string; issueId?: string; runId?: string; agentId?: string }) { + await notifyStatusCardIssueChange(tx, scope.companyId); await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors); await notifyExternalConversationSettlement(tx, scope); await notifyNativeReviewResponse(tx, scope); diff --git a/server/src/services/issue-title.ts b/server/src/services/issue-title.ts index af6716e3aa4..51e360269d7 100644 --- a/server/src/services/issue-title.ts +++ b/server/src/services/issue-title.ts @@ -1,3 +1,4 @@ +import { notifyStatusCardIssueChange } from "./status-card-notifications.js"; import { and, eq } from "drizzle-orm"; import { heartbeatRuns, issues, type Db } from "@paperclipai/db"; import { setIssueTitleSchema, type SetIssueTitle } from "@paperclipai/shared"; @@ -76,6 +77,7 @@ export async function setIssueTitle( || (issue.title === title && !issue.titleNeedsGeneration)) { return { result: await saveReceipt({ id: issue.id, title: issue.title, titleNeedsGeneration: issue.titleNeedsGeneration, changed: false }), publication: null }; } + await notifyStatusCardIssueChange(tx, companyId); await tx.update(issues).set({ title, titleNeedsGeneration: false, updatedAt: new Date() }).where(predicate); await issueReferenceService(tx).syncIssue(issueId, tx); const { publication } = await persistActivity(tx, { diff --git a/server/src/services/issue-tree-control.ts b/server/src/services/issue-tree-control.ts index 0d7633f846a..a8009559827 100644 --- a/server/src/services/issue-tree-control.ts +++ b/server/src/services/issue-tree-control.ts @@ -1,3 +1,4 @@ +import { notifyStatusCardIssueChange } from "./status-card-notifications.js"; import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js"; import { notifyNativeStatusReassessment } from "./native-status-reassessment-notifications.js"; import { notifyUnsafeWorkspaceExportRecovery } from "./unsafe-workspace-export-notifications.js"; @@ -1011,6 +1012,7 @@ export function issueTreeControlService(db: Db) { await notifyChatResponseAvailability(tx, { companyId, issueId }); if (status === "blocked" || status === "in_review") await notifyUnsafeWorkspaceExportRecovery(tx, { companyId, issueId }); } + await notifyStatusCardIssueChange(tx, companyId); const rows = await tx .update(issues) .set({ diff --git a/server/src/services/issues.ts b/server/src/services/issues.ts index 42c65bb5d02..8787ac867a8 100644 --- a/server/src/services/issues.ts +++ b/server/src/services/issues.ts @@ -1,3 +1,4 @@ +import { notifyStatusCardIssueChange } from "./status-card-notifications.js"; import { notifyExternalConversationSettlement } from "./external-conversation-settlement-notifications.js"; import { notifyNativeRecoveryNotices } from "./native-recovery-notice-notifications.js"; import { notifyNativeRunProjection } from "./native-run-projection-notifications.js"; @@ -10606,6 +10607,7 @@ export function issueService(db: Db) { } if (values.monitorNextCheckAt) await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors); await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers); + await notifyStatusCardIssueChange(tx, companyId); const [issue] = await tx.insert(issues).values(values).returning(); await attachOwnedDraftImages(tx, issue, issue.description, { userId: issueData.createdByUserId, agentId: issueData.createdByAgentId }); if (issue.visibility === "private" && !inheritedPrivateSubtree && issueData.createdByAgentId) { @@ -11486,6 +11488,7 @@ export function issueService(db: Db) { (blockedByIssueIds !== undefined || issueData.unblockDescriptor !== undefined)))) { patch.statusVersion = sql`${issues.statusVersion} + 1` as unknown as number; } + await notifyStatusCardIssueChange(tx, receiptExisting.companyId); if ((patch.status !== undefined && patch.status !== receiptExisting.status) || patch.completedAt !== undefined || patch.cancelledAt !== undefined || issueData.parentId !== undefined || issueData.executionWorkspaceId !== undefined) { @@ -11954,6 +11957,7 @@ export function issueService(db: Db) { if (owner) { await notifyExternalConversationSettlement(tx, { companyId: owner.companyId }); await notifyApprovalWork(tx, owner.companyId); + await notifyStatusCardIssueChange(tx, owner.companyId); } await notifyDecisionTargetIssueRemoval(tx, id); await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors); @@ -12726,6 +12730,7 @@ export function issueService(db: Db) { .from(issueComments).where(eq(issueComments.id, commentId)); if (!source) return null; await notifyExternalConversationSettlement(tx, source); + await notifyStatusCardIssueChange(tx, source.companyId); const [comment] = await tx .delete(issueComments) .where(eq(issueComments.id, commentId)) @@ -12765,6 +12770,7 @@ export function issueService(db: Db) { .for("update"); if (!source) return null; await notifyExternalConversationSettlement(tx, source); + await notifyStatusCardIssueChange(tx, source.companyId); await notifyChatInboundWakeNotices(tx, source); const now = new Date(); const [comment] = await tx diff --git a/server/src/services/native-runtime/native-finalization-reconciler.ts b/server/src/services/native-runtime/native-finalization-reconciler.ts index af828edfb98..c5cf832982e 100644 --- a/server/src/services/native-runtime/native-finalization-reconciler.ts +++ b/server/src/services/native-runtime/native-finalization-reconciler.ts @@ -1,3 +1,4 @@ +import { notifyStatusCardIssueChange } from "../status-card-notifications.js"; import { notifyExternalAgentSettlement } from "../../modules/external-agents/index.js"; import { pendingNativeRunProjectionCondition } from "./native-run-projection.js"; import { nativeStatusReassessmentSourceIsSuperseded, reassessNativeStatusFromSource } from "./native-status-reassessment.js"; @@ -334,6 +335,7 @@ export async function claimNativeSessionResumptions(input: { nativePhaseUpdatedAt: now, updatedAt: now, }).where(eq(heartbeatRuns.id, row.run.id)); + await notifyStatusCardIssueChange(tx, run.companyId); await tx.update(issues).set({ executionRunId: run.id, executionAgentNameKey: agent.name.trim().toLowerCase() || null, diff --git a/server/src/services/native-runtime/native-review-participant.ts b/server/src/services/native-runtime/native-review-participant.ts index 573574dd19d..6391d506d5d 100644 --- a/server/src/services/native-runtime/native-review-participant.ts +++ b/server/src/services/native-runtime/native-review-participant.ts @@ -1,3 +1,4 @@ +import { notifyStatusCardIssueChange } from "../status-card-notifications.js"; import { agents, heartbeatRuns, @@ -240,6 +241,7 @@ export async function claimNativeReviewExecutionLock(db: Db, input: { if (!run || !["queued", "running"].includes(run.status) || !admitted || !expected || admitted.nativeReviewInteractionId !== expected.nativeReviewInteractionId || admitted.nativeReviewDecisionId !== expected.nativeReviewDecisionId) return false; + await notifyStatusCardIssueChange(tx, input.companyId); const claimed = await tx.update(issues).set({ executionRunId: input.runId, executionAgentNameKey: input.agentNameKey, diff --git a/server/src/services/native-runtime/native-workspace-export-recovery.ts b/server/src/services/native-runtime/native-workspace-export-recovery.ts index 41e51f2ddc7..0c3aac34799 100644 --- a/server/src/services/native-runtime/native-workspace-export-recovery.ts +++ b/server/src/services/native-runtime/native-workspace-export-recovery.ts @@ -1,3 +1,4 @@ +import { notifyStatusCardIssueChange } from "../status-card-notifications.js"; import { notifyNativeFinalizationWork } from "../native-finalization-notifications.js"; import { alias } from "drizzle-orm/pg-core"; import { notifyUnsafeWorkspaceExportRecovery, type UnsafeWorkspaceExportScope } from "../unsafe-workspace-export-notifications.js"; @@ -107,6 +108,7 @@ async function recoverCandidate(db: Db, candidate: { runId: string; companyId: s nativePhase: "result_accepted", nativePhaseUpdatedAt: now, resultJson: { ...run.resultJson, finalizationPhase: "result_accepted", failureCode: null, originalFailureCode: null, nextAttemptAt: null }, updatedAt: now, }).where(eq(heartbeatRuns.id, run.id)).returning(); + await notifyStatusCardIssueChange(tx, run.companyId); await tx.update(issues).set({ executionRunId: run.id, updatedAt: now }).where(eq(issues.id, issue.id)); await appendHeartbeatRunEvent(tx as unknown as Db, { companyId: run.companyId, agentId: run.agentId, runId: run.id, eventType: "workspace_export_omitted", stream: "system", level: "info", diff --git a/server/src/services/native-runtime/native-workspace-export-retry.ts b/server/src/services/native-runtime/native-workspace-export-retry.ts index 47de8c73cf4..5a79c9e782e 100644 --- a/server/src/services/native-runtime/native-workspace-export-retry.ts +++ b/server/src/services/native-runtime/native-workspace-export-retry.ts @@ -1,3 +1,4 @@ +import { notifyStatusCardIssueChange } from "../status-card-notifications.js"; import { notifyNativeFinalizationWork } from "../native-finalization-notifications.js"; import { notifyEnvironmentCleanup } from "../environment-cleanup-notifications.js"; import { randomUUID } from "node:crypto"; @@ -141,6 +142,7 @@ export async function retryNativeWorkspaceExport(input: { await tx.update(heartbeatRuns).set({ status: "running", finishedAt: null, error: null, errorCode: null, nativePhase: "result_accepted", nativePhaseUpdatedAt: now, resultJson: { ...current.run.resultJson, workspaceExportRetry: retry, finalizationPhase: "result_accepted", failureCode: null, originalFailureCode: null, nextAttemptAt: null }, updatedAt: now }).where(eq(heartbeatRuns.id, current.run.id)); + await notifyStatusCardIssueChange(tx, current.run.companyId); await tx.update(issues).set({ executionRunId: current.run.id, updatedAt: now }).where(eq(issues.id, current.issue.id)); // Reclaim the same lease only for control-plane copyback. The normal // finalization settlement will stop/retain it again on success or failure. diff --git a/server/src/services/native-runtime/native-workspace-reconstruction.ts b/server/src/services/native-runtime/native-workspace-reconstruction.ts index 5273b30216a..33e37704389 100644 --- a/server/src/services/native-runtime/native-workspace-reconstruction.ts +++ b/server/src/services/native-runtime/native-workspace-reconstruction.ts @@ -1,3 +1,4 @@ +import { notifyStatusCardIssueChange } from "../status-card-notifications.js"; import { DELIVERY_QUEUES, notifyDeliveryWork } from "../delivery-work-notifications.js"; import { notifyUnsafeWorkspaceExportRecovery } from "../unsafe-workspace-export-notifications.js"; import { and, eq, gt, inArray, ne, notInArray, or, sql } from "drizzle-orm"; @@ -187,6 +188,7 @@ export async function reconstructNativeWorkspace(input: { }).where(eq(issueRecoveryActions.id, source.action.id)); await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors); await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers); + await notifyStatusCardIssueChange(tx, input.companyId); await tx.update(issues).set({ status: "todo", unblockDescriptor: null, blockedTransitionAt: null, blockedOwnerNotifiedAt: null, updatedAt: now }).where(eq(issues.id, issue.id)); const publication = await persistActivity(db, { companyId: input.companyId, actorType: "user", actorId: input.actorId, diff --git a/server/src/services/native-runtime/paperclip-runner-tool-authority.ts b/server/src/services/native-runtime/paperclip-runner-tool-authority.ts index aa8970db19f..cf07e70eb7a 100644 --- a/server/src/services/native-runtime/paperclip-runner-tool-authority.ts +++ b/server/src/services/native-runtime/paperclip-runner-tool-authority.ts @@ -1,3 +1,4 @@ +import { notifyStatusCardIssueChange } from "../status-card-notifications.js"; import { requiredQualityReviewSchema } from "@paperclipai/shared"; import { priorRecoveryApiReceipt } from "../recovery-effect-receipts.js"; import { DELIVERY_QUEUES, notifyDeliveryWork } from "../delivery-work-notifications.js"; @@ -2059,6 +2060,7 @@ export class PaperclipRunnerToolAuthority { } const patch = prepareIssueMonitorUpdate(target, input.monitor, actor); await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors); + await notifyStatusCardIssueChange(tx, this.binding.companyId); const [updated] = await tx.update(issues).set({ ...patch, updatedAt: new Date() }) .where(and(eq(issues.id, target.id), eq(issues.companyId, this.binding.companyId))).returning(); if (!updated) throw notFound("Task not found"); diff --git a/server/src/services/plugin-workspaces.ts b/server/src/services/plugin-workspaces.ts index e2383dcfc74..ca04b081e48 100644 --- a/server/src/services/plugin-workspaces.ts +++ b/server/src/services/plugin-workspaces.ts @@ -1,3 +1,4 @@ +import { notifyStatusCardIssueChange } from "./status-card-notifications.js"; import { effectiveProjectWorkspacePolicy, isWorkspaceProvider } from "./project-workspace-provider.js"; import { releaseProviderRecoveryTransfers, stageProviderFreshWorkspace } from "./provider-fresh-workspace.js"; import { randomUUID, createHash } from "node:crypto"; @@ -418,6 +419,7 @@ export async function preparePluginWorkspace( }, }) .returning(); + await notifyStatusCardIssueChange(tx, task.companyId); await tx .update(issues) .set({ diff --git a/server/src/services/provider-fresh-workspace.ts b/server/src/services/provider-fresh-workspace.ts index fcc184e15f4..98a478a375d 100644 --- a/server/src/services/provider-fresh-workspace.ts +++ b/server/src/services/provider-fresh-workspace.ts @@ -1,3 +1,4 @@ +import { notifyStatusCardIssueChange } from "./status-card-notifications.js"; import { assertTaskWorkspaceExecutionAccess } from "./task-workspace-access.js"; import { isDeepStrictEqual } from "node:util"; import { createHash, randomUUID } from "node:crypto"; @@ -152,6 +153,7 @@ export async function reserveFreshPluginTaskWorkspace(input: { providerType: "adapter_managed", cwd: `/workspaces/${input.fresh.attemptId}`, metadata: { runnerTaskFreshWorkspace: input.fresh, runnerTaskWorkspaceOwner: input.owner }, }); + await notifyStatusCardIssueChange(tx, input.companyId); await tx.update(issues).set({ executionWorkspaceId: input.fresh.attemptId, executionWorkspacePreference: "reuse_existing", workspaceBindingRevision: task.workspaceBindingRevision + 1, workspacePendingSelection: task.workspacePendingSelection?.expectedBindingRevision === task.workspaceBindingRevision diff --git a/server/src/services/routines.ts b/server/src/services/routines.ts index 23787b45249..a831a3e489a 100644 --- a/server/src/services/routines.ts +++ b/server/src/services/routines.ts @@ -1,3 +1,4 @@ +import { notifyStatusCardIssueChange } from "./status-card-notifications.js"; import { notifyApprovalWork } from "./approval-work-notifications.js"; import { notifyDecisionTargetIssueRemoval } from "./decision-targets.js"; import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js"; @@ -1949,6 +1950,7 @@ export function routineService( if (createdIssue) { await notifyApprovalWork(txDb, createdIssue.companyId); await notifyDecisionTargetIssueRemoval(txDb, createdIssue.id); + await notifyStatusCardIssueChange(txDb, createdIssue.companyId); await txDb.delete(issues).where(eq(issues.id, createdIssue.id)); } const failureReason = error instanceof Error ? error.message : String(error); diff --git a/server/src/services/slack-conversation-lifecycle.ts b/server/src/services/slack-conversation-lifecycle.ts index 0fd213dc084..9e784ff005d 100644 --- a/server/src/services/slack-conversation-lifecycle.ts +++ b/server/src/services/slack-conversation-lifecycle.ts @@ -1,3 +1,4 @@ +import { notifyStatusCardIssueChange } from "./status-card-notifications.js"; import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js"; import { notifyUnsafeWorkspaceExportRecovery } from "./unsafe-workspace-export-notifications.js"; import { and, asc, desc, eq, gt, inArray, isNull, sql } from "drizzle-orm"; @@ -133,6 +134,7 @@ export async function settleExternalConversation(db: Db, companyId: string, issu await notifyUnsafeWorkspaceExportRecovery(tx, { companyId, issueId }); await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors); await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers); + await notifyStatusCardIssueChange(tx, companyId); await tx.update(issues).set({ status: "in_review", updatedAt: new Date() }) .where(and(eq(issues.id, issueId), eq(issues.companyId, companyId))); activity = (await persistActivity(tx as unknown as Db, { diff --git a/server/src/services/slack-conversation-state.ts b/server/src/services/slack-conversation-state.ts index dedaa8d6676..08f6c9c038b 100644 --- a/server/src/services/slack-conversation-state.ts +++ b/server/src/services/slack-conversation-state.ts @@ -1,3 +1,4 @@ +import { notifyStatusCardIssueChange } from "./status-card-notifications.js"; import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js"; import { notifyExternalConversationSettlement } from "./external-conversation-settlement-notifications.js"; import { chatConversations, issues, type Db } from "@paperclipai/db"; @@ -49,6 +50,7 @@ export async function resumeExternalConversation(tx: Db, companyId: string, issu await notifyExternalConversationSettlement(tx, { companyId, issueId }); // Only the verified idle convention may leave In Review automatically. // A real decision or delivery problem keeps the existing review semantics. + await notifyStatusCardIssueChange(tx, companyId); await tx.update(issues).set({ status: "todo", updatedAt: new Date() }) .where(and(eq(issues.companyId, companyId), eq(issues.id, issueId), sql`${externalConversationStateSql()} = 'waiting'`)); diff --git a/server/src/services/status-card-evaluation.ts b/server/src/services/status-card-evaluation.ts new file mode 100644 index 00000000000..143df32344a --- /dev/null +++ b/server/src/services/status-card-evaluation.ts @@ -0,0 +1,90 @@ +import { and, asc, eq, isNotNull, isNull } from "drizzle-orm"; +import { companies, statusCards, type Db } from "@paperclipai/db"; +import { logger } from "../middleware/logger.js"; +import { registerDeadlineWork } from "./deadline-work.js"; +import type { createDeliveryWorkCoordinator } from "./delivery-work-coordinator.js"; +import { DELIVERY_QUEUES } from "./delivery-work-notifications.js"; +import { instanceSettingsService } from "./instance-settings.js"; +import { issueService } from "./issues.js"; +import { queueIssueAssignmentWakeup, type IssueAssignmentWakeupDeps } from "./issue-assignment-wakeup.js"; +import { statusCardService } from "./status-cards.js"; + +/** Feature-owned generation and assignment. Existing issue services retain + * execution authority, durable wake dedupe and stalled-generation cleanup. */ +export function statusCardEvaluationService(db: Db, heartbeat: IssueAssignmentWakeupDeps) { + const cards = statusCardService(db); + const issues = issueService(db); + const settings = instanceSettingsService(db); + let reconstructReactive = true; + let allChanges = false; + const changes = new Set(); + let enabled = false; + + return { + changed(hints?: ReadonlySet) { + for (const hint of hints ?? ["*"]) { + if (hint === "*" || hint === "changes:*") allChanges = true; + else if (hint.startsWith("changes:")) changes.add(hint.slice("changes:".length)); + } + }, + async run(signal: AbortSignal) { + enabled = (await settings.getExperimental()).enableStatusCards === true; + if (!enabled || signal.aborted) return; + const reconstruct = reconstructReactive; + const evaluateAllChanges = allChanges; + const companiesChanged = new Set(changes); + changes.clear(); + reconstructReactive = false; + allChanges = false; + try { + const result = await cards.tickDueStatusCards(new Date(), evaluateAllChanges ? "changed" : reconstruct && companiesChanged.size === 0 ? "all" : companiesChanged); + const assignments = await Promise.allSettled(result.enqueued.map(async ({ cardId, generatingIssue }) => { + // Once generated, dispatch or cancel even if shutdown begins: leaving + // an unassigned claimed card is not a completed sweep. + try { + await queueIssueAssignmentWakeup({ + heartbeat, issue: generatingIssue, + reason: "status_card_update_assigned", + mutation: "status_card.scheduler_update_requested", + contextSource: "status_card_scheduler", + requestedByActorType: "system", taskKey: `status-card:${cardId}`, + rethrowOnError: true, + }); + } catch (error) { + await issues.update(generatingIssue.id, { status: "cancelled" }); + throw error; + } + })); + const failures = assignments.filter(result => result.status === "rejected"); + if (failures.length) throw new AggregateError(failures.map(result => result.reason), "Status-card assignment failed"); + if (result.evaluated || result.enqueued.length) { + logger.info({ evaluated: result.evaluated, enqueued: result.enqueued.length }, "status-card evaluation complete"); + } + } catch (error) { + reconstructReactive ||= reconstruct; + allChanges ||= evaluateAllChanges; + for (const companyId of companiesChanged) changes.add(companyId); + throw error; + } + }, + async next() { + if (!enabled) return null; + const [next] = await db.select({ at: statusCards.nextEvalAt }).from(statusCards) + .innerJoin(companies, eq(companies.id, statusCards.companyId)) + .where(and(eq(companies.status, "active"), isNull(statusCards.archivedAt), + isNull(statusCards.generatingIssueId), isNotNull(statusCards.nextEvalAt))) + .orderBy(asc(statusCards.nextEvalAt)).limit(1); + return next?.at?.getTime() ?? null; + }, + }; +} + +export function registerStatusCardEvaluationWork( + coordinator: ReturnType, + service: ReturnType, + canRun: () => boolean, +) { + return registerDeadlineWork(coordinator, DELIVERY_QUEUES.statusCardEvaluation, { + run: service.run, next: service.next, onSettled: service.changed, canRun, retryMs: 30_000, + }); +} diff --git a/server/src/services/status-card-finalization.ts b/server/src/services/status-card-finalization.ts index 7c7d260992e..5d7ee62b523 100644 --- a/server/src/services/status-card-finalization.ts +++ b/server/src/services/status-card-finalization.ts @@ -1,3 +1,4 @@ +import { notifyStatusCardEvaluation } from "./status-card-notifications.js"; import { and, eq, isNull } from "drizzle-orm"; import type { Db } from "@paperclipai/db"; import { statusCards, statusCardUpdates } from "@paperclipai/db"; @@ -36,6 +37,7 @@ export async function finalizeStatusCardsForStalledGeneration( ) { if (!STALLED_GENERATION_STATUSES.has(issue.status)) return []; + await notifyStatusCardEvaluation(dbOrTx, issue.companyId); const now = new Date(); const failureReason = failureReasonForIssue(issue); const cards = await dbOrTx diff --git a/server/src/services/status-card-notifications.ts b/server/src/services/status-card-notifications.ts new file mode 100644 index 00000000000..6fc12985847 --- /dev/null +++ b/server/src/services/status-card-notifications.ts @@ -0,0 +1,10 @@ +import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js"; + +/** Card/settings changes reconstruct deadlines; issue changes also evaluate reactive scope. */ +export function notifyStatusCardEvaluation(transaction: object, companyId?: string) { + return notifyDeliveryWork(transaction, DELIVERY_QUEUES.statusCardEvaluation, companyId ? `cards:${companyId}` : "*"); +} + +export function notifyStatusCardIssueChange(transaction: object, companyId?: string) { + return notifyDeliveryWork(transaction, DELIVERY_QUEUES.statusCardEvaluation, companyId ? `changes:${companyId}` : "changes:*"); +} diff --git a/server/src/services/status-card-update-engine.ts b/server/src/services/status-card-update-engine.ts index ff14d35c640..1f734227808 100644 --- a/server/src/services/status-card-update-engine.ts +++ b/server/src/services/status-card-update-engine.ts @@ -128,10 +128,8 @@ export function isWithinStatusCardActiveHours(policy: StatusCardRefreshPolicy, n } export function nextStatusCardEvaluationAt(policy: StatusCardRefreshPolicy, now: Date) { - if (policy.mode === "manual") return null; - const seconds = policy.mode === "interval" - ? (policy.intervalMinutes ?? 15) * 60 - : Math.min(policy.debounceSeconds ?? 60, 60); + if (policy.mode !== "interval") return null; + const seconds = (policy.intervalMinutes ?? 15) * 60; return new Date(now.getTime() + seconds * 1000); } @@ -172,3 +170,37 @@ export function evaluateStatusCardPolicy(input: { } return { action: "run" as const }; } + +/** A retry exists only while filtered changes remain. Use the owning policy's + * actual eligibility boundary instead of repeatedly rediscovering a closed gate. */ +export function nextStatusCardPolicyEligibility(input: { + policy: StatusCardRefreshPolicy; + now: Date; + lastChangeAt: Date; + completedUpdateTimes: Date[]; + tokensToday: number; +}): Date | null { + if (input.policy.mode === "manual") return null; + let at = input.now.getTime(); + if (input.tokensToday >= (input.policy.dailyTokenCap ?? 100_000)) { + const day = new Date(input.now); + day.setUTCHours(24, 0, 0, 0); + at = Math.max(at, day.getTime()); + } + if (input.policy.mode === "reactive") { + at = Math.max(at, input.lastChangeAt.getTime() + (input.policy.debounceSeconds ?? 60) * 1000); + const cap = input.policy.maxUpdatesPerHour ?? 6; + const recent = input.completedUpdateTimes.map(time => time.getTime()) + .filter(time => time > at - 3_600_000).sort((a, b) => a - b); + if (recent.length >= cap) at = Math.max(at, recent[recent.length - cap]! + 3_600_000 + 1); + } + if (isWithinStatusCardActiveHours(input.policy, new Date(at))) return new Date(at); + // Active-hour precision is one local minute. Walking UTC minutes also honors + // skipped/repeated local times at daylight-saving transitions. + const end = at + 48 * 3_600_000; + for (at = Math.floor(at / 60_000) * 60_000 + 60_000; at <= end; at += 60_000) { + if (isWithinStatusCardActiveHours(input.policy, new Date(at))) return new Date(at); + } + // Equal start/end describes an empty window; retain changes, awaiting config. + return null; +} diff --git a/server/src/services/status-cards.ts b/server/src/services/status-cards.ts index 4a7439b6aa5..84bbf7b2cd4 100644 --- a/server/src/services/status-cards.ts +++ b/server/src/services/status-cards.ts @@ -1,7 +1,9 @@ +import { notifyStatusCardEvaluation } from "./status-card-notifications.js"; import { createHash } from "node:crypto"; import { and, desc, eq, gte, inArray, isNotNull, isNull, lte, ne, or, sql, type SQL } from "drizzle-orm"; import { agents, + companies, costEvents, documentRevisions, documents, @@ -23,7 +25,7 @@ import { conflict, forbidden, notFound, unprocessable } from "../errors.js"; import { logger } from "../middleware/logger.js"; import { readBuiltInAgentMarker } from "../lib/built-in-agent-metadata.js"; import { builtInAgentService } from "./built-in-agents.js"; -import { companySearchService } from "./company-search.js"; +import { companySearchService, companySearchUpdatedWithinMs } from "./company-search.js"; import { issueService } from "./issues.js"; import { SUMMARIZER_BUILT_IN_KEY } from "./summary-slots.js"; import { @@ -34,6 +36,7 @@ import { extractIssueMentions, filterStatusCardChanges, nextStatusCardEvaluationAt, + nextStatusCardPolicyEligibility, STATUS_CARD_MAX_MENTIONED_ISSUES, statusCardChangesHash, statusCardFingerprintHash, @@ -44,6 +47,7 @@ import { type StatusCardActor = { agentId: string | null; userId: string | null }; type StatusCardWriter = { agentId: string | null; runId: string | null }; type StatusCardRow = typeof statusCards.$inferSelect; +type WatchedIssue = CompanySearchIssueSummary & { scopeExpiresAt?: Date | null }; type StatusCardReadOptions = { issueReadCondition?: SQL }; const TERMINAL_ISSUE_STATUSES = new Set(["done", "cancelled"]); @@ -151,9 +155,15 @@ export function statusCardService( db: Db, deps: { issuesSvc?: ReturnType } = {}, ) { - const builtIns = builtInAgentService(db); - const issuesSvc = deps.issuesSvc ?? issueService(db); + const issuesSvc = () => deps.issuesSvc ?? issueService(db); const searchSvc = companySearchService(db); + type Transaction = Parameters[0]>[0]; + function mutate(companyId: string | undefined, write: (tx: Transaction) => Promise) { + return db.transaction(async tx => { + await notifyStatusCardEvaluation(tx, companyId); + return write(tx); + }); + } async function readWatchedIssueVisibility(card: StatusCardRow, options: StatusCardReadOptions = {}) { if (card.queries.length === 0 && (card.mentionedIssueIds?.length ?? 0) === 0) { @@ -240,10 +250,10 @@ export function statusCardService( }; const agentId = actor.agentId; if (!agentId) { - return db.insert(statusCards).values(values).returning().then((rows) => rows[0]!); + return mutate(companyId, tx => tx.insert(statusCards).values(values).returning().then((rows) => rows[0]!)); } - return db.transaction(async (tx) => { + return mutate(companyId, async (tx) => { const author = await tx .select({ id: agents.id }) .from(agents) @@ -292,7 +302,7 @@ export function statusCardService( ...(input.refreshPolicy !== undefined ? { refreshPolicy: input.refreshPolicy, - nextEvalAt: card.archivedAt ? null : nextStatusCardEvaluationAt(input.refreshPolicy, now), + nextEvalAt: card.archivedAt ? null : input.refreshPolicy.mode === "reactive" && card.queries.length > 0 ? now : nextStatusCardEvaluationAt(input.refreshPolicy, now), } : {}), ...(archiveChanged && input.archived @@ -309,21 +319,21 @@ export function statusCardService( } : {}), }; - const next = await db.update(statusCards).set({ + const next = await mutate(card.companyId, tx => tx.update(statusCards).set({ ...values, ...(archiveChanged && input.archived ? { generatingIssueId: null, pendingChangeHash: null } : {}), - }).where(eq(statusCards.id, card.id)).returning().then((rows) => rows[0]!); + }).where(eq(statusCards.id, card.id)).returning().then((rows) => rows[0]!)); if (archiveChanged && input.archived && card.generatingIssueId) { const generationIssue = await db.select().from(issues).where(eq(issues.id, card.generatingIssueId)).then((rows) => rows[0] ?? null); if (generationIssue && !TERMINAL_ISSUE_STATUSES.has(generationIssue.status)) { - await issuesSvc.update(generationIssue.id, { status: "cancelled" }); + await issuesSvc().update(generationIssue.id, { status: "cancelled" }); } } return next; } async function remove(id: string) { - return db.delete(statusCards).where(eq(statusCards.id, id)).returning().then((rows) => rows[0] ?? null); + return mutate(undefined, tx => tx.delete(statusCards).where(eq(statusCards.id, id)).returning().then((rows) => rows[0] ?? null)); } async function listUpdates(cardId: string) { @@ -360,7 +370,7 @@ export function statusCardService( .then((rows) => rows[0] ?? null); if (override) return override.id; } - const builtIn = await builtIns.get(card.companyId, SUMMARIZER_BUILT_IN_KEY); + const builtIn = await builtInAgentService(db).get(card.companyId, SUMMARIZER_BUILT_IN_KEY); if (builtIn.status !== "ready" || !builtIn.agentId) { throw unprocessable("Summarizer built-in agent is not configured", { code: "summarizer_not_configured", @@ -391,7 +401,7 @@ export function statusCardService( let deduplicated = false; const createdAt = new Date(); - const created = await issuesSvc.create(card.companyId, { + const created = await issuesSvc().create(card.companyId, { title: `Compile status card: ${card.title ?? card.interestPrompt.slice(0, 80)}`, description: compileDescription(card, null, hash), status: "todo", @@ -409,16 +419,16 @@ export function statusCardService( // covers idempotency-key hits that resolve to a terminal task (done/cancelled) // as well as a `blocked` one that a manual re-kick is reviving. const reopened = deduplicated && (TERMINAL_ISSUE_STATUSES.has(created.status) || created.status === "blocked") - ? await issuesSvc.update(created.id, { status: "todo", assigneeAgentId: summarizerAgentId }) + ? await issuesSvc().update(created.id, { status: "todo", assigneeAgentId: summarizerAgentId }) : created; - const generationIssue = await issuesSvc.update(reopened!.id, { + const generationIssue = await issuesSvc().update(reopened!.id, { description: compileDescription(card, reopened!.id, hash), }); - const [nextCard] = await db + const [nextCard] = await mutate(card.companyId, tx => tx .update(statusCards) .set({ generatingIssueId: generationIssue!.id, state: "compiling", failureReason: null, updatedAt: createdAt }) .where(eq(statusCards.id, card.id)) - .returning(); + .returning()); return { card: nextCard!, generatingIssue: generationIssue!, @@ -464,7 +474,7 @@ export function statusCardService( if (card.archivedAt) throw unprocessable("Archived status cards cannot accept generation writes"); await assertSummarizerWriter(card, input.generationIssueId, actor); const now = new Date(); - return db.transaction(async (tx) => { + return mutate(card.companyId, async (tx) => { const current = await tx.select().from(statusCards).where(eq(statusCards.id, card.id)).then((rows) => rows[0] ?? null); if (!current || current.archivedAt || current.generatingIssueId !== input.generationIssueId) { throw conflict("Status-card compilation was superseded by a newer task"); @@ -581,14 +591,21 @@ export function statusCardService( } async function executeQueries(card: StatusCardRow, options: StatusCardReadOptions = {}) { - const issueMap = new Map(); + const issueMap = new Map(); for (const storedQuery of card.queries) { const query = companySearchQuerySchema.parse(storedQuery); const response = await searchSvc.search(card.companyId, query, { issueReadCondition: options.issueReadCondition, }); for (const result of response.results) { - if (result.type === "issue" && result.issue) issueMap.set(result.issue.id, result.issue); + if (result.type !== "issue" || !result.issue) continue; + const duration = companySearchUpdatedWithinMs(query.updatedWithin); + const expiry = duration === null ? null : new Date(new Date(result.issue.updatedAt).getTime() + duration + 1); + const previous = issueMap.get(result.issue.id); + issueMap.set(result.issue.id, { ...result.issue, scopeExpiresAt: previous + ? previous.scopeExpiresAt == null || expiry === null ? null + : new Date(Math.max(previous.scopeExpiresAt.getTime(), expiry.getTime())) + : expiry }); } } // Issues mentioned in the latest summary join the watched set alongside the @@ -598,7 +615,11 @@ export function statusCardService( (card.mentionedIssueIds ?? []).filter((issueId) => !issueMap.has(issueId)), options, ); - for (const issue of mentioned) issueMap.set(issue.id, issue); + for (const issue of mentioned) issueMap.set(issue.id, { ...issue, scopeExpiresAt: null }); + for (const id of card.mentionedIssueIds ?? []) { + const issue = issueMap.get(id); + if (issue) issue.scopeExpiresAt = null; + } const snapshot = [...issueMap.values()]; if (snapshot.length === 0) return snapshot; const latestHumanComments = await db @@ -622,6 +643,13 @@ export function statusCardService( return snapshot.map((issue) => ({ ...issue, latestHumanCommentAt: commentByIssueId.get(issue.id) ?? null })); } + function nextEvaluationAt(card: StatusCardRow, snapshot: WatchedIssue[], now: Date) { + const interval = nextStatusCardEvaluationAt(card.refreshPolicy, now); + if (card.refreshPolicy.mode !== "reactive" || !card.refreshPolicy.triggers.membershipChanges) return interval; + const expiries = snapshot.flatMap(issue => issue.scopeExpiresAt && issue.scopeExpiresAt > now ? [issue.scopeExpiresAt.getTime()] : []); + return expiries.length ? new Date(Math.min(...expiries)) : null; + } + async function requestRefresh(cardId: string, input: { full?: boolean; trigger?: "manual" | "interval" | "reactive" | "restore"; @@ -648,15 +676,15 @@ export function statusCardService( const changes = filterStatusCardChanges(allChanges, card.refreshPolicy); const trigger = input.trigger ?? "manual"; const forceRun = trigger === "manual" || trigger === "restore"; - const nextEvalAt = nextStatusCardEvaluationAt(card.refreshPolicy, now); + const nextEvalAt = nextEvaluationAt(card, snapshot, now); if (!forceRun && changes.length === 0) { - const [next] = await db.update(statusCards).set({ + const [next] = await mutate(card.companyId, tx => tx.update(statusCards).set({ pendingChangeCount: 0, pendingChangeHash: null, lastChangeAt: null, state: "active", nextEvalAt, - }).where(eq(statusCards.id, card.id)).returning(); + }).where(eq(statusCards.id, card.id)).returning()); return { card: next!, generatingIssue: null, alreadyGenerating: false, enqueued: false }; } @@ -678,13 +706,17 @@ export function statusCardService( manual: forceRun, }); if (decision.action !== "run") { - const [next] = await db.update(statusCards).set({ + const [next] = await mutate(card.companyId, tx => tx.update(statusCards).set({ pendingChangeCount: changes.length, pendingChangeHash, lastChangeAt, state: decision.action === "pause_budget" ? "paused_budget" : decision.action === "pause_hours" ? "paused_hours" : "active", - nextEvalAt: decision.action === "wait" && "dueAt" in decision ? decision.dueAt : nextEvalAt, - }).where(eq(statusCards.id, card.id)).returning(); + nextEvalAt: nextStatusCardPolicyEligibility({ + policy: card.refreshPolicy, now, lastChangeAt, + completedUpdateTimes: recent.filter(row => row.kind !== "compile" && row.finishedAt).map(row => row.startedAt), + tokensToday: Number(daily[0]?.tokens ?? 0), + }), + }).where(eq(statusCards.id, card.id)).returning()); return { card: next!, generatingIssue: null, alreadyGenerating: false, enqueued: false }; } @@ -707,7 +739,7 @@ export function statusCardService( : null; const fingerprintHash = statusCardFingerprintHash(fingerprint); let deduplicated = false; - const created = await issuesSvc.create(card.companyId, { + const created = await issuesSvc().create(card.companyId, { title: `${kind === "full" ? "Rebuild" : "Update"} status card: ${card.title ?? card.interestPrompt.slice(0, 80)}`, description: updateDescription({ card, generationIssueId: null, fingerprint, changes, kind, trigger, previousSummary, snapshot }), status: "todo", @@ -720,15 +752,15 @@ export function statusCardService( onDeduplicated: (reason) => { deduplicated = reason === "idempotency_key"; }, }); const reopened = deduplicated && TERMINAL_ISSUE_STATUSES.has(created.status) - ? await issuesSvc.update(created.id, { status: "todo", assigneeAgentId: summarizerAgentId }) + ? await issuesSvc().update(created.id, { status: "todo", assigneeAgentId: summarizerAgentId }) : created; - const generationIssue = await issuesSvc.update(reopened!.id, { + const generationIssue = await issuesSvc().update(reopened!.id, { description: updateDescription({ card, generationIssueId: reopened!.id, fingerprint, changes, kind, trigger, previousSummary, snapshot }), }); const priorGenerationPredicate = card.generatingIssueId ? eq(statusCards.generatingIssueId, card.generatingIssueId) : isNull(statusCards.generatingIssueId); - const [next] = await db.update(statusCards).set({ + const [next] = await mutate(card.companyId, tx => tx.update(statusCards).set({ generatingIssueId: generationIssue!.id, pendingChangeCount: changes.length, pendingChangeHash, @@ -736,17 +768,17 @@ export function statusCardService( state: "active", nextEvalAt, failureReason: null, - }).where(and(eq(statusCards.id, card.id), isNull(statusCards.archivedAt), or(isNull(statusCards.generatingIssueId), priorGenerationPredicate))).returning(); + }).where(and(eq(statusCards.id, card.id), isNull(statusCards.archivedAt), or(isNull(statusCards.generatingIssueId), priorGenerationPredicate))).returning()); if (!next) { const winner = await getById(card.id); if (!winner?.generatingIssueId) { if (!TERMINAL_ISSUE_STATUSES.has(generationIssue!.status)) { - await issuesSvc.update(generationIssue!.id, { status: "cancelled" }); + await issuesSvc().update(generationIssue!.id, { status: "cancelled" }); } throw conflict("Status-card refresh claim was lost"); } if (generationIssue!.id !== winner.generatingIssueId && !TERMINAL_ISSUE_STATUSES.has(generationIssue!.status)) { - await issuesSvc.update(generationIssue!.id, { status: "cancelled" }); + await issuesSvc().update(generationIssue!.id, { status: "cancelled" }); } const winnerIssue = await db.select().from(issues).where(eq(issues.id, winner.generatingIssueId)).then((rows) => rows[0] ?? null); return { card: winner, generatingIssue: winnerIssue, alreadyGenerating: true, enqueued: false, kind, changes }; @@ -765,26 +797,44 @@ export function statusCardService( return { card: next, generatingIssue: generationIssue!, alreadyGenerating: deduplicated, enqueued: true, kind, changes }; } - async function tickDueStatusCards(now = new Date()) { - const due = await db.select().from(statusCards).where(and(isNull(statusCards.archivedAt), isNull(statusCards.generatingIssueId), isNotNull(statusCards.nextEvalAt), lte(statusCards.nextEvalAt, now))); + async function tickDueStatusCards(now = new Date(), reactiveCompanies?: ReadonlySet | "all" | "changed") { + const reactiveScope = reactiveCompanies === "all" ? isNull(statusCards.nextEvalAt) : reactiveCompanies === "changed" ? sql`true` : reactiveCompanies?.size + ? inArray(statusCards.companyId, [...reactiveCompanies]) : sql`false`; + const due = await db.select({ card: statusCards }).from(statusCards) + .innerJoin(companies, eq(companies.id, statusCards.companyId)) + .where(and(eq(companies.status, "active"), isNull(statusCards.archivedAt), isNull(statusCards.generatingIssueId), + or(and(isNotNull(statusCards.nextEvalAt), lte(statusCards.nextEvalAt, now)), + and(reactiveScope, ne(statusCards.state, "error"), ne(statusCards.state, "compiling"), sql`${statusCards.refreshPolicy}->>'mode' = 'reactive'`, sql`jsonb_array_length(${statusCards.queries}) > 0`)))) + .then(rows => rows.map(row => row.card)); const enqueued: Array<{ cardId: string; generatingIssue: typeof issues.$inferSelect }> = []; let evaluated = 0; for (const candidate of due) { - const claimUntil = new Date(now.getTime() + 5 * 60 * 1000); - const [claimed] = await db.update(statusCards).set({ nextEvalAt: claimUntil }) - .where(and(eq(statusCards.id, candidate.id), isNull(statusCards.generatingIssueId), lte(statusCards.nextEvalAt, now))) - .returning(); - if (!claimed) continue; - evaluated += 1; - try { - const result = await requestRefresh(claimed.id, { trigger: claimed.refreshPolicy.mode === "reactive" ? "reactive" : "interval", now }); - if (result.enqueued && result.generatingIssue) enqueued.push({ cardId: claimed.id, generatingIssue: result.generatingIssue }); - } catch (err) { - logger.warn( - { err, cardId: claimed.id, companyId: claimed.companyId }, - "status card scheduled refresh failed", + // nextEvalAt is both a retry lease and an evaluation deadline. A source + // event may preempt a genuine future deadline, so serialize scheduler + // evaluation across connections without holding a row lock while the + // existing issue/card services commit through their own transactions. + await db.transaction(async lockTx => { + const [lock] = await lockTx.execute<{ acquired: boolean }>( + sql`select pg_try_advisory_xact_lock(hashtextextended(${candidate.id}, 0)) as acquired`, ); - } + if (!lock?.acquired) return; + const claimUntil = new Date(now.getTime() + 5 * 60 * 1000); + const [claimed] = await mutate(candidate.companyId, tx => tx.update(statusCards).set({ nextEvalAt: claimUntil }) + .where(and(eq(statusCards.id, candidate.id), isNull(statusCards.archivedAt), isNull(statusCards.generatingIssueId), + candidate.nextEvalAt ? eq(statusCards.nextEvalAt, candidate.nextEvalAt) : isNull(statusCards.nextEvalAt))) + .returning()); + if (!claimed) return; + evaluated += 1; + try { + const result = await requestRefresh(claimed.id, { trigger: claimed.refreshPolicy.mode === "reactive" ? "reactive" : "interval", now }); + if (result.enqueued && result.generatingIssue) enqueued.push({ cardId: claimed.id, generatingIssue: result.generatingIssue }); + } catch (err) { + logger.warn( + { err, cardId: claimed.id, companyId: claimed.companyId }, + "status card scheduled refresh failed", + ); + } + }); } return { evaluated, enqueued }; } @@ -796,7 +846,7 @@ export function statusCardService( await assertSummarizerWriter(card, input.generationIssueId, actor); if (card.queries.length === 0) throw conflict("Compile the status-card query before writing its summary"); const now = new Date(); - return db.transaction(async (tx) => { + return mutate(card.companyId, async (tx) => { const current = await tx.select().from(statusCards).where(eq(statusCards.id, card.id)).then((rows) => rows[0] ?? null); if (!current || current.archivedAt || current.generatingIssueId !== input.generationIssueId) { throw conflict("Status-card generation was superseded by a newer task"); @@ -816,9 +866,8 @@ export function statusCardService( const mentionedIssueIds = await resolveMentionedIssueIds(current.companyId, input.markdown); // Current watched membership: compiled-query matches plus the issues this // summary mentions. - const watchedNow = buildStatusCardFingerprint( - await executeQueries({ ...current, mentionedIssueIds }), - ); + const watchedSnapshot = await executeQueries({ ...current, mentionedIssueIds }); + const watchedNow = buildStatusCardFingerprint(watchedSnapshot); let snapshot: StatusCardFingerprint; if (payloadFingerprint) { // Keep the generation-time fingerprint as the change baseline: issues @@ -889,7 +938,9 @@ export function statusCardService( pendingChangeCount: 0, pendingChangeHash: null, lastChangeAt: null, - nextEvalAt: nextStatusCardEvaluationAt(current.refreshPolicy, now), + nextEvalAt: current.refreshPolicy.mode === "reactive" && + filterStatusCardChanges(diffStatusCardFingerprint(snapshot, watchedNow), current.refreshPolicy).length > 0 + ? now : nextEvaluationAt(current, watchedSnapshot, now), updatedAt: now, }).where(and(eq(statusCards.id, current.id), eq(statusCards.generatingIssueId, input.generationIssueId))).returning(); if (!next) throw conflict("Status-card generation was superseded by a newer task"); From 46f2c0bb9a999b414497ba901b64a7c750c7fbc0 Mon Sep 17 00:00:00 2001 From: Dotta Date: Fri, 30 Oct 2026 17:25:24 -0500 Subject: [PATCH 30/74] fix(server): bound and scope routine projection recovery Co-Authored-By: Paperclip --- doc/plans/2026-10-30-routine-status-sleep.md | 13 +++-- .../__tests__/execution-control-work.test.ts | 4 +- ...tbeat-workspace-branch-containment.test.ts | 20 +++---- .../routine-execution-status-work.test.ts | 52 +++++++++++++++++++ .../routine-execution-status-worker.test.ts | 37 ++++++++++--- .../routine-status-reconciliation.test.ts | 17 +++++- .../server-startup-feedback-export.test.ts | 27 ++++++++++ .../execution-control-notifications.ts | 9 ++-- .../execution-control-reconciliation.ts | 2 +- .../services/execution-recovery-resolution.ts | 6 ++- .../services/explicit-native-continuation.ts | 2 +- server/src/services/heartbeat.ts | 2 +- server/src/services/heartbeat/queue.ts | 2 + .../src/services/heartbeat/run-lifecycle.ts | 4 +- .../services/heartbeat/runtime-execution.ts | 4 +- server/src/services/issue-recovery-actions.ts | 10 ++-- .../issue-review-state-notifications.ts | 2 +- server/src/services/issues.ts | 2 +- .../src/services/legacy-execution-recovery.ts | 2 +- .../native-failed-workspace-recovery.ts | 2 +- .../native-runtime/native-restart-recovery.ts | 2 +- .../native-runtime/native-run-finalizer.ts | 4 +- .../native-runtime/native-run-projection.ts | 2 +- .../native-runtime/native-safe-replacement.ts | 4 +- .../native-runtime/native-session-executor.ts | 6 +-- .../native-runtime/native-superseded-retry.ts | 2 +- .../native-workspace-reconstruction.test.ts | 8 ++- .../native-workspace-reconstruction.ts | 2 + .../routine-execution-status-notifications.ts | 7 ++- .../services/routine-execution-status-work.ts | 17 ++++-- server/src/services/routine-run-status.ts | 29 +++++------ server/src/services/routines.ts | 5 +- server/src/services/system-recovery.ts | 2 +- server/src/services/task-retry.ts | 2 +- 34 files changed, 227 insertions(+), 84 deletions(-) diff --git a/doc/plans/2026-10-30-routine-status-sleep.md b/doc/plans/2026-10-30-routine-status-sleep.md index ea6f9364999..271d0b544a7 100644 --- a/doc/plans/2026-10-30-routine-status-sleep.md +++ b/doc/plans/2026-10-30-routine-status-sleep.md @@ -7,14 +7,16 @@ scanned every five minutes, even with no routines or only completed history. ## Ownership and trigger `routine-run-status.ts` owns the projection, custody checks and repair transaction. -The worker loads actual mismatches at startup. It accepts issue hints from issue +The worker loads actual mismatches in keyset pages of 50 at startup. It accepts issue hints from issue creation, status/origin changes, checkout/release, routine issue association, execution control writes and review state writes. It uses the existing transaction-aware work notification protocol. An outer transaction owns the hint; a savepoint release does not wake the repair worker before commit. -A standard execution must name the same routine, invocation, company and linked -task. Plugin-managed executions may use their declared custom origin. Repair +An execution must retain reciprocal task/invocation links in the same company +and belong to a routine in that company. This is the existing routine status +owner contract. Origin labels can change when a plugin declaration changes or +is removed; retained reciprocal links preserve historical execution custody. Repair locks the routine and task and rechecks the linked invocation before writing. It preserves skipped/coalesced history and unrelated dispatch failures. A task that resumes clears an execution-task transient failure through the same owner @@ -33,8 +35,9 @@ history leave no recurring worker SQL. The existing Cloud and heartbeat schedule enablement gates remain in place. Ordinary routine scheduling is unchanged. Supported app writers register `notifyRoutineExecutionStatus(tx, issueId)` before -their mutation. Common execution writers without an issue hint request debt -reconstruction. Backups and independent SQL import/repair tools must restore +their mutation. Common execution writers without an issue hint request debt reconstruction only +for their recorded company. Projection debt settles even when that company is +paused; company activity does not create or cancel this obligation. Backups and independent SQL import/repair tools must restore before startup or call that same transaction contract; process-local hints do not observe independent SQL or another process. No permanent listener or periodic configured-routine scan is added. diff --git a/server/src/__tests__/execution-control-work.test.ts b/server/src/__tests__/execution-control-work.test.ts index 0046d8574aa..14cd9456eb5 100644 --- a/server/src/__tests__/execution-control-work.test.ts +++ b/server/src/__tests__/execution-control-work.test.ts @@ -383,11 +383,11 @@ describe.skipIf(!support.supported)("execution-control durable work", () => { const due = new Date(Date.now() + 120_000), earlier = new Date(Date.now() + 60_000); for (const at of [due, earlier]) { await writeExecutionControl(db, DELIVERY_QUEUES.executionFinalization, tx => tx.update(heartbeatRuns) - .set({ executionControlDeadlineAt: at }).where(eq(heartbeatRuns.id, run.id))); + .set({ executionControlDeadlineAt: at }).where(eq(heartbeatRuns.id, run.id)), { companyId }); await settle(); expect(c.nextWakeAt()).toBe(at.getTime()); } await writeExecutionControl(db, DELIVERY_QUEUES.executionFinalization, tx => tx.update(heartbeatRuns) - .set({ executionControlDeadlineAt: null }).where(eq(heartbeatRuns.id, run.id))); + .set({ executionControlDeadlineAt: null }).where(eq(heartbeatRuns.id, run.id)), { companyId }); await quiet(c); }); it("delivers a reconciled continuation when marking and resolving commit together", async () => { diff --git a/server/src/__tests__/heartbeat-workspace-branch-containment.test.ts b/server/src/__tests__/heartbeat-workspace-branch-containment.test.ts index 52d434a3078..484cf55e8ff 100644 --- a/server/src/__tests__/heartbeat-workspace-branch-containment.test.ts +++ b/server/src/__tests__/heartbeat-workspace-branch-containment.test.ts @@ -1,3 +1,5 @@ +import { subscribeDatabaseWork } from "@paperclipai/db"; +import { DELIVERY_QUEUES } from "../services/delivery-work-notifications.js"; import { EMBEDDED_POSTGRES_TEST_TIMEOUT_MS } from "@paperclipai/db"; import { execFile } from "node:child_process"; import { createHash, randomUUID } from "node:crypto"; @@ -888,7 +890,7 @@ describeEmbeddedPostgres("heartbeat workspace branch containment", () => { const tempRoots: string[] = []; beforeAll(async () => { - if (externalTestDatabaseUrl) db = createDb(externalTestDatabaseUrl); + if (externalTestDatabaseUrl) db = createDb(externalTestDatabaseUrl, { maxConnections: 6 }); else { tempDb = await startEmbeddedPostgresTestDatabase("paperclip-branch-containment-"); db = createDb(tempDb.connectionString); @@ -1173,15 +1175,15 @@ describeEmbeddedPostgres("heartbeat workspace branch containment", () => { }, }); + const routineEvents: string[] = []; + const unsubscribeRoutine = subscribeDatabaseWork(db, DELIVERY_QUEUES.routineExecutionStatus, event => routineEvents.push(event)); const heartbeat = heartbeatService(db); - const run = await heartbeat.wakeup(agentId, { - source: "assignment", - triggerDetail: "system", - reason: "issue_assigned", - payload: { issueId }, - contextSnapshot: { issueId, wakeReason: "issue_assigned" }, - }); - + let run; + try { + run = await heartbeat.wakeup(agentId, { source: "assignment", triggerDetail: "system", reason: "issue_assigned", + payload: { issueId }, contextSnapshot: { issueId, wakeReason: "issue_assigned" } }); + } finally { unsubscribeRoutine(); } + expect(routineEvents).toContain("intent"); expect(routineEvents).toContain("settled"); expect(run).toBeNull(); expect(adapterExecute).not.toHaveBeenCalled(); diff --git a/server/src/__tests__/routine-execution-status-work.test.ts b/server/src/__tests__/routine-execution-status-work.test.ts index 7415b63bfd5..53d7e5c0225 100644 --- a/server/src/__tests__/routine-execution-status-work.test.ts +++ b/server/src/__tests__/routine-execution-status-work.test.ts @@ -1,3 +1,5 @@ +import { writeExecutionControl } from "../services/execution-control-notifications.js"; +import { DELIVERY_QUEUES } from "../services/delivery-work-notifications.js"; import { randomUUID } from "node:crypto"; import { eq, sql } from "drizzle-orm"; import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; @@ -122,6 +124,56 @@ const support = external ? { supported: true } : await getEmbeddedPostgresTestSu await settled(); await quiet(); expect(await saved(run.id)).toEqual(before); expect((await saved(otherRun.id)).status).toBe("issue_created"); expect(errors).toEqual([]); }); + it.each(["changed", "removed"])("keeps historical custom-origin custody after its plugin declaration is %s", async mode => { + const f = await fixture(), pluginKey = `routine-history-${randomUUID()}`, originId = `operation:${randomUUID()}`; + const [plugin] = await db.insert(plugins).values({ pluginKey, packageName: pluginKey, version: "0.1.0", + manifestJson: { id: pluginKey, apiVersion: 1, version: "0.1.0", displayName: "Routine test", description: "Routine test", + author: "Paperclip", categories: ["automation"], capabilities: ["routines.managed"], entrypoints: { worker: "./worker.js" } } }).returning(); + const [resource] = await db.insert(pluginManagedResources).values({ companyId: f.companyId, pluginId: plugin.id, pluginKey, + resourceKind: "routine", resourceKey: "history", resourceId: f.routine.id, defaultsJson: { issueTemplate: { originId } } }).returning(); + const run = await dispatch(f); + expect((await db.select().from(issues).where(eq(issues.id, run.linkedIssueId!)))[0].originId).toBe(originId); + if (mode === "changed") await db.update(pluginManagedResources).set({ defaultsJson: { issueTemplate: { originId: "new-origin" } } }).where(eq(pluginManagedResources.id, resource.id)); + else await db.delete(pluginManagedResources).where(eq(pluginManagedResources.id, resource.id)); + await db.update(issues).set({ status: "done" }).where(eq(issues.id, run.linkedIssueId!)); + await start(); await projected(run.id, "completed"); await quiet(); expect(errors).toEqual([]); + }); + it("reconstructs more than one page and does not lose a committed issue hint behind the startup cursor", async () => { + const f = await fixture(), ids = Array.from({ length: 53 }, () => randomUUID()).sort(); + const issueIds = ids.map(() => randomUUID()); + await db.insert(issues).values(ids.map((id, index) => ({ id: issueIds[index], companyId: f.companyId, title: "Legacy execution", + status: index === 0 ? "todo" : "done", originKind: "routine_execution", originRunId: id, originId: f.routine.id }))); + await db.insert(routineRuns).values(ids.map((id, index) => ({ id, companyId: f.companyId, routineId: f.routine.id, + source: "schedule", linkedIssueId: issueIds[index], status: "issue_created" }))); + const service = routineExecutionStatusService(db), load = service.load; + let entered!: () => void, release!: () => void; + const enteredGate = new Promise(resolve => { entered = resolve; }); + const releaseGate = new Promise(resolve => { release = resolve; }); + const loaded = vi.spyOn(service, "load").mockImplementationOnce(async (scope, afterId) => { + const page = await load(scope, afterId); expect(page).toHaveLength(50); entered(); await releaseGate; return page; + }); + const starting = start(service); await enteredGate; + await issueService(db).update(issueIds[0]!, { status: "done" }); release(); await starting; + await vi.advanceTimersByTimeAsync(3000); + await vi.waitFor(async () => expect((await db.select().from(routineRuns)).every(row => row.status === "completed")).toBe(true)); + expect(loaded.mock.calls.some(([scope, afterId]) => scope === undefined && afterId !== undefined)).toBe(true); + expect(loaded.mock.calls.some(([scope]) => scope?.issueId === issueIds[0])).toBe(true); + await quiet(); expect(errors).toEqual([]); + }); + it("scopes shared execution writes to their company and settles paused-company projection debt", async () => { + const f = await fixture(), other = await fixture(), run = await dispatch(f), otherRun = await dispatch(other); + const service = routineExecutionStatusService(db), load = vi.spyOn(service, "load"); + await start(service); await quiet(); load.mockClear(); + await db.update(companies).set({ status: "paused" }).where(eq(companies.id, f.companyId)); + await db.update(issues).set({ status: "done" }).where(eq(issues.id, otherRun.linkedIssueId!)); + await writeExecutionControl(db, DELIVERY_QUEUES.executionStatus, + tx => tx.update(issues).set({ status: "done" }).where(eq(issues.id, run.linkedIssueId!)), { companyId: f.companyId }); + await projected(run.id, "completed"); + expect(load.mock.calls.every(([scope]) => scope?.companyId === f.companyId)).toBe(true); + expect((await saved(otherRun.id)).status).toBe("issue_created"); await quiet(); + await db.transaction(async tx => { await notifyRoutineExecutionStatus(tx, otherRun.linkedIssueId!); }); + await projected(otherRun.id, "completed"); await quiet(); expect(errors).toEqual([]); + }); it("waits for the real outer transaction and forgets a rolled-back nested status write", async () => { const f = await fixture(), run = await dispatch(f); await start(); await quiet(); let release!: () => void, entered!: () => void; diff --git a/server/src/__tests__/routine-execution-status-worker.test.ts b/server/src/__tests__/routine-execution-status-worker.test.ts index b88babbbd98..b62245ce3e2 100644 --- a/server/src/__tests__/routine-execution-status-worker.test.ts +++ b/server/src/__tests__/routine-execution-status-worker.test.ts @@ -1,3 +1,4 @@ +import type { RoutineExecutionStatusScope } from "../services/routine-execution-status-notifications.js"; import { installDatabaseWorkSignals } from "../../../packages/db/src/work-signals.js"; import { afterEach, beforeEach, expect, it, vi } from "vitest"; import { createDeliveryWorkCoordinator } from "../services/delivery-work-coordinator.js"; @@ -18,19 +19,19 @@ afterEach(async () => { await coordinator.stop(); vi.useRealTimers(); }); it("loads actual debt once and leaves no recurring owner work", async () => { const service = { load: vi.fn(async () => []), settle: vi.fn(async () => true) }; await registerRoutineExecutionStatusWork(coordinator, service, () => canRun).ready; - expect(service.load).toHaveBeenCalledExactlyOnceWith(undefined); + expect(service.load).toHaveBeenCalledExactlyOnceWith(undefined, undefined); expect(coordinator.nextWakeAt()).toBeNull(); await vi.advanceTimersByTimeAsync(20 * 60_000); expect(service.load).toHaveBeenCalledOnce(); expect(service.settle).not.toHaveBeenCalled(); expect(errors).toEqual([]); }); it("reconstructs only an invalidated issue and coalesces duplicate requests", async () => { - const service = { load: vi.fn(async (_issueId?: string) => [] as typeof entry[]), settle: vi.fn(async () => true) }; + const service = { load: vi.fn(async (_scope?: RoutineExecutionStatusScope, _afterId?: string) => [] as typeof entry[]), settle: vi.fn(async () => true) }; const worker = registerRoutineExecutionStatusWork(coordinator, service, () => canRun); await worker.ready; service.load.mockResolvedValueOnce([entry]); - worker.invalidate("issue"); worker.invalidate("issue"); + worker.invalidate(JSON.stringify({ issueId: "issue" })); worker.invalidate(JSON.stringify({ issueId: "issue" })); await vi.advanceTimersByTimeAsync(1000); - expect(service.load).toHaveBeenLastCalledWith("issue"); expect(service.load).toHaveBeenCalledTimes(2); + expect(service.load).toHaveBeenLastCalledWith({ issueId: "issue" }, undefined); expect(service.load).toHaveBeenCalledTimes(2); expect(service.settle).toHaveBeenCalledExactlyOnceWith(expect.objectContaining(entry)); expect(coordinator.nextWakeAt()).toBeNull(); expect(errors).toEqual([]); }); @@ -50,7 +51,7 @@ it("retains startup and first work during drain without passive owner calls", as canRun = false; const service = { load: vi.fn(async () => [entry]), settle: vi.fn(async () => true) }; const worker = registerRoutineExecutionStatusWork(coordinator, service, () => canRun); await worker.ready; - worker.invalidate("issue"); + worker.invalidate(JSON.stringify({ issueId: "issue" })); await vi.advanceTimersByTimeAsync(20 * 60_000); expect(service.load).not.toHaveBeenCalled(); expect(service.settle).not.toHaveBeenCalled(); canRun = true; await vi.advanceTimersByTimeAsync(30_000); @@ -69,11 +70,11 @@ it("keeps a failed owner across drain and retries after resume", async () => { it("retains a notification that arrives while the startup reconstruction is in flight", async () => { let release!: () => void; const gate = new Promise(resolve => { release = resolve; }); - const service = { load: vi.fn(async (_issueId?: string) => [] as typeof entry[]), settle: vi.fn(async () => true) }; + const service = { load: vi.fn(async (_scope?: RoutineExecutionStatusScope, _afterId?: string) => [] as typeof entry[]), settle: vi.fn(async () => true) }; service.load.mockImplementationOnce(async () => { await gate; return []; }); const worker = registerRoutineExecutionStatusWork(coordinator, service, () => canRun); await vi.waitFor(() => expect(service.load).toHaveBeenCalledOnce()); - worker.invalidate("issue"); service.load.mockResolvedValueOnce([entry]); release(); await worker.ready; + worker.invalidate(JSON.stringify({ issueId: "issue" })); service.load.mockResolvedValueOnce([entry]); release(); await worker.ready; await vi.advanceTimersByTimeAsync(1000); expect(service.settle).toHaveBeenCalledOnce(); expect(coordinator.nextWakeAt()).toBeNull(); expect(errors).toEqual([]); }); @@ -81,6 +82,26 @@ it("retains a notification that arrives while the startup reconstruction is in f it("cancels retries and subscriptions at shutdown", async () => { const service = { load: vi.fn(async () => [entry]), settle: vi.fn(async () => true).mockRejectedValue(new Error("offline")) }; const worker = registerRoutineExecutionStatusWork(coordinator, service, () => canRun); await worker.ready; - await worker.stop(); worker.invalidate("issue"); await vi.advanceTimersByTimeAsync(20 * 60_000); + await worker.stop(); worker.invalidate(JSON.stringify({ issueId: "issue" })); await vi.advanceTimersByTimeAsync(20 * 60_000); expect(service.settle).toHaveBeenCalledOnce(); expect(coordinator.nextWakeAt()).toBeNull(); }); + + +it("reconstructs a backlog in bounded keyset pages while retaining a scoped hint behind the cursor", async () => { + const firstPage = Array.from({ length: 50 }, (_, i) => ({ ...entry, id: `run-${String(i).padStart(3, "0")}` })); + const late = { ...entry, id: "run-late", issueId: "late-issue" }; + const service = { + load: vi.fn(async (scope?: RoutineExecutionStatusScope, afterId?: string) => scope?.issueId ? [late] + : afterId ? [{ ...entry, id: "run-050" }] : firstPage), + settle: vi.fn(async () => true), + }; + const worker = registerRoutineExecutionStatusWork(coordinator, service, () => canRun); await worker.ready; + expect(service.load).toHaveBeenCalledExactlyOnceWith(undefined, undefined); + worker.invalidate(JSON.stringify({ issueId: "late-issue" })); + await vi.advanceTimersByTimeAsync(3000); + expect(service.load).toHaveBeenCalledWith(undefined, "run-049"); + expect(service.load).toHaveBeenCalledWith({ issueId: "late-issue" }, undefined); + expect(service.settle).toHaveBeenCalledTimes(52); + expect(service.settle).toHaveBeenCalledWith(expect.objectContaining(late)); + expect(coordinator.nextWakeAt()).toBeNull(); expect(errors).toEqual([]); +}); diff --git a/server/src/__tests__/routine-status-reconciliation.test.ts b/server/src/__tests__/routine-status-reconciliation.test.ts index 964e43d6035..343c6fb38b6 100644 --- a/server/src/__tests__/routine-status-reconciliation.test.ts +++ b/server/src/__tests__/routine-status-reconciliation.test.ts @@ -38,11 +38,10 @@ if (!support.supported) console.warn(`Skipping routine status database tests: ${ expect((await db.select().from(routineRuns))[0]).toMatchObject({ status: "issue_created", completedAt: null }); } }); - it.each(["active_issue", "wrong_origin", "wrong_routine", "wrong_run", "wrong_company", "wrong_routine_company", "different_status"])("does not settle an unsupported handoff: %s", async mode => { + it.each(["active_issue", "wrong_origin", "wrong_run", "wrong_company", "wrong_routine_company", "different_status"])("does not settle an unsupported handoff: %s", async mode => { const f = await seed(); if (mode === "active_issue") await db.update(issues).set({ status: "in_progress" }).where(eq(issues.id, f.issue.id)); if (mode === "wrong_origin") await db.update(issues).set({ originKind: "manual" }).where(eq(issues.id, f.issue.id)); - if (mode === "wrong_routine") await db.update(issues).set({ originId: randomUUID() }).where(eq(issues.id, f.issue.id)); if (mode === "wrong_run") await db.update(issues).set({ originRunId: randomUUID() }).where(eq(issues.id, f.issue.id)); if (mode === "wrong_company") { const other = await seed("done"); await db.update(routineRuns).set({ companyId: other.companyId }).where(eq(routineRuns.id, f.run.id)); } if (mode === "wrong_routine_company") { const other = await seed("done"); @@ -55,6 +54,20 @@ if (!support.supported) console.warn(`Skipping routine status database tests: ${ expect((await db.select().from(routineRuns)).find(row => row.id === f.run.id)).toEqual(before.find(row => row.id === f.run.id)); if (mode === "wrong_routine_company") await expect(routineService(db).syncRunStatusForIssue(f.issue.id)).resolves.toBeNull(); }); + it("uses reciprocal durable custody when a retained origin no longer names the current routine declaration", async () => { + const f = await seed("done"); + await db.update(issues).set({ originId: "previous-declaration-origin" }).where(eq(issues.id, f.issue.id)); + expect(await routineService(db).reconcileExecutionRunStatuses()).toEqual({ scanned: 1, settled: 1, deferred: 0 }); + expect((await db.select().from(routineRuns).where(eq(routineRuns.id, f.run.id)))[0].status).toBe("completed"); + }); + it.each(["unknown-status", "other-code"])("preserves a failed invocation with an unrelated transient marker: %s", async mode => { + const f = await seed("in_progress"); + await db.update(routineRuns).set({ status: "failed", failureReason: null, + triggerPayload: { transientFailure: { code: mode === "other-code" ? "unrelated_failure" : "execution_issue_status", + status: mode === "unknown-status" ? "unrecognized" : "blocked" } } }).where(eq(routineRuns.id, f.run.id)); + expect(await routineService(db).reconcileExecutionRunStatuses()).toEqual({ scanned: 0, settled: 0, deferred: 0 }); + expect((await db.select().from(routineRuns).where(eq(routineRuns.id, f.run.id)))[0].status).toBe("failed"); + }); it.each(["resume", "retarget", "routine_transfer"])("rechecks %s after candidate selection and before repair", async mode => { const f = await seed(); const transaction = db.transaction.bind(db); vi.spyOn(db, "transaction").mockImplementationOnce(async callback => { diff --git a/server/src/__tests__/server-startup-feedback-export.test.ts b/server/src/__tests__/server-startup-feedback-export.test.ts index bbb020c865d..caf5565762a 100644 --- a/server/src/__tests__/server-startup-feedback-export.test.ts +++ b/server/src/__tests__/server-startup-feedback-export.test.ts @@ -1,3 +1,4 @@ +import { routineExecutionStatusService } from "../services/routine-run-status.js"; import { nativeHarnessBackupExpirationService } from "../services/native-runtime/native-harness-backup-expiration.js"; import { skillVersionRetentionService } from "../services/skill-version-retention.js"; import { approvalQueueSeedingService } from "../services/approval-queue-seeding-work.js"; @@ -392,6 +393,11 @@ vi.mock("../services/approval-retention.js", async importOriginal => ({ approvalRetentionService: vi.fn(() => ({ sweep: vi.fn(async () => ({ archived: 0 })), next: vi.fn(async () => null) })), })); +vi.mock("../services/routine-run-status.js", async importOriginal => ({ + ...await importOriginal(), + routineExecutionStatusService: vi.fn(() => ({ load: vi.fn(async () => []), settle: vi.fn(async () => false) })), +})); + vi.mock("../services/index.js", () => ({ backfillLegacyToolOAuthTokens: vi.fn(async () => ({ scannedConnections: 0, @@ -1366,6 +1372,27 @@ describe("startServer feedback export wiring", () => { } }); + it.each([{ cloud: false, enabled: true }, { cloud: true, enabled: false }, { cloud: true, enabled: true }])( + "registers routine status only for enabled Cloud scheduling ($cloud/$enabled)", async ({ cloud, enabled }) => { + if (cloud) vi.stubEnv("PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN", "startup-test-token"); + else vi.stubEnv("PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN", undefined); + loadConfigMock.mockReturnValue(buildTestConfig({ heartbeatSchedulerEnabled: enabled })); + const intervalCallbacks: Array<() => void> = []; + const intervals = vi.spyOn(globalThis, "setInterval").mockImplementation(((callback: () => void) => { + intervalCallbacks.push(callback); return 1 as unknown as ReturnType; + }) as typeof setInterval); + try { + await startServer(); await settleStartupHousekeeping(); + expect(deliveryWorkRegisterMock.mock.calls.filter(([queue]) => queue === DELIVERY_QUEUES.routineExecutionStatus)).toHaveLength(cloud && enabled ? 1 : 0); + const service = vi.mocked(routineExecutionStatusService).mock.results.at(-1)?.value; + if (cloud && enabled) expect(service?.load).toHaveBeenCalledExactlyOnceWith(undefined, undefined); + for (let index = 0; index < 4; index++) for (const callback of intervalCallbacks) callback(); + await settleStartupHousekeeping(); + if (cloud && enabled) expect(service?.load).toHaveBeenCalledOnce(); + } finally { intervals.mockRestore(); vi.unstubAllEnvs(); } + }, + ); + it("keeps routine ticks and setup cleanup active when heartbeat scheduling is suppressed", async () => { loadConfigMock.mockReturnValue(buildTestConfig({ heartbeatSchedulerEnabled: true, diff --git a/server/src/services/execution-control-notifications.ts b/server/src/services/execution-control-notifications.ts index 33a99774a48..fcbb892578c 100644 --- a/server/src/services/execution-control-notifications.ts +++ b/server/src/services/execution-control-notifications.ts @@ -2,8 +2,10 @@ import { notifyRoutineExecutionStatus } from "./routine-execution-status-notific import type { Db } from "@paperclipai/db"; import { DELIVERY_QUEUES, notifyDeliveryWork, type DeliveryQueue } from "./delivery-work-notifications.js"; -export async function notifyExecutionRunWork(transaction: object, issueId?: string) { - await notifyRoutineExecutionStatus(transaction, issueId); +export type ExecutionWorkScope = { companyId: string; agentId?: string; runId?: string; issueId?: string }; + +export async function notifyExecutionRunWork(transaction: object, scope?: ExecutionWorkScope) { + if (scope) await notifyRoutineExecutionStatus(transaction, scope.issueId, scope.companyId); for (const queue of [DELIVERY_QUEUES.executionStatus, DELIVERY_QUEUES.executionFinalization, DELIVERY_QUEUES.executionReplacement, DELIVERY_QUEUES.executionDisposition, DELIVERY_QUEUES.issueMonitors]) { await notifyDeliveryWork(transaction, queue); @@ -14,9 +16,10 @@ export function writeExecutionControl( db: Db | Parameters[0]>[0], queue: DeliveryQueue | readonly DeliveryQueue[], write: (tx: Db) => PromiseLike, + scope: ExecutionWorkScope | undefined, ): Promise { return db.transaction(async transaction => { - await notifyRoutineExecutionStatus(transaction); + if (scope) await notifyRoutineExecutionStatus(transaction, scope.issueId, scope.companyId); await notifyDeliveryWork(transaction, DELIVERY_QUEUES.issueMonitors); for (const topic of typeof queue === "string" ? [queue] : queue) await notifyDeliveryWork(transaction, topic); return write(transaction as unknown as Db); diff --git a/server/src/services/execution-control-reconciliation.ts b/server/src/services/execution-control-reconciliation.ts index 2f2a435982d..621fd210cfc 100644 --- a/server/src/services/execution-control-reconciliation.ts +++ b/server/src/services/execution-control-reconciliation.ts @@ -196,7 +196,7 @@ export async function reconcileAbandonedExecutionControl( await notifyNativeEnvironmentSettlement(tx, { companyId: run.companyId, runId: run.id }); await notifyCancelledWorkspacePreservation(tx, { companyId: run.companyId, runId: run.id }); await notifyUnsafeWorkspaceExportRunChange(tx, run); - await notifyExecutionRunWork(tx); + await notifyExecutionRunWork(tx, { companyId: run.companyId }); const [updatedRun] = await tx .update(heartbeatRuns) .set({ diff --git a/server/src/services/execution-recovery-resolution.ts b/server/src/services/execution-recovery-resolution.ts index f3f5a3e4310..a6f80f26ba0 100644 --- a/server/src/services/execution-recovery-resolution.ts +++ b/server/src/services/execution-recovery-resolution.ts @@ -1,3 +1,4 @@ +import { notifyRoutineExecutionStatus } from "./routine-execution-status-notifications.js"; import { notifyNativeStatusReassessment } from "./native-status-reassessment-notifications.js"; import { notifyUnsafeWorkspaceExportRecovery } from "./unsafe-workspace-export-notifications.js"; import { notifyChatResponseAvailability } from "./chat-response-availability.js"; @@ -546,7 +547,7 @@ export async function settleUnrecoverableExecutions( companyId: run.companyId, issueId: task.id, runId: run.id, }).limit(1).for("update"); if (!hold) return; - await notifyExecutionRunWork(tx); + await notifyExecutionRunWork(tx, { companyId: run.companyId }); const note = "Exact session recovery stopped. The recorded workspace recovery holds this execution without replaying its actions."; await tx.update(issueRecoveryActions).set({ status: "cancelled", outcome: "cancelled", resolvedAt: now, updatedAt: now, wakePolicy: null, monitorPolicy: null, @@ -596,6 +597,7 @@ export async function settleUnrecoverableExecutions( if (task.status !== "blocked") await notifyChatResponseAvailability(tx, { companyId: task.companyId, issueId: task.id }); await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors); await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers); + await notifyRoutineExecutionStatus(tx, task.id, task.companyId); const [projected] = await tx .update(issues) .set({ @@ -654,7 +656,7 @@ export async function settleUnrecoverableExecutions( replay: "not_authorized", }, }); - await notifyExecutionRunWork(tx); + await notifyExecutionRunWork(tx, { companyId: run.companyId }); await tx .update(heartbeatRuns) .set({ executionStatusDeliveryId: randomUUID() }) diff --git a/server/src/services/explicit-native-continuation.ts b/server/src/services/explicit-native-continuation.ts index 074ffc3bdf5..084a25e457f 100644 --- a/server/src/services/explicit-native-continuation.ts +++ b/server/src/services/explicit-native-continuation.ts @@ -475,7 +475,7 @@ export async function admitExplicitNativeContinuation(input: { }).where(and(eq(agentWakeupRequests.companyId, companyId), eq(agentWakeupRequests.id, input.queuedCommentInterruptId), eq(agentWakeupRequests.agentId, agentId), eq(agentWakeupRequests.status, "deferred_issue_execution"))); if (cancelledStartupIds.size) await db.transaction(async tx => { - await notifyExecutionRunWork(tx); + await notifyExecutionRunWork(tx, { companyId, agentId }); for (const runId of cancelledStartupIds) { await tx.update(nativeRunFinalizations).set({ phase: "terminal_failure", failureCode: "native_startup_cancelled", nextAttemptAt: null, diff --git a/server/src/services/heartbeat.ts b/server/src/services/heartbeat.ts index eb655fdef25..a4cec7b81b0 100644 --- a/server/src/services/heartbeat.ts +++ b/server/src/services/heartbeat.ts @@ -3724,7 +3724,7 @@ export function heartbeatService( await writeExecutionControl(db, DELIVERY_QUEUES.executionFinalization, tx => tx .update(heartbeatRuns) .set({ executionControlDeadlineAt: null }) - .where(eq(heartbeatRuns.id, run.id))); + .where(eq(heartbeatRuns.id, run.id)), { companyId: run.companyId }); } nativeOwnershipHeld = nativeOwnershipHeld || diff --git a/server/src/services/heartbeat/queue.ts b/server/src/services/heartbeat/queue.ts index a41b486b647..d2786778cf7 100644 --- a/server/src/services/heartbeat/queue.ts +++ b/server/src/services/heartbeat/queue.ts @@ -1,3 +1,4 @@ +import { notifyRoutineExecutionStatus } from "../routine-execution-status-notifications.js"; import { DELIVERY_QUEUES, notifyDeliveryWork } from "../delivery-work-notifications.js"; import { notifyExternalAgentSettlement } from "../../modules/external-agents/index.js"; import { notifyNativeStatusReassessment } from "../native-status-reassessment-notifications.js"; @@ -3919,6 +3920,7 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors); await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers); if (issue.status !== "blocked") await notifyChatResponseAvailability(tx, { companyId: issue.companyId, issueId: issue.id }); + await notifyRoutineExecutionStatus(tx, issue.id, issue.companyId); await tx .update(issues) .set({ diff --git a/server/src/services/heartbeat/run-lifecycle.ts b/server/src/services/heartbeat/run-lifecycle.ts index f796da03a5e..ac4e800f279 100644 --- a/server/src/services/heartbeat/run-lifecycle.ts +++ b/server/src/services/heartbeat/run-lifecycle.ts @@ -1044,8 +1044,8 @@ export function createHeartbeatLifecycle(db: Db, dependencies: HeartbeatLifecycl if (previous && isHeartbeatRunTerminalStatus(status)) await notifyRetainedNativeSessionCleanup(tx, { companyId: previous.companyId, runId: previous.id }); if (isHeartbeatRunTerminalStatus(status)) await notifyDeliveryWork(tx, DELIVERY_QUEUES.accounting, companyId); - await notifyExecutionRunWork(tx, previous?.nativeIssueId ?? previous?.issueId - ?? readNonEmptyString(parseObject(previous?.contextSnapshot).issueId) ?? undefined); + await notifyExecutionRunWork(tx, previous ? { companyId: previous.companyId, agentId: previous.agentId, runId: previous.id, + issueId: previous.nativeIssueId ?? previous.issueId ?? readNonEmptyString(parseObject(previous.contextSnapshot).issueId) ?? undefined } : undefined); if (previous) await notifyChatRunMilestones(tx, { companyId: previous.companyId, runId: previous.id, issueId: previous.nativeIssueId ?? previous.issueId ?? readNonEmptyString(parseObject(previous.contextSnapshot).issueId) ?? undefined }); return write(tx); diff --git a/server/src/services/heartbeat/runtime-execution.ts b/server/src/services/heartbeat/runtime-execution.ts index b5b64abc948..bdbbfa07752 100644 --- a/server/src/services/heartbeat/runtime-execution.ts +++ b/server/src/services/heartbeat/runtime-execution.ts @@ -966,7 +966,7 @@ export async function executeHeartbeatRuntime(db: Db, input: HeartbeatRuntimeExe eq(heartbeatRuns.id, run.id), eq(heartbeatRuns.status, "running"), ), - )); + ), { companyId: run.companyId }); const workspaceFinalizeStatus = hasWorkspaceRestoreFailure(adapterResult.resultJson) ? "failed" : "succeeded"; await recordWorkspaceFinalize(workspaceFinalizeStatus); if (adapterResult.nativeFinalization) { @@ -1042,7 +1042,7 @@ export async function executeHeartbeatRuntime(db: Db, input: HeartbeatRuntimeExe eq(heartbeatRuns.id, run.id), eq(heartbeatRuns.status, "running"), ), - )); + ), { companyId: run.companyId }); if ( issueRef && context.resumeSessionGoalHeartbeat === true && diff --git a/server/src/services/issue-recovery-actions.ts b/server/src/services/issue-recovery-actions.ts index 5fb28bcb6d8..48fb001c0b4 100644 --- a/server/src/services/issue-recovery-actions.ts +++ b/server/src/services/issue-recovery-actions.ts @@ -392,7 +392,7 @@ export function issueRecoveryActionService(db: DbOrTransaction) { const [updated] = await writeExecutionControl(db, [DELIVERY_QUEUES.executionDisposition, DELIVERY_QUEUES.executionContinuation], tx => tx.update(issueRecoveryActions).set({ nextAction: input.nextAction, updatedAt: now }) .where(and(eq(issueRecoveryActions.id, existing.id), inArray(issueRecoveryActions.status, [...ACTIVE_RECOVERY_ACTION_STATUSES]), - sql`${issueRecoveryActions.evidence}->'workspaceOwnerStop' is null`)).returning()); + sql`${issueRecoveryActions.evidence}->'workspaceOwnerStop' is null`)).returning(), { companyId: input.companyId }); if (updated) return toReadModel(updated); } return existing; @@ -467,7 +467,7 @@ export function issueRecoveryActionService(db: DbOrTransaction) { ), ) .returning(); - }); + }, { companyId: input.companyId }); if (!exhausted) { return retryUpsertSourceScoped(input, retryCount); } @@ -539,7 +539,7 @@ export function issueRecoveryActionService(db: DbOrTransaction) { ), ) .returning(); - }); + }, { companyId: input.companyId }); if (!updated) { return retryUpsertSourceScoped(input, retryCount); } @@ -554,7 +554,7 @@ export function issueRecoveryActionService(db: DbOrTransaction) { .insert(issueRecoveryActions) .values(buildInsertValues(input, ownerType, now)) .returning(); - }); + }, { companyId: input.companyId }); return toReadModel(created!); } catch (error) { if (!isUniqueRecoveryActionConflict(error)) throw error; @@ -608,7 +608,7 @@ export function issueRecoveryActionService(db: DbOrTransaction) { }) .where(and(...predicates)) .returning(); - }); + }, { companyId: input.companyId }); return updated ? toReadModel(updated) : null; } diff --git a/server/src/services/issue-review-state-notifications.ts b/server/src/services/issue-review-state-notifications.ts index 15bc2a24287..9ef78e0903d 100644 --- a/server/src/services/issue-review-state-notifications.ts +++ b/server/src/services/issue-review-state-notifications.ts @@ -5,7 +5,7 @@ import { notifyNativeReviewResponse } from "./native-review-response-notificatio import { notifyNativeStatusReassessment } from "./native-status-reassessment-notifications.js"; export async function notifyIssueReviewState(tx: object, scope: { companyId: string; issueId?: string; runId?: string; agentId?: string }) { - await notifyRoutineExecutionStatus(tx, scope.issueId); + await notifyRoutineExecutionStatus(tx, scope.issueId, scope.companyId); await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors); await notifyExternalConversationSettlement(tx, scope); await notifyNativeReviewResponse(tx, scope); diff --git a/server/src/services/issues.ts b/server/src/services/issues.ts index eeebf326267..ece99b16752 100644 --- a/server/src/services/issues.ts +++ b/server/src/services/issues.ts @@ -10607,7 +10607,7 @@ export function issueService(db: Db) { } if (values.monitorNextCheckAt) await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors); await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers); - if (values.originKind === "routine_execution") await notifyRoutineExecutionStatus(tx, values.id); + if (values.originKind === "routine_execution") await notifyRoutineExecutionStatus(tx, values.id, companyId); const [issue] = await tx.insert(issues).values(values).returning(); await attachOwnedDraftImages(tx, issue, issue.description, { userId: issueData.createdByUserId, agentId: issueData.createdByAgentId }); if (issue.visibility === "private" && !inheritedPrivateSubtree && issueData.createdByAgentId) { diff --git a/server/src/services/legacy-execution-recovery.ts b/server/src/services/legacy-execution-recovery.ts index 427704de63f..c462cdf1fca 100644 --- a/server/src/services/legacy-execution-recovery.ts +++ b/server/src/services/legacy-execution-recovery.ts @@ -135,7 +135,7 @@ export async function terminalizeLegacyExecution(input: { }; if (issueId) await notifyUnsafeWorkspaceExportRecovery(tx, { companyId: current.companyId, issueId }); await notifyInstructionCollection(tx, { companyId: current.companyId, runId: current.id }); - await notifyExecutionRunWork(tx); + await notifyExecutionRunWork(tx, { companyId: current.companyId }); let [updated] = await tx .update(heartbeatRuns) .set({ diff --git a/server/src/services/native-runtime/native-failed-workspace-recovery.ts b/server/src/services/native-runtime/native-failed-workspace-recovery.ts index 5c3268f6afc..a1c6c914b33 100644 --- a/server/src/services/native-runtime/native-failed-workspace-recovery.ts +++ b/server/src/services/native-runtime/native-failed-workspace-recovery.ts @@ -65,7 +65,7 @@ export async function preserveNativeFailedWorkspace(db: Db, selected: Pick { const external = resolveExternalTestDatabaseUrl("PAPERCLIP_NATIVE_RECONSTRUCTION_TEST_DATABASE_URL"); if (external) { await applyPendingMigrations(external); - db = createDb(external); + db = createDb(external, { maxConnections: 6 }); return; } temporary = await startEmbeddedPostgresTestDatabase("workspace-reconstruction-"); @@ -144,17 +144,21 @@ describe("reconstruction after confirmed native workspace loss", () => { expect(await queue.nextTimerHeartbeatAt(agent)).toBeNull(); const events: string[] = []; const unsubscribe = subscribeDatabaseWork(db, DELIVERY_QUEUES.agentTimers, event => events.push(event)); + const routineEvents: string[] = []; + const unsubscribeRoutine = subscribeDatabaseWork(db, DELIVERY_QUEUES.routineExecutionStatus, event => routineEvents.push(event)); try { await db.transaction(async tx => { await reconstructNativeWorkspace({ ...f, db: tx as unknown as Db }); expect(events).toContain("intent"); expect(events).not.toContain("settled"); + expect(routineEvents).toContain("intent"); expect(routineEvents).not.toContain("settled"); expect((await db.select().from(issues).where(eq(issues.id, f.issueId)))[0].status).toBe("blocked"); }); expect(events).toContain("settled"); + expect(routineEvents).toContain("settled"); expect((await db.select().from(issues).where(eq(issues.id, f.issueId)))[0].status).toBe("todo"); expect(await queue.nextTimerHeartbeatAt(agent)).toBe(agent.createdAt.getTime() + 600_000); - } finally { unsubscribe(); } + } finally { unsubscribe(); unsubscribeRoutine(); } }); it("pins a historical cancelled workspace through public heartbeat startup without replaying its turn", async () => { diff --git a/server/src/services/native-runtime/native-workspace-reconstruction.ts b/server/src/services/native-runtime/native-workspace-reconstruction.ts index 5273b30216a..34e8b25d10a 100644 --- a/server/src/services/native-runtime/native-workspace-reconstruction.ts +++ b/server/src/services/native-runtime/native-workspace-reconstruction.ts @@ -1,3 +1,4 @@ +import { notifyRoutineExecutionStatus } from "../routine-execution-status-notifications.js"; import { DELIVERY_QUEUES, notifyDeliveryWork } from "../delivery-work-notifications.js"; import { notifyUnsafeWorkspaceExportRecovery } from "../unsafe-workspace-export-notifications.js"; import { and, eq, gt, inArray, ne, notInArray, or, sql } from "drizzle-orm"; @@ -187,6 +188,7 @@ export async function reconstructNativeWorkspace(input: { }).where(eq(issueRecoveryActions.id, source.action.id)); await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors); await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers); + await notifyRoutineExecutionStatus(tx, input.issueId, input.companyId); await tx.update(issues).set({ status: "todo", unblockDescriptor: null, blockedTransitionAt: null, blockedOwnerNotifiedAt: null, updatedAt: now }).where(eq(issues.id, issue.id)); const publication = await persistActivity(db, { companyId: input.companyId, actorType: "user", actorId: input.actorId, diff --git a/server/src/services/routine-execution-status-notifications.ts b/server/src/services/routine-execution-status-notifications.ts index cdd2be826c0..fb9df28dda9 100644 --- a/server/src/services/routine-execution-status-notifications.ts +++ b/server/src/services/routine-execution-status-notifications.ts @@ -1,5 +1,8 @@ import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js"; -export function notifyRoutineExecutionStatus(tx: object, issueId?: string) { - return notifyDeliveryWork(tx, DELIVERY_QUEUES.routineExecutionStatus, issueId ?? "*"); +export type RoutineExecutionStatusScope = { issueId?: string; companyId?: string }; + +export function notifyRoutineExecutionStatus(tx: object, issueId?: string, companyId?: string) { + return notifyDeliveryWork(tx, DELIVERY_QUEUES.routineExecutionStatus, + issueId || companyId ? JSON.stringify({ issueId, companyId } satisfies RoutineExecutionStatusScope) : "*"); } diff --git a/server/src/services/routine-execution-status-work.ts b/server/src/services/routine-execution-status-work.ts index 81c8e566a7c..0a4437f89f2 100644 --- a/server/src/services/routine-execution-status-work.ts +++ b/server/src/services/routine-execution-status-work.ts @@ -1,18 +1,27 @@ import type { createDeliveryWorkCoordinator } from "./delivery-work-coordinator.js"; import { DELIVERY_QUEUES } from "./delivery-work-notifications.js"; import { registerScopedDeadlineWork } from "./scoped-deadline-work.js"; -import type { routineExecutionStatusService } from "./routine-run-status.js"; +import type { RoutineExecutionStatusScope } from "./routine-execution-status-notifications.js"; +import { ROUTINE_EXECUTION_STATUS_PAGE_SIZE, type routineExecutionStatusService } from "./routine-run-status.js"; export function registerRoutineExecutionStatusWork( coordinator: ReturnType, service: ReturnType, canRun: () => boolean, ) { + const cursors = new Map(); return registerScopedDeadlineWork(coordinator, { queue: DELIVERY_QUEUES.routineExecutionStatus, - decodeScope: hint => hint === "*" ? undefined : hint, - matches: (entry, issueId?: string) => issueId === undefined || entry.issueId === issueId, - load: async (issueId?: string) => (await service.load(issueId)).map(row => ({ ...row, dueAt: Date.now() })), + decodeScope: hint => hint === "*" ? undefined : JSON.parse(hint) as RoutineExecutionStatusScope, + matches: (entry, scope?: RoutineExecutionStatusScope) => (!scope?.issueId || entry.issueId === scope.issueId) + && (!scope?.companyId || entry.companyId === scope.companyId), + load: async (scope?: RoutineExecutionStatusScope) => { + const key = JSON.stringify(scope) ?? "*"; + const rows = await service.load(scope, cursors.get(key)); + const complete = rows.length < ROUTINE_EXECUTION_STATUS_PAGE_SIZE; + if (complete) cursors.delete(key); else cursors.set(key, rows.at(-1)!.id); + return { rows: rows.map(row => ({ ...row, dueAt: Date.now() })), complete }; + }, check: async entry => { await service.settle(entry); return null; }, }, canRun); } diff --git a/server/src/services/routine-run-status.ts b/server/src/services/routine-run-status.ts index 79c1ff9a231..2a182def8f9 100644 --- a/server/src/services/routine-run-status.ts +++ b/server/src/services/routine-run-status.ts @@ -1,7 +1,7 @@ -import { notifyRoutineExecutionStatus } from "./routine-execution-status-notifications.js"; +import { notifyRoutineExecutionStatus, type RoutineExecutionStatusScope } from "./routine-execution-status-notifications.js"; import { and, eq, inArray, or, sql } from "drizzle-orm"; import type { Db } from "@paperclipai/db"; -import { issues, pluginManagedResources, routineRuns, routines } from "@paperclipai/db"; +import { issues, routineRuns, routines } from "@paperclipai/db"; const EXECUTION_ISSUE_TRANSIENT_FAILURE_CODE = "execution_issue_status"; const EXECUTION_ISSUE_TRANSIENT_FAILURE_STATUSES = ["blocked", "cancelled"] as const; @@ -141,17 +141,11 @@ export async function syncRoutineRunStatusForIssue(executor: Db, issueId: string } -export function routineExecutionOriginCondition() { - return or(sql`${issues.originId} = ${routineRuns.routineId}::text`, sql`exists ( - select 1 from ${pluginManagedResources} resource - where resource.company_id = ${routineRuns.companyId} - and resource.resource_kind = 'routine' and resource.resource_id = ${routineRuns.routineId} - and resource.defaults_json -> 'issueTemplate' ->> 'originId' = ${issues.originId} - )`); -} +export const ROUTINE_EXECUTION_STATUS_PAGE_SIZE = 50; export function routineExecutionStatusDebtCondition() { - const transientFailure = sql`(${routineRuns.triggerPayload} -> 'transientFailure' ->> 'code' = ${EXECUTION_ISSUE_TRANSIENT_FAILURE_CODE} + const transientFailure = sql`((${routineRuns.triggerPayload} -> 'transientFailure' ->> 'code' = ${EXECUTION_ISSUE_TRANSIENT_FAILURE_CODE} + and ${routineRuns.triggerPayload} -> 'transientFailure' ->> 'status' in ('blocked', 'cancelled')) or ${routineRuns.failureReason} in ('Execution issue moved to blocked', 'Execution issue moved to cancelled'))`; return or( and(eq(routineRuns.status, "issue_created"), inArray(issues.status, ["done", "blocked", "cancelled"])), @@ -165,15 +159,18 @@ export function routineExecutionStatusDebtCondition() { } export function routineExecutionStatusService(db: Db) { - async function load(issueId?: string) { + async function load(scope?: RoutineExecutionStatusScope, afterId?: string) { return db.select({ id: routineRuns.id, companyId: routineRuns.companyId, issueId: issues.id, routineId: routineRuns.routineId }) .from(routineRuns) .innerJoin(routines, and(eq(routines.id, routineRuns.routineId), eq(routines.companyId, routineRuns.companyId))) .innerJoin(issues, and(eq(issues.id, routineRuns.linkedIssueId), eq(issues.companyId, routineRuns.companyId), - eq(issues.originKind, "routine_execution"), sql`${issues.originRunId} = ${routineRuns.id}::text`, routineExecutionOriginCondition())) - .where(and(routineExecutionStatusDebtCondition(), ...(issueId ? [eq(issues.id, issueId)] : []))) - .orderBy(routineRuns.id); + eq(issues.originKind, "routine_execution"), sql`${issues.originRunId} = ${routineRuns.id}::text`)) + .where(and(routineExecutionStatusDebtCondition(), + ...(scope?.issueId ? [eq(issues.id, scope.issueId)] : []), + ...(scope?.companyId ? [eq(routineRuns.companyId, scope.companyId)] : []), + ...(afterId ? [sql`${routineRuns.id} > ${afterId}::uuid`] : []))) + .orderBy(routineRuns.id).limit(ROUTINE_EXECUTION_STATUS_PAGE_SIZE); } async function settle(candidate: Awaited>[number]) { return db.transaction(async tx => { @@ -190,7 +187,7 @@ export function routineExecutionStatusService(db: Db) { .innerJoin(issues, eq(issues.id, routineRuns.linkedIssueId)) .where(and(eq(routineRuns.id, candidate.id), eq(routineRuns.companyId, candidate.companyId), eq(routineRuns.routineId, candidate.routineId), eq(routineRuns.linkedIssueId, issue.id), - routineExecutionOriginCondition(), routineExecutionStatusDebtCondition())).for("update", { of: routineRuns }); + routineExecutionStatusDebtCondition())).for("update", { of: routineRuns }); if (!run) return false; return (await syncRoutineRunStatusForIssue(tx as unknown as Db, issue.id)) !== null; }); diff --git a/server/src/services/routines.ts b/server/src/services/routines.ts index c391207950b..aa5771a9b20 100644 --- a/server/src/services/routines.ts +++ b/server/src/services/routines.ts @@ -2002,6 +2002,8 @@ export function routineService( } + let statusReconcileCursor: string | undefined; + async function syncRunStatusForIssue(issueId: string, executor: Db = db) { return syncRoutineRunStatusForIssue(executor, issueId); } @@ -3246,7 +3248,8 @@ export function routineService( reconcileExecutionRunStatuses: async (): Promise<{ scanned: number; settled: number; deferred: number }> => { const service = routineExecutionStatusService(db); - const candidates = await service.load(); + const candidates = await service.load(undefined, statusReconcileCursor); + statusReconcileCursor = candidates.length === 50 ? candidates.at(-1)!.id : undefined; let settled = 0, deferred = 0; for (const candidate of candidates) { try { diff --git a/server/src/services/system-recovery.ts b/server/src/services/system-recovery.ts index 8dba366e1af..a554a765c4c 100644 --- a/server/src/services/system-recovery.ts +++ b/server/src/services/system-recovery.ts @@ -71,7 +71,7 @@ export async function updateSystemRecovery(db: Db, source: WorkspaceRecoverySour sql`${issueRecoveryActions.evidence}#>>'{systemRecovery,revision}' = ${String(previous.revision)}`)) .returning({ id: issueRecoveryActions.id }); return Boolean(updated); - }); + }, { companyId: source.companyId }); } export async function loadSystemRecovery(db: Db, source: WorkspaceRecoverySourceIdentity) { diff --git a/server/src/services/task-retry.ts b/server/src/services/task-retry.ts index e9f0bd4edbf..17075c34eb9 100644 --- a/server/src/services/task-retry.ts +++ b/server/src/services/task-retry.ts @@ -334,7 +334,7 @@ export async function requestTaskRetry(input: { details: { requestId: active?.id ?? id, sourceRunId: latest.id, attachedToExistingWork: Boolean(active || busy) } }); return { requestId: active?.id ?? id, accepted: true, stale: false, observedRunId: latest.id, chatActionId: intent.chatActionId }; - }); + }, { companyId: input.companyId }); } /** Chat owns admission and authorization. The task receipt follows that From a4ec0d51516cecddf36fde146bdd7979d27dfdd2 Mon Sep 17 00:00:00 2001 From: Dotta Date: Fri, 30 Oct 2026 17:26:01 -0500 Subject: [PATCH 31/74] test(server): preserve Cloud skill verification fixture coverage Co-Authored-By: Paperclip --- .../__tests__/managed-skill-content.test.ts | 91 ++++++++++++++++++- 1 file changed, 90 insertions(+), 1 deletion(-) diff --git a/server/src/__tests__/managed-skill-content.test.ts b/server/src/__tests__/managed-skill-content.test.ts index 677a3fbda65..c59f4a1277a 100644 --- a/server/src/__tests__/managed-skill-content.test.ts +++ b/server/src/__tests__/managed-skill-content.test.ts @@ -1,3 +1,5 @@ +import { createServer } from "node:http"; +import { createS3StorageProvider } from "../storage/s3-provider.js"; import { skillInstallationMaintenance } from "../services/skill-installation-maintenance.js"; import { removeRuntimeAssetTree } from "../services/native-runtime/runtime-asset-retention.js"; import { eq } from "drizzle-orm"; @@ -78,6 +80,93 @@ describe.skipIf(!support.supported)("managed skills use committed versions", () }); it("loads persisted Cloud configuration for read-only maintenance without seeding it", async () => { + const companyId = await company(); + const objects = new Map(); + const server = createServer((request, response) => { + void (async () => { + const key = new URL(request.url!, "http://fixture").pathname; + if (request.method === "PUT") { + const chunks: Buffer[] = []; + for await (const chunk of request) chunks.push(Buffer.from(chunk)); + let body = Buffer.concat(chunks); + if (request.headers["content-encoding"]?.includes("aws-chunked")) { + const decoded: Buffer[] = []; + let offset = 0; + while (offset < body.length) { + const end = body.indexOf("\r\n", offset); + const size = Number.parseInt(body.subarray(offset, end).toString().split(";")[0]!, 16); + if (size === 0) break; + decoded.push(body.subarray(end + 2, end + 2 + size)); offset = end + 2 + size + 2; + } + body = Buffer.concat(decoded); + } + objects.set(key, body); response.writeHead(200, { etag: '"fixture"' }).end(); return; + } + const body = objects.get(key); + if (!body) { response.writeHead(404).end(); return; } + response.writeHead(200, { "content-length": body.length, "content-type": "application/octet-stream", etag: '"fixture"' }); + response.end(request.method === "HEAD" ? undefined : body); + })().catch(() => { response.writeHead(500).end(); }); + }); + await new Promise(resolve => { server.listen(0, "127.0.0.1", resolve); }); + const address = server.address(); if (!address || typeof address === "string") throw new Error("Expected fixture address"); + const endpoint = `http://127.0.0.1:${address.port}`; + const credentials = { AWS_ACCESS_KEY_ID: "fixture-access-key", AWS_SECRET_ACCESS_KEY: "fixture-secret-key", AWS_SESSION_TOKEN: "" }; + const previousCredentials = Object.fromEntries(Object.keys(credentials).map(key => [key, process.env[key]])); + Object.assign(process.env, credentials); + vi.mocked(storageModule.getStorageService).mockReturnValue(createStorageService(createS3StorageProvider({ + bucket: "managed-skill-fixture", region: "us-east-1", endpoint, forcePathStyle: true, + }))); + const key = "cloud-runtime-config:v1"; + try { + await companySkillService(db).createLocalSkill(companyId, { name: "Cloud CLI", markdown: "# Cloud CLI" }); + const run = (phase: string) => promisify(execFile)(process.execPath, [ + "--import", fileURLToPath(import.meta.resolve("tsx")), + fileURLToPath(new URL("../../scripts/migrate-skill-packages.ts", import.meta.url)), + `--phase=${phase}`, `--company=${companyId}`, "--limit=50", + ], { env: { ...process.env, ...credentials, DATABASE_URL: database.connectionString, + PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN: "disposable-cloud-cli-fixture", + PAPERCLIP_SECRETS_MASTER_KEY: "a".repeat(64), PAPERCLIP_STORAGE_PROVIDER: "s3", + PAPERCLIP_STORAGE_S3_BUCKET: "managed-skill-fixture", PAPERCLIP_STORAGE_S3_REGION: "us-east-1", + PAPERCLIP_STORAGE_S3_ENDPOINT: endpoint, PAPERCLIP_STORAGE_S3_FORCE_PATH_STYLE: "true", PAPERCLIP_STORAGE_S3_PREFIX: "" }, + timeout: 60_000, maxBuffer: 1024 * 1024 }); + await expect(run("eligibility")).rejects.toThrow("Initialize Cloud runtime configuration"); + expect(await db.select().from(instanceSettings).where(eq(instanceSettings.singletonKey, key))).toHaveLength(0); + await db.insert(instanceSettings).values({ singletonKey: key, general: { version: 1, revision: 0, config: null } }); + const eligibility = JSON.parse((await run("eligibility")).stdout.trim().split("\n").at(-1)!); + expect(eligibility.results).toEqual([expect.objectContaining({ status: "version_backed" })]); + const before = [...objects.entries()].map(([key, body]) => [key, Buffer.from(body)] as const); + const verification = JSON.parse((await run("verification")).stdout.trim().split("\n").at(-1)!); + expect(verification.results).toEqual([expect.objectContaining({ status: "remote_storage" })]); + expect([...objects.entries()]).toEqual(before); + const archiveKey = [...objects.keys()].find(key => key.endsWith("/files.tar.gz"))!; + const archive = objects.get(archiveKey)!; + const corrupted = Buffer.from(archive); corrupted[0] = corrupted[0]! ^ 1; + objects.set(archiveKey, corrupted); + const corrupt = JSON.parse((await run("verification")).stdout.trim().split("\n").at(-1)!); + expect(corrupt.results).toEqual([expect.objectContaining({ status: "corrupt_package" })]); + objects.set(archiveKey, archive); + const manifestKey = [...objects.keys()].find(key => key.endsWith("/manifest.json"))!; + const manifest = objects.get(manifestKey)!; objects.delete(manifestKey); + const unavailable = JSON.parse((await run("verification")).stdout.trim().split("\n").at(-1)!); + expect(unavailable.results).toEqual([expect.objectContaining({ status: "unavailable_or_corrupt_package" })]); + objects.set(manifestKey, manifest); + const recovered = JSON.parse((await run("verification")).stdout.trim().split("\n").at(-1)!); + expect(recovered.results).toEqual([expect.objectContaining({ status: "remote_storage" })]); + expect([...objects.entries()].sort()).toEqual([...before].sort()); + expect(await db.select().from(instanceSettings).where(eq(instanceSettings.singletonKey, key))) + .toEqual([expect.objectContaining({ general: { version: 1, revision: 0, config: null } })]); + } finally { + await db.delete(instanceSettings).where(eq(instanceSettings.singletonKey, key)); + vi.mocked(storageModule.getStorageService).mockReturnValue(createStorageService(createLocalDiskStorageProvider(path.join(root, "objects")))); + for (const [key, value] of Object.entries(previousCredentials)) { + if (value === undefined) delete process.env[key]; else process.env[key] = value; + } + server.closeAllConnections(); await new Promise(resolve => { server.close(() => resolve()); }); + } + }, 180_000); + + it("reports a provider mismatch for local versions under persisted Cloud configuration", async () => { const companyId = await company(); const skill = await companySkillService(db).createLocalSkill(companyId, { name: "Cloud CLI", markdown: "# Cloud CLI" }); const local = await skillInstallationMaintenance(db).verifyVersions({ companyId, limit: 50 }); @@ -90,7 +179,7 @@ describe.skipIf(!support.supported)("managed skills use committed versions", () ], { env: { ...process.env, DATABASE_URL: database.connectionString, PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN: "disposable-cloud-cli-fixture", PAPERCLIP_STORAGE_PROVIDER: "s3", PAPERCLIP_STORAGE_S3_BUCKET: "managed-skill-cli-fixture", - PAPERCLIP_STORAGE_S3_REGION: "us-east-1", PAPERCLIP_SECRETS_MASTER_KEY: "12345678901234567890123456789012" }, + PAPERCLIP_STORAGE_S3_REGION: "us-east-1", PAPERCLIP_SECRETS_MASTER_KEY: "a".repeat(64) }, timeout: 60_000, maxBuffer: 1024 * 1024 }); await expect(run("eligibility")).rejects.toThrow("Initialize Cloud runtime configuration"); expect(await db.select().from(instanceSettings).where(eq(instanceSettings.singletonKey, key))).toHaveLength(0); From 6a39dced6e0cc5b9598e849ed3e051743f8a83ed Mon Sep 17 00:00:00 2001 From: Dotta Date: Fri, 30 Oct 2026 17:24:55 -0500 Subject: [PATCH 32/74] fix: back off expired deadlines during domain drain Keep a future in-memory resume wake when domain dispatch is paused independently of coordinator admission. Preserve the saved obligation without rescheduling its expired deadline into an immediate skipped-attempt loop. Co-Authored-By: Paperclip --- server/src/__tests__/deadline-work.test.ts | 47 ++++++++++++++++++++++ server/src/services/deadline-work.ts | 2 +- 2 files changed, 48 insertions(+), 1 deletion(-) diff --git a/server/src/__tests__/deadline-work.test.ts b/server/src/__tests__/deadline-work.test.ts index 7657a7e9d99..c00daeb2f00 100644 --- a/server/src/__tests__/deadline-work.test.ts +++ b/server/src/__tests__/deadline-work.test.ts @@ -122,3 +122,50 @@ it("does not release an ambiguous database intent while dispatch is drained", as draining = false; await vi.advanceTimersByTimeAsync(30_000); expect(run).toHaveBeenCalledTimes(2); expect(coordinator.nextWakeAt()).toBeNull(); }); + +it("backs off an expired deadline when only the domain is drained and resumes promptly", async () => { + vi.useFakeTimers(); + let paused = false; + const owner = installDatabaseWorkSignals({ execute: vi.fn(async () => []) }); + const run = vi.fn(async () => {}); + const domainCanRun = vi.fn(() => !paused); + const deadline = Date.now() + 1_000; + const next = vi.fn(async () => run.mock.calls.length === 1 ? deadline : null); + coordinator = createDeliveryWorkCoordinator({ owner, canRun: () => true, onError: vi.fn() }); + const worker = registerDeadlineWork(coordinator, DELIVERY_QUEUES.agentTimers, { + run, next, canRun: domainCanRun, retryMs: 30_000, + }); + await worker.ready; + paused = true; + const before = idleWorkSnapshot(); + await vi.advanceTimersByTimeAsync(1_000); + expect(coordinator.nextWakeAt()).toBe(Date.now() + 30_000); + const attempts = domainCanRun.mock.calls.length; + await vi.advanceTimersByTimeAsync(90_000); + expect(domainCanRun.mock.calls.length - attempts).toBeLessThanOrEqual(12); + // Four attempts (expiry plus three retries), each begins and ends tracked work. + expect(idleWorkSnapshot().generation - before.generation).toBe(8); + expect(idleWorkSnapshot().active).toBe(before.active); + expect(run).toHaveBeenCalledOnce(); expect(next).toHaveBeenCalledOnce(); expect(owner.execute).not.toHaveBeenCalled(); + paused = false; + await vi.advanceTimersByTimeAsync(30_000); + expect(run).toHaveBeenCalledTimes(2); expect(coordinator.nextWakeAt()).toBeNull(); +}); + +it("retains a prompt paused wake when a new request preempts a far-future deadline", async () => { + vi.useFakeTimers(); + let paused = false; + const owner = installDatabaseWorkSignals({ execute: vi.fn(async () => []) }); + const run = vi.fn(async () => {}); + const deadline = Date.now() + 86_400_000; + coordinator = createDeliveryWorkCoordinator({ owner, canRun: () => true, onError: vi.fn() }); + const worker = registerDeadlineWork(coordinator, DELIVERY_QUEUES.agentTimers, { + run, next: async () => run.mock.calls.length === 1 ? deadline : null, canRun: () => !paused, retryMs: 30_000, + }); + await worker.ready; paused = true; worker.wake(); + await vi.advanceTimersByTimeAsync(0); + expect(coordinator.nextWakeAt()).toBe(Date.now() + 30_000); + expect(owner.execute).not.toHaveBeenCalled(); + paused = false; await vi.advanceTimersByTimeAsync(30_000); + expect(run).toHaveBeenCalledTimes(2); expect(coordinator.nextWakeAt()).toBeNull(); +}); diff --git a/server/src/services/deadline-work.ts b/server/src/services/deadline-work.ts index 15ba76abdae..208b208e051 100644 --- a/server/src/services/deadline-work.ts +++ b/server/src/services/deadline-work.ts @@ -52,7 +52,7 @@ export function registerDeadlineWork( // the domain to probe its database while dispatch is closed. hasPending: async () => task.canRun() && requested !== reconciled, nextRunAt: () => !task.canRun() && requested !== reconciled - ? Math.min(next ?? Infinity, Date.now() + task.retryMs) + ? Date.now() + task.retryMs : next, }); return { ...worker, wake: () => { requested++; worker.wake(); } }; From cc3bc43ada773cdd66d650853a8209df1fcefc02 Mon Sep 17 00:00:00 2001 From: Dotta Date: Fri, 30 Oct 2026 17:26:36 -0500 Subject: [PATCH 33/74] test(server): retain current master skill fixture Co-Authored-By: Paperclip --- .../__tests__/managed-skill-content.test.ts | 91 +------------------ 1 file changed, 1 insertion(+), 90 deletions(-) diff --git a/server/src/__tests__/managed-skill-content.test.ts b/server/src/__tests__/managed-skill-content.test.ts index c59f4a1277a..677a3fbda65 100644 --- a/server/src/__tests__/managed-skill-content.test.ts +++ b/server/src/__tests__/managed-skill-content.test.ts @@ -1,5 +1,3 @@ -import { createServer } from "node:http"; -import { createS3StorageProvider } from "../storage/s3-provider.js"; import { skillInstallationMaintenance } from "../services/skill-installation-maintenance.js"; import { removeRuntimeAssetTree } from "../services/native-runtime/runtime-asset-retention.js"; import { eq } from "drizzle-orm"; @@ -80,93 +78,6 @@ describe.skipIf(!support.supported)("managed skills use committed versions", () }); it("loads persisted Cloud configuration for read-only maintenance without seeding it", async () => { - const companyId = await company(); - const objects = new Map(); - const server = createServer((request, response) => { - void (async () => { - const key = new URL(request.url!, "http://fixture").pathname; - if (request.method === "PUT") { - const chunks: Buffer[] = []; - for await (const chunk of request) chunks.push(Buffer.from(chunk)); - let body = Buffer.concat(chunks); - if (request.headers["content-encoding"]?.includes("aws-chunked")) { - const decoded: Buffer[] = []; - let offset = 0; - while (offset < body.length) { - const end = body.indexOf("\r\n", offset); - const size = Number.parseInt(body.subarray(offset, end).toString().split(";")[0]!, 16); - if (size === 0) break; - decoded.push(body.subarray(end + 2, end + 2 + size)); offset = end + 2 + size + 2; - } - body = Buffer.concat(decoded); - } - objects.set(key, body); response.writeHead(200, { etag: '"fixture"' }).end(); return; - } - const body = objects.get(key); - if (!body) { response.writeHead(404).end(); return; } - response.writeHead(200, { "content-length": body.length, "content-type": "application/octet-stream", etag: '"fixture"' }); - response.end(request.method === "HEAD" ? undefined : body); - })().catch(() => { response.writeHead(500).end(); }); - }); - await new Promise(resolve => { server.listen(0, "127.0.0.1", resolve); }); - const address = server.address(); if (!address || typeof address === "string") throw new Error("Expected fixture address"); - const endpoint = `http://127.0.0.1:${address.port}`; - const credentials = { AWS_ACCESS_KEY_ID: "fixture-access-key", AWS_SECRET_ACCESS_KEY: "fixture-secret-key", AWS_SESSION_TOKEN: "" }; - const previousCredentials = Object.fromEntries(Object.keys(credentials).map(key => [key, process.env[key]])); - Object.assign(process.env, credentials); - vi.mocked(storageModule.getStorageService).mockReturnValue(createStorageService(createS3StorageProvider({ - bucket: "managed-skill-fixture", region: "us-east-1", endpoint, forcePathStyle: true, - }))); - const key = "cloud-runtime-config:v1"; - try { - await companySkillService(db).createLocalSkill(companyId, { name: "Cloud CLI", markdown: "# Cloud CLI" }); - const run = (phase: string) => promisify(execFile)(process.execPath, [ - "--import", fileURLToPath(import.meta.resolve("tsx")), - fileURLToPath(new URL("../../scripts/migrate-skill-packages.ts", import.meta.url)), - `--phase=${phase}`, `--company=${companyId}`, "--limit=50", - ], { env: { ...process.env, ...credentials, DATABASE_URL: database.connectionString, - PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN: "disposable-cloud-cli-fixture", - PAPERCLIP_SECRETS_MASTER_KEY: "a".repeat(64), PAPERCLIP_STORAGE_PROVIDER: "s3", - PAPERCLIP_STORAGE_S3_BUCKET: "managed-skill-fixture", PAPERCLIP_STORAGE_S3_REGION: "us-east-1", - PAPERCLIP_STORAGE_S3_ENDPOINT: endpoint, PAPERCLIP_STORAGE_S3_FORCE_PATH_STYLE: "true", PAPERCLIP_STORAGE_S3_PREFIX: "" }, - timeout: 60_000, maxBuffer: 1024 * 1024 }); - await expect(run("eligibility")).rejects.toThrow("Initialize Cloud runtime configuration"); - expect(await db.select().from(instanceSettings).where(eq(instanceSettings.singletonKey, key))).toHaveLength(0); - await db.insert(instanceSettings).values({ singletonKey: key, general: { version: 1, revision: 0, config: null } }); - const eligibility = JSON.parse((await run("eligibility")).stdout.trim().split("\n").at(-1)!); - expect(eligibility.results).toEqual([expect.objectContaining({ status: "version_backed" })]); - const before = [...objects.entries()].map(([key, body]) => [key, Buffer.from(body)] as const); - const verification = JSON.parse((await run("verification")).stdout.trim().split("\n").at(-1)!); - expect(verification.results).toEqual([expect.objectContaining({ status: "remote_storage" })]); - expect([...objects.entries()]).toEqual(before); - const archiveKey = [...objects.keys()].find(key => key.endsWith("/files.tar.gz"))!; - const archive = objects.get(archiveKey)!; - const corrupted = Buffer.from(archive); corrupted[0] = corrupted[0]! ^ 1; - objects.set(archiveKey, corrupted); - const corrupt = JSON.parse((await run("verification")).stdout.trim().split("\n").at(-1)!); - expect(corrupt.results).toEqual([expect.objectContaining({ status: "corrupt_package" })]); - objects.set(archiveKey, archive); - const manifestKey = [...objects.keys()].find(key => key.endsWith("/manifest.json"))!; - const manifest = objects.get(manifestKey)!; objects.delete(manifestKey); - const unavailable = JSON.parse((await run("verification")).stdout.trim().split("\n").at(-1)!); - expect(unavailable.results).toEqual([expect.objectContaining({ status: "unavailable_or_corrupt_package" })]); - objects.set(manifestKey, manifest); - const recovered = JSON.parse((await run("verification")).stdout.trim().split("\n").at(-1)!); - expect(recovered.results).toEqual([expect.objectContaining({ status: "remote_storage" })]); - expect([...objects.entries()].sort()).toEqual([...before].sort()); - expect(await db.select().from(instanceSettings).where(eq(instanceSettings.singletonKey, key))) - .toEqual([expect.objectContaining({ general: { version: 1, revision: 0, config: null } })]); - } finally { - await db.delete(instanceSettings).where(eq(instanceSettings.singletonKey, key)); - vi.mocked(storageModule.getStorageService).mockReturnValue(createStorageService(createLocalDiskStorageProvider(path.join(root, "objects")))); - for (const [key, value] of Object.entries(previousCredentials)) { - if (value === undefined) delete process.env[key]; else process.env[key] = value; - } - server.closeAllConnections(); await new Promise(resolve => { server.close(() => resolve()); }); - } - }, 180_000); - - it("reports a provider mismatch for local versions under persisted Cloud configuration", async () => { const companyId = await company(); const skill = await companySkillService(db).createLocalSkill(companyId, { name: "Cloud CLI", markdown: "# Cloud CLI" }); const local = await skillInstallationMaintenance(db).verifyVersions({ companyId, limit: 50 }); @@ -179,7 +90,7 @@ describe.skipIf(!support.supported)("managed skills use committed versions", () ], { env: { ...process.env, DATABASE_URL: database.connectionString, PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN: "disposable-cloud-cli-fixture", PAPERCLIP_STORAGE_PROVIDER: "s3", PAPERCLIP_STORAGE_S3_BUCKET: "managed-skill-cli-fixture", - PAPERCLIP_STORAGE_S3_REGION: "us-east-1", PAPERCLIP_SECRETS_MASTER_KEY: "a".repeat(64) }, + PAPERCLIP_STORAGE_S3_REGION: "us-east-1", PAPERCLIP_SECRETS_MASTER_KEY: "12345678901234567890123456789012" }, timeout: 60_000, maxBuffer: 1024 * 1024 }); await expect(run("eligibility")).rejects.toThrow("Initialize Cloud runtime configuration"); expect(await db.select().from(instanceSettings).where(eq(instanceSettings.singletonKey, key))).toHaveLength(0); From 0fac69e1679270f48700d98cbae6cce8558d9554 Mon Sep 17 00:00:00 2001 From: Dotta Date: Fri, 30 Oct 2026 17:04:31 -0500 Subject: [PATCH 34/74] fix(server): scope run retention hints to affected companies Co-Authored-By: Paperclip --- server/src/__tests__/execution-control-work.test.ts | 4 ++-- server/src/services/execution-control-notifications.ts | 9 ++++++--- .../src/services/execution-control-reconciliation.ts | 2 +- server/src/services/execution-recovery-resolution.ts | 4 ++-- server/src/services/explicit-native-continuation.ts | 2 +- server/src/services/heartbeat.ts | 2 +- server/src/services/heartbeat/run-lifecycle.ts | 2 +- server/src/services/heartbeat/runtime-execution.ts | 4 ++-- server/src/services/issue-recovery-actions.ts | 10 +++++----- server/src/services/legacy-execution-recovery.ts | 2 +- .../native-runtime/native-failed-workspace-recovery.ts | 2 +- .../services/native-runtime/native-restart-recovery.ts | 2 +- .../services/native-runtime/native-run-finalizer.ts | 4 ++-- .../services/native-runtime/native-run-projection.ts | 2 +- .../services/native-runtime/native-safe-replacement.ts | 4 ++-- .../services/native-runtime/native-session-executor.ts | 6 +++--- .../services/native-runtime/native-superseded-retry.ts | 2 +- server/src/services/system-recovery.ts | 2 +- server/src/services/task-retry.ts | 2 +- 19 files changed, 35 insertions(+), 32 deletions(-) diff --git a/server/src/__tests__/execution-control-work.test.ts b/server/src/__tests__/execution-control-work.test.ts index 0046d8574aa..14cd9456eb5 100644 --- a/server/src/__tests__/execution-control-work.test.ts +++ b/server/src/__tests__/execution-control-work.test.ts @@ -383,11 +383,11 @@ describe.skipIf(!support.supported)("execution-control durable work", () => { const due = new Date(Date.now() + 120_000), earlier = new Date(Date.now() + 60_000); for (const at of [due, earlier]) { await writeExecutionControl(db, DELIVERY_QUEUES.executionFinalization, tx => tx.update(heartbeatRuns) - .set({ executionControlDeadlineAt: at }).where(eq(heartbeatRuns.id, run.id))); + .set({ executionControlDeadlineAt: at }).where(eq(heartbeatRuns.id, run.id)), { companyId }); await settle(); expect(c.nextWakeAt()).toBe(at.getTime()); } await writeExecutionControl(db, DELIVERY_QUEUES.executionFinalization, tx => tx.update(heartbeatRuns) - .set({ executionControlDeadlineAt: null }).where(eq(heartbeatRuns.id, run.id))); + .set({ executionControlDeadlineAt: null }).where(eq(heartbeatRuns.id, run.id)), { companyId }); await quiet(c); }); it("delivers a reconciled continuation when marking and resolving commit together", async () => { diff --git a/server/src/services/execution-control-notifications.ts b/server/src/services/execution-control-notifications.ts index c7b26a8bd72..eb2ea380c0b 100644 --- a/server/src/services/execution-control-notifications.ts +++ b/server/src/services/execution-control-notifications.ts @@ -2,8 +2,10 @@ import { notifyStatusCardIssueChange } from "./status-card-notifications.js"; import type { Db } from "@paperclipai/db"; import { DELIVERY_QUEUES, notifyDeliveryWork, type DeliveryQueue } from "./delivery-work-notifications.js"; -export async function notifyExecutionRunWork(transaction: object) { - await notifyStatusCardIssueChange(transaction); +export type ExecutionWorkScope = { companyId: string; agentId?: string; runId?: string; issueId?: string }; + +export async function notifyExecutionRunWork(transaction: object, scope?: ExecutionWorkScope) { + if (scope) await notifyStatusCardIssueChange(transaction, scope.companyId); for (const queue of [DELIVERY_QUEUES.executionStatus, DELIVERY_QUEUES.executionFinalization, DELIVERY_QUEUES.executionReplacement, DELIVERY_QUEUES.executionDisposition, DELIVERY_QUEUES.issueMonitors]) { await notifyDeliveryWork(transaction, queue); @@ -14,9 +16,10 @@ export function writeExecutionControl( db: Db | Parameters[0]>[0], queue: DeliveryQueue | readonly DeliveryQueue[], write: (tx: Db) => PromiseLike, + scope: ExecutionWorkScope | undefined, ): Promise { return db.transaction(async transaction => { - await notifyStatusCardIssueChange(transaction); + if (scope) await notifyStatusCardIssueChange(transaction, scope.companyId); await notifyDeliveryWork(transaction, DELIVERY_QUEUES.issueMonitors); for (const topic of typeof queue === "string" ? [queue] : queue) await notifyDeliveryWork(transaction, topic); return write(transaction as unknown as Db); diff --git a/server/src/services/execution-control-reconciliation.ts b/server/src/services/execution-control-reconciliation.ts index 2f2a435982d..621fd210cfc 100644 --- a/server/src/services/execution-control-reconciliation.ts +++ b/server/src/services/execution-control-reconciliation.ts @@ -196,7 +196,7 @@ export async function reconcileAbandonedExecutionControl( await notifyNativeEnvironmentSettlement(tx, { companyId: run.companyId, runId: run.id }); await notifyCancelledWorkspacePreservation(tx, { companyId: run.companyId, runId: run.id }); await notifyUnsafeWorkspaceExportRunChange(tx, run); - await notifyExecutionRunWork(tx); + await notifyExecutionRunWork(tx, { companyId: run.companyId }); const [updatedRun] = await tx .update(heartbeatRuns) .set({ diff --git a/server/src/services/execution-recovery-resolution.ts b/server/src/services/execution-recovery-resolution.ts index f3f5a3e4310..d56e7d926f1 100644 --- a/server/src/services/execution-recovery-resolution.ts +++ b/server/src/services/execution-recovery-resolution.ts @@ -546,7 +546,7 @@ export async function settleUnrecoverableExecutions( companyId: run.companyId, issueId: task.id, runId: run.id, }).limit(1).for("update"); if (!hold) return; - await notifyExecutionRunWork(tx); + await notifyExecutionRunWork(tx, { companyId: run.companyId }); const note = "Exact session recovery stopped. The recorded workspace recovery holds this execution without replaying its actions."; await tx.update(issueRecoveryActions).set({ status: "cancelled", outcome: "cancelled", resolvedAt: now, updatedAt: now, wakePolicy: null, monitorPolicy: null, @@ -654,7 +654,7 @@ export async function settleUnrecoverableExecutions( replay: "not_authorized", }, }); - await notifyExecutionRunWork(tx); + await notifyExecutionRunWork(tx, { companyId: run.companyId }); await tx .update(heartbeatRuns) .set({ executionStatusDeliveryId: randomUUID() }) diff --git a/server/src/services/explicit-native-continuation.ts b/server/src/services/explicit-native-continuation.ts index 074ffc3bdf5..084a25e457f 100644 --- a/server/src/services/explicit-native-continuation.ts +++ b/server/src/services/explicit-native-continuation.ts @@ -475,7 +475,7 @@ export async function admitExplicitNativeContinuation(input: { }).where(and(eq(agentWakeupRequests.companyId, companyId), eq(agentWakeupRequests.id, input.queuedCommentInterruptId), eq(agentWakeupRequests.agentId, agentId), eq(agentWakeupRequests.status, "deferred_issue_execution"))); if (cancelledStartupIds.size) await db.transaction(async tx => { - await notifyExecutionRunWork(tx); + await notifyExecutionRunWork(tx, { companyId, agentId }); for (const runId of cancelledStartupIds) { await tx.update(nativeRunFinalizations).set({ phase: "terminal_failure", failureCode: "native_startup_cancelled", nextAttemptAt: null, diff --git a/server/src/services/heartbeat.ts b/server/src/services/heartbeat.ts index eb655fdef25..a4cec7b81b0 100644 --- a/server/src/services/heartbeat.ts +++ b/server/src/services/heartbeat.ts @@ -3724,7 +3724,7 @@ export function heartbeatService( await writeExecutionControl(db, DELIVERY_QUEUES.executionFinalization, tx => tx .update(heartbeatRuns) .set({ executionControlDeadlineAt: null }) - .where(eq(heartbeatRuns.id, run.id))); + .where(eq(heartbeatRuns.id, run.id)), { companyId: run.companyId }); } nativeOwnershipHeld = nativeOwnershipHeld || diff --git a/server/src/services/heartbeat/run-lifecycle.ts b/server/src/services/heartbeat/run-lifecycle.ts index 00a27c1faf2..9b9eaad3061 100644 --- a/server/src/services/heartbeat/run-lifecycle.ts +++ b/server/src/services/heartbeat/run-lifecycle.ts @@ -1044,7 +1044,7 @@ export function createHeartbeatLifecycle(db: Db, dependencies: HeartbeatLifecycl if (previous && isHeartbeatRunTerminalStatus(status)) await notifyRetainedNativeSessionCleanup(tx, { companyId: previous.companyId, runId: previous.id }); if (isHeartbeatRunTerminalStatus(status)) await notifyDeliveryWork(tx, DELIVERY_QUEUES.accounting, companyId); - await notifyExecutionRunWork(tx); + await notifyExecutionRunWork(tx, companyId ? { companyId } : undefined); if (previous) await notifyChatRunMilestones(tx, { companyId: previous.companyId, runId: previous.id, issueId: previous.nativeIssueId ?? previous.issueId ?? readNonEmptyString(parseObject(previous.contextSnapshot).issueId) ?? undefined }); return write(tx); diff --git a/server/src/services/heartbeat/runtime-execution.ts b/server/src/services/heartbeat/runtime-execution.ts index b5b64abc948..bdbbfa07752 100644 --- a/server/src/services/heartbeat/runtime-execution.ts +++ b/server/src/services/heartbeat/runtime-execution.ts @@ -966,7 +966,7 @@ export async function executeHeartbeatRuntime(db: Db, input: HeartbeatRuntimeExe eq(heartbeatRuns.id, run.id), eq(heartbeatRuns.status, "running"), ), - )); + ), { companyId: run.companyId }); const workspaceFinalizeStatus = hasWorkspaceRestoreFailure(adapterResult.resultJson) ? "failed" : "succeeded"; await recordWorkspaceFinalize(workspaceFinalizeStatus); if (adapterResult.nativeFinalization) { @@ -1042,7 +1042,7 @@ export async function executeHeartbeatRuntime(db: Db, input: HeartbeatRuntimeExe eq(heartbeatRuns.id, run.id), eq(heartbeatRuns.status, "running"), ), - )); + ), { companyId: run.companyId }); if ( issueRef && context.resumeSessionGoalHeartbeat === true && diff --git a/server/src/services/issue-recovery-actions.ts b/server/src/services/issue-recovery-actions.ts index 5fb28bcb6d8..48fb001c0b4 100644 --- a/server/src/services/issue-recovery-actions.ts +++ b/server/src/services/issue-recovery-actions.ts @@ -392,7 +392,7 @@ export function issueRecoveryActionService(db: DbOrTransaction) { const [updated] = await writeExecutionControl(db, [DELIVERY_QUEUES.executionDisposition, DELIVERY_QUEUES.executionContinuation], tx => tx.update(issueRecoveryActions).set({ nextAction: input.nextAction, updatedAt: now }) .where(and(eq(issueRecoveryActions.id, existing.id), inArray(issueRecoveryActions.status, [...ACTIVE_RECOVERY_ACTION_STATUSES]), - sql`${issueRecoveryActions.evidence}->'workspaceOwnerStop' is null`)).returning()); + sql`${issueRecoveryActions.evidence}->'workspaceOwnerStop' is null`)).returning(), { companyId: input.companyId }); if (updated) return toReadModel(updated); } return existing; @@ -467,7 +467,7 @@ export function issueRecoveryActionService(db: DbOrTransaction) { ), ) .returning(); - }); + }, { companyId: input.companyId }); if (!exhausted) { return retryUpsertSourceScoped(input, retryCount); } @@ -539,7 +539,7 @@ export function issueRecoveryActionService(db: DbOrTransaction) { ), ) .returning(); - }); + }, { companyId: input.companyId }); if (!updated) { return retryUpsertSourceScoped(input, retryCount); } @@ -554,7 +554,7 @@ export function issueRecoveryActionService(db: DbOrTransaction) { .insert(issueRecoveryActions) .values(buildInsertValues(input, ownerType, now)) .returning(); - }); + }, { companyId: input.companyId }); return toReadModel(created!); } catch (error) { if (!isUniqueRecoveryActionConflict(error)) throw error; @@ -608,7 +608,7 @@ export function issueRecoveryActionService(db: DbOrTransaction) { }) .where(and(...predicates)) .returning(); - }); + }, { companyId: input.companyId }); return updated ? toReadModel(updated) : null; } diff --git a/server/src/services/legacy-execution-recovery.ts b/server/src/services/legacy-execution-recovery.ts index 427704de63f..c462cdf1fca 100644 --- a/server/src/services/legacy-execution-recovery.ts +++ b/server/src/services/legacy-execution-recovery.ts @@ -135,7 +135,7 @@ export async function terminalizeLegacyExecution(input: { }; if (issueId) await notifyUnsafeWorkspaceExportRecovery(tx, { companyId: current.companyId, issueId }); await notifyInstructionCollection(tx, { companyId: current.companyId, runId: current.id }); - await notifyExecutionRunWork(tx); + await notifyExecutionRunWork(tx, { companyId: current.companyId }); let [updated] = await tx .update(heartbeatRuns) .set({ diff --git a/server/src/services/native-runtime/native-failed-workspace-recovery.ts b/server/src/services/native-runtime/native-failed-workspace-recovery.ts index 5c3268f6afc..a1c6c914b33 100644 --- a/server/src/services/native-runtime/native-failed-workspace-recovery.ts +++ b/server/src/services/native-runtime/native-failed-workspace-recovery.ts @@ -65,7 +65,7 @@ export async function preserveNativeFailedWorkspace(db: Db, selected: Pick>'{systemRecovery,revision}' = ${String(previous.revision)}`)) .returning({ id: issueRecoveryActions.id }); return Boolean(updated); - }); + }, { companyId: source.companyId }); } export async function loadSystemRecovery(db: Db, source: WorkspaceRecoverySourceIdentity) { diff --git a/server/src/services/task-retry.ts b/server/src/services/task-retry.ts index e9f0bd4edbf..17075c34eb9 100644 --- a/server/src/services/task-retry.ts +++ b/server/src/services/task-retry.ts @@ -334,7 +334,7 @@ export async function requestTaskRetry(input: { details: { requestId: active?.id ?? id, sourceRunId: latest.id, attachedToExistingWork: Boolean(active || busy) } }); return { requestId: active?.id ?? id, accepted: true, stale: false, observedRunId: latest.id, chatActionId: intent.chatActionId }; - }); + }, { companyId: input.companyId }); } /** Chat owns admission and authorization. The task receipt follows that From 9a5c429043cd5927499aa16cb24cbdc6375ec7ac Mon Sep 17 00:00:00 2001 From: Dotta Date: Fri, 30 Oct 2026 17:33:00 -0500 Subject: [PATCH 35/74] fix(server): scope lifecycle hints from selected run fields Co-Authored-By: Paperclip --- server/src/services/heartbeat/run-lifecycle.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/services/heartbeat/run-lifecycle.ts b/server/src/services/heartbeat/run-lifecycle.ts index ac4e800f279..a1c4fa20fa3 100644 --- a/server/src/services/heartbeat/run-lifecycle.ts +++ b/server/src/services/heartbeat/run-lifecycle.ts @@ -1044,7 +1044,7 @@ export function createHeartbeatLifecycle(db: Db, dependencies: HeartbeatLifecycl if (previous && isHeartbeatRunTerminalStatus(status)) await notifyRetainedNativeSessionCleanup(tx, { companyId: previous.companyId, runId: previous.id }); if (isHeartbeatRunTerminalStatus(status)) await notifyDeliveryWork(tx, DELIVERY_QUEUES.accounting, companyId); - await notifyExecutionRunWork(tx, previous ? { companyId: previous.companyId, agentId: previous.agentId, runId: previous.id, + await notifyExecutionRunWork(tx, previous ? { companyId: previous.companyId, runId: previous.id, issueId: previous.nativeIssueId ?? previous.issueId ?? readNonEmptyString(parseObject(previous.contextSnapshot).issueId) ?? undefined } : undefined); if (previous) await notifyChatRunMilestones(tx, { companyId: previous.companyId, runId: previous.id, issueId: previous.nativeIssueId ?? previous.issueId ?? readNonEmptyString(parseObject(previous.contextSnapshot).issueId) ?? undefined }); From 2cc2f9ae8cbf355ffba46596b38ce8eb7a04270f Mon Sep 17 00:00:00 2001 From: Dotta Date: Fri, 30 Oct 2026 17:39:30 -0500 Subject: [PATCH 36/74] fix(server): serialize direct routine status hooks Co-Authored-By: Paperclip --- doc/plans/2026-10-30-routine-status-sleep.md | 4 ++- .../routine-execution-status-work.test.ts | 34 +++++++++++++++++++ server/src/services/routine-run-status.ts | 8 ++++- 3 files changed, 44 insertions(+), 2 deletions(-) diff --git a/doc/plans/2026-10-30-routine-status-sleep.md b/doc/plans/2026-10-30-routine-status-sleep.md index 271d0b544a7..e852bf94861 100644 --- a/doc/plans/2026-10-30-routine-status-sleep.md +++ b/doc/plans/2026-10-30-routine-status-sleep.md @@ -18,7 +18,9 @@ and belong to a routine in that company. This is the existing routine status owner contract. Origin labels can change when a plugin declaration changes or is removed; retained reciprocal links preserve historical execution custody. Repair locks the routine and task and rechecks the linked invocation before writing. -It preserves skipped/coalesced history and unrelated dispatch failures. A task +Direct route hooks also lock the execution task and linked invocation in their +own transaction, nested under caller transactions when present. An older hook +cannot write after a newer task status has committed. It preserves skipped/coalesced history and unrelated dispatch failures. A task that resumes clears an execution-task transient failure through the same owner used by native completion and issue routes. diff --git a/server/src/__tests__/routine-execution-status-work.test.ts b/server/src/__tests__/routine-execution-status-work.test.ts index 53d7e5c0225..e6bd818852c 100644 --- a/server/src/__tests__/routine-execution-status-work.test.ts +++ b/server/src/__tests__/routine-execution-status-work.test.ts @@ -174,6 +174,40 @@ const support = external ? { supported: true } : await getEmbeddedPostgresTestSu await db.transaction(async tx => { await notifyRoutineExecutionStatus(tx, otherRun.linkedIssueId!); }); await projected(otherRun.id, "completed"); await quiet(); expect(errors).toEqual([]); }); + it("serializes a direct status hook with later task changes before projecting the routine", async () => { + const f = await fixture(), run = await dispatch(f); await start(); await quiet(); + await db.update(issues).set({ status: "done" }).where(eq(issues.id, run.linkedIssueId!)); + let entered!: () => void, release!: () => void; + const enteredGate = new Promise(resolve => { entered = resolve; }); + const releaseGate = new Promise(resolve => { release = resolve; }); + const holding = db.transaction(async tx => { + await tx.select().from(routineRuns).where(eq(routineRuns.id, run.id)).for("update"); + entered(); await releaseGate; + }); + let direct: Promise | undefined, changing: Promise | undefined; + let directPid: number | undefined; + try { + await enteredGate; direct = f.service.syncRunStatusForIssue(run.linkedIssueId!); + await vi.waitFor(async () => { + const rows = await db.execute(sql`select pid from pg_stat_activity + where datname = current_database() and wait_event_type = 'Lock' + and (query like 'update "routine_runs"%' or query like 'select %"routine_runs"%')`); + expect(rows).toHaveLength(1); directPid = Number(rows[0]!.pid); + }, { timeout: 5000 }); + await expect(db.transaction(tx => tx.select({ id: issues.id }).from(issues) + .where(eq(issues.id, run.linkedIssueId!)).for("update", { noWait: true }))) + .rejects.toMatchObject({ cause: expect.objectContaining({ code: "55P03" }) }); + changing = issueService(db).update(run.linkedIssueId!, { status: "blocked" }); + await vi.waitFor(async () => { + const rows = await db.execute(sql`select count(*)::int as count from pg_stat_activity + where datname = current_database() and wait_event_type = 'Lock' + and ${directPid}::int = any(pg_blocking_pids(pid))`); + expect(Number(rows[0]?.count)).toBeGreaterThan(0); + }, { timeout: 5000 }); + release(); await holding; await direct; await changing; + await projected(run.id, "failed"); await quiet(); expect(errors).toEqual([]); + } finally { release(); await holding; await direct; await changing; } + }); it("waits for the real outer transaction and forgets a rolled-back nested status write", async () => { const f = await fixture(), run = await dispatch(f); await start(); await quiet(); let release!: () => void, entered!: () => void; diff --git a/server/src/services/routine-run-status.ts b/server/src/services/routine-run-status.ts index 2a182def8f9..46a2bae4cf5 100644 --- a/server/src/services/routine-run-status.ts +++ b/server/src/services/routine-run-status.ts @@ -51,6 +51,10 @@ export async function finalizeRoutineRun(executor: Db, runId: string, patch: Par } export async function syncRoutineRunStatusForIssue(executor: Db, issueId: string) { + return executor.transaction(tx => syncLockedRoutineRunStatusForIssue(tx as unknown as Db, issueId)); +} + +async function syncLockedRoutineRunStatusForIssue(executor: Db, issueId: string) { const issue = await executor .select({ id: issues.id, @@ -60,7 +64,8 @@ export async function syncRoutineRunStatusForIssue(executor: Db, issueId: string originRunId: issues.originRunId, }) .from(issues) - .where(eq(issues.id, issueId)) + .where(and(eq(issues.id, issueId), eq(issues.originKind, "routine_execution"))) + .for("update") .then((rows) => rows[0] ?? null); if (!issue || issue.originKind !== "routine_execution" || !issue.originRunId) return null; const run = await executor @@ -75,6 +80,7 @@ export async function syncRoutineRunStatusForIssue(executor: Db, issueId: string .innerJoin(routines, and(eq(routines.id, routineRuns.routineId), eq(routines.companyId, issue.companyId))) .where(and(eq(routineRuns.id, issue.originRunId), eq(routineRuns.companyId, issue.companyId), eq(routineRuns.linkedIssueId, issue.id))) + .for("update", { of: routineRuns }) .then((rows) => rows[0] ?? null); if (!run || ["received", "skipped", "coalesced"].includes(run.status)) return null; if (issue.status === "done") { From 6b9b421905ae3691e82626c9cf5c7f06a23ddb49 Mon Sep 17 00:00:00 2001 From: Dotta Date: Fri, 30 Oct 2026 17:40:27 -0500 Subject: [PATCH 37/74] fix: accept authentication URLs drained after their result Keep validation and process retirement intact when stdout is delivered before stderr. Reproduce both valid and invalid late URLs with a real child process. Co-Authored-By: Paperclip --- .../src/drivers/acpx/antigravity-auth.test.ts | 21 ++++++++++++------- .../src/drivers/acpx/antigravity-auth.ts | 4 +++- 2 files changed, 17 insertions(+), 8 deletions(-) diff --git a/packages/paperclip-runner/src/drivers/acpx/antigravity-auth.test.ts b/packages/paperclip-runner/src/drivers/acpx/antigravity-auth.test.ts index 897a4d675be..af78eef13bb 100644 --- a/packages/paperclip-runner/src/drivers/acpx/antigravity-auth.test.ts +++ b/packages/paperclip-runner/src/drivers/acpx/antigravity-auth.test.ts @@ -45,9 +45,9 @@ reader.on('line', line => { if(request.method==='initialize') { process.stdout.write(JSON.stringify({jsonrpc:'2.0',id:request.id,result:{protocolVersion:1,agentInfo:{name:'antigravity-acp',version:'1.4.3'},agentCapabilities:{_meta:{antigravity:{paperclipAdmission:1}}},authMethods:[{id:'oauth-personal'}]}})+'\\n'); } else if(request.method==='authenticate') { - const url = mode==='bad-url' ? 'https://example.invalid/steal' : ${JSON.stringify(authorizationUrl())}; + const url = mode.startsWith('bad-url') ? 'https://example.invalid/steal' : ${JSON.stringify(authorizationUrl())}; const message='Open the following link to authenticate the ACP server: '+url+'\\n'; - process.stderr.write(message.slice(0,42));process.stderr.write(message.slice(42)); + if (!mode.endsWith('url-after-result')) {process.stderr.write(message.slice(0,42));process.stderr.write(message.slice(42));} if(mode==='blocked' || mode==='bad-url' || mode==='stubborn' || mode==='retained') return; if(mode==='error') {process.stdout.write(JSON.stringify({jsonrpc:'2.0',id:request.id,error:{code:-32000,message:'do not expose fixture-refresh-token'}})+'\\n');return;} fs.mkdirSync(path.join(home,'antigravity-acp'),{mode:0o700}); @@ -55,7 +55,14 @@ reader.on('line', line => { process.stdout.write(JSON.stringify({jsonrpc:'2.0',id:request.id,result:{}})+'\\n'); } else {throw new Error('inference/session request forbidden');} }); -reader.on('close',()=>mode==='stubborn'||mode==='retained'?setInterval(()=>{},1000):process.exit(0)); +reader.on('close',()=> { + if (mode.endsWith('url-after-result')) { + const url = mode.startsWith('bad-url') ? 'https://example.invalid/steal' : ${JSON.stringify(authorizationUrl())}; + // EOF proves the client received the authenticate result before this stderr. + process.stderr.write('Open the following link to authenticate the ACP server: '+url+'\\n', () => process.exit(0)); + } else if(mode==='stubborn'||mode==='retained') setInterval(()=>{},1000); + else process.exit(0); +}); `); setup.verify.mockResolvedValue({ openCommand: async () => ({ spawn: setup.spawn, close: setup.close }) }); setup.spawn.mockImplementation((_args, options) => { @@ -95,8 +102,8 @@ describe("Antigravity setup-only Google authentication", () => { // The login predicate keeps its five-second bound. Allow ten more seconds // for fixture startup and the durable mocked credential checks on a busy CI worker. - it("performs native authenticate, retires it, verifies identity, and preserves refresh rotation without repeated status network calls", async () => { - const home = await fixture(), fetch = googleIdentity(); vi.stubGlobal("fetch", fetch); + it.each(["success", "url-after-result"])("performs native authenticate, retires it, verifies identity, and preserves refresh rotation without repeated status network calls (%s)", async mode => { + const home = await fixture(mode), fetch = googleIdentity(); vi.stubGlobal("fetch", fetch); const login = startAntigravityBrowserLogin(home); await vi.waitFor(() => expect(login.outcome).toBe("success"), { timeout: 5_000 }); expect(login.authorizationUrl).toBe(authorizationUrl()); @@ -118,13 +125,13 @@ describe("Antigravity setup-only Google authentication", () => { expect(fetch).toHaveBeenCalledTimes(6); }, 15_000); - it.each(["error", "bad-url"])("rejects %s without leaking provider text or calling Google", async mode => { + it.each(["error", "bad-url", "bad-url-after-result"])("rejects %s without leaking provider text or calling Google", async mode => { const home = await fixture(mode), fetch = vi.fn(); vi.stubGlobal("fetch", fetch); const login = startAntigravityBrowserLogin(home); await vi.waitFor(() => expect(login.outcome).toBe("failure"), { timeout: 5_000 }); await vi.waitFor(() => expect(setup.close).toHaveBeenCalledOnce(), { timeout: 5_000 }); expect(login.error).not.toContain("fixture"); - if (mode === "bad-url") expect(login.authorizationUrl).toBeUndefined(); + if (mode.startsWith("bad-url")) expect(login.authorizationUrl).toBeUndefined(); expect(fetch).not.toHaveBeenCalled(); }); diff --git a/packages/paperclip-runner/src/drivers/acpx/antigravity-auth.ts b/packages/paperclip-runner/src/drivers/acpx/antigravity-auth.ts index 2921fd141db..2561be31d56 100644 --- a/packages/paperclip-runner/src/drivers/acpx/antigravity-auth.ts +++ b/packages/paperclip-runner/src/drivers/acpx/antigravity-auth.ts @@ -203,7 +203,9 @@ export function startAntigravityBrowserLogin(home: string): AntigravityBrowserLo if (!line.startsWith(AUTH_PREFIX)) continue; try { const url = validateAntigravityAuthorizationUrl(line.slice(AUTH_PREFIX.length)); - if (phase !== 2 || state.authorizationUrl !== undefined) throw new Error(LOGIN_ERROR); + // stdout and stderr are separate pipes: the authenticate result + // can arrive before its authorization URL is drained from stderr. + if ((phase !== 2 && phase !== 3) || state.authorizationUrl !== undefined) throw new Error(LOGIN_ERROR); state.authorizationUrl = url; } catch { fail(); } } From 938e22aa21cbd6c887f2451d9a339b2db44d11f4 Mon Sep 17 00:00:00 2001 From: Dotta Date: Fri, 30 Oct 2026 17:47:09 -0500 Subject: [PATCH 38/74] test: prove timer deadlines survive settled drain holds Expect committed producers to release idle admission while retaining a resume deadline, then verify actual dispatch and final quiet after resume. Co-Authored-By: Paperclip --- server/src/__tests__/agent-timer-deadlines.test.ts | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/server/src/__tests__/agent-timer-deadlines.test.ts b/server/src/__tests__/agent-timer-deadlines.test.ts index 4c4c0d324a9..f925fb8dfd7 100644 --- a/server/src/__tests__/agent-timer-deadlines.test.ts +++ b/server/src/__tests__/agent-timer-deadlines.test.ts @@ -597,8 +597,10 @@ describe.skipIf(!support.supported)("agent timer owned deadlines", () => { await agentService(db).update(f.agent.id, { runtimeConfig: { heartbeat: { enabled: true, intervalSec: 600 } } }); await vi.advanceTimersByTimeAsync(2000); - - expect(idleWorkSnapshot().active).toBe(1); + // The producer committed: retain its resume deadline without keeping the + // instance busy throughout the drain. Uncertain writes still hold admission. + expect(idleWorkSnapshot().active).toBe(0); + expect(coordinator!.nextWakeAt()).toBeGreaterThan(Date.now()); const select = vi.spyOn(db, "select"), execute = vi.spyOn(db, "execute"), transaction = vi.spyOn(db, "transaction"); try { await vi.advanceTimersByTimeAsync(60_000); @@ -608,6 +610,11 @@ describe.skipIf(!support.supported)("agent timer owned deadlines", () => { stopTaskDrain(); await vi.advanceTimersByTimeAsync(1000); await vi.waitFor(() => expect(coordinator!.nextWakeAt()).toBe(now.getTime() + 600_000), { timeout: 3000 }); + await vi.advanceTimersByTimeAsync(coordinator!.nextWakeAt()! - Date.now()); + await vi.waitFor(() => expect(f.enqueueWakeup).toHaveBeenCalledOnce()); + expect(f.enqueueWakeup).toHaveBeenCalledWith(f.agent.id, expect.objectContaining({ source: "timer", reason: "heartbeat_timer" })); + await agentService(db).update(f.agent.id, { runtimeConfig: { heartbeat: { enabled: false } } }); + await quiet(); }); it("does not inspect timer rows during startup drain, then reconstructs them on resume", async () => { From a12f68ef134d10b69357ec6d193ccf56b0db971f Mon Sep 17 00:00:00 2001 From: Dotta Date: Fri, 30 Oct 2026 17:50:38 -0500 Subject: [PATCH 39/74] fix: preserve scoped status-card work across mutations and recovery Complete transactional source coverage, cached feature gating, startup scope reconstruction, off-page relative query expiry and busy-lock retry. Verify producer, persistence, failure, budget resume, compatibility and startup paths with isolated PostgreSQL and focused tests. Co-Authored-By: Paperclip --- doc/plans/2026-10-30-status-cards-sleep.md | 113 ++++++++-- .../__tests__/company-search-service.test.ts | 14 +- .../server-startup-feedback-export.test.ts | 30 +++ .../status-card-evaluation-work.test.ts | 209 +++++++++++++++++- .../status-card-notifications.test.ts | 31 +++ .../modules/wake-queue/adapters/postgres.ts | 2 + .../adapters/queued-comment-postgres.ts | 3 + server/src/routes/execution-workspaces.ts | 6 +- server/src/services/agent-conversations.ts | 3 + server/src/services/budgets.ts | 3 + .../telegram/attachment-failure.ts | 2 + server/src/services/company-search.ts | 15 +- server/src/services/documents.ts | 5 + server/src/services/execution-workspaces.ts | 5 +- server/src/services/heartbeat/queue.ts | 3 + server/src/services/heartbeat/scheduling.ts | 6 + .../src/services/issue-thread-interactions.ts | 63 +++--- server/src/services/issues.ts | 34 ++- ...ative-status-reassessment-notifications.ts | 4 +- server/src/services/projects.ts | 3 + server/src/services/status-card-evaluation.ts | 14 +- server/src/services/status-cards.ts | 53 ++--- server/src/services/task-watchdogs.ts | 6 +- server/src/services/workspace-runtime.ts | 22 +- 24 files changed, 524 insertions(+), 125 deletions(-) create mode 100644 server/src/__tests__/status-card-notifications.test.ts diff --git a/doc/plans/2026-10-30-status-cards-sleep.md b/doc/plans/2026-10-30-status-cards-sleep.md index dcd6c5ca226..2af51ff65cf 100644 --- a/doc/plans/2026-10-30-status-cards-sleep.md +++ b/doc/plans/2026-10-30-status-cards-sleep.md @@ -1,33 +1,102 @@ # Status-card evaluation deadlines -Baseline: `76fde891ccc29283b3f5c8dcdce089c5b3d937d1`. +Status-card evaluation uses existing cards, updates and issues as durable truth. +`index.ts` registers a typed worker. The status-card owner controls evaluation, +generation claims, assignment through the ordinary issue wake API, and failed +assignment cancellation. Shared delivery infrastructure controls transaction +settlement, coalescing, retries, admission drain, standby and shutdown. -Status-card ownership covers evaluation policy, card claims, generation, ordinary assignment wakeups, and failed-assignment cancellation. `index.ts` registers a typed capability. Shared delivery/deadline infrastructure owns transaction settlement, coalescing, admission/drain, retries and shutdown. Existing cards, updates and issues remain durable truth; no timer table or sleep-safety exemption is introduced. +Manual cards have no automatic obligation. Interval cards retain the requested +evaluation cadence. Reactive cards evaluate committed source changes and retain +only an actual debounce, hourly rate, daily budget, active-hour or saved-query +membership deadline. Unchanged cards disarm. A changed experimental setting +invalidates the worker's cached feature gate. Other source writes do not reread +settings. Startup reconstructs every disarmed reactive scope together with any +concurrent company-scoped hint, without moving saved eligibility/retry deadlines. -Manual cards have no automatic obligation. Interval cards retain their requested evaluation cadence. Reactive cards evaluate committed source changes, retain the actual debounce/rate/budget/active-hours deadline while relevant changes remain, and disarm once unchanged. Saved relative-search windows retain their actual membership expiry; the duration interpretation stays with company search. Startup reconstructs existing work without advancing business timestamps. +Company search reports the earliest matching `updatedWithin` expiry through an +optional internal callback in its existing aggregate query. It includes rows +outside the returned page: an expiring priority-ranked row ahead of an offset +can change the watched set. The callback exposes only a normalized deadline; +ordinary search consumers and API responses retain their existing contract. +Overlapping query scopes can require an evaluation when one query expires even +if another still retains the same issue; no generation occurs without a filtered +card change. No recurring scan replaces the expired query obligation. -Scheduled reactive source events can preempt a future evaluation. A feature-owned transaction advisory lock serializes scheduler evaluation across connections while existing card/issue services perform their own committed writes. The lock holder has no card row lock, avoiding self-deadlock. Existing next-evaluation CAS, generating-issue ownership and issue idempotency keys remain required, including for manual refreshes and older server instances that do not use the advisory lock. The persisted five-minute claim remains the failure retry after a scheduled refresh cannot finish. Internal timers do not establish a Cloud wake alarm. +## Claims and recovery -## Producer audit (in progress) +A feature-owned PostgreSQL transaction advisory lock serializes scheduled +per-card evaluation across connections. Its transaction holds no card row lock +while existing issue/card services commit through their own transactions. The +lock remains held through actual refresh settlement. Existing card deadline CAS, +generating-issue ownership and issue idempotency keys remain in force. Manual +refresh and older instances continue to use those existing guards. A busy lock +retains a company-scoped retry; consuming a hint is not successful evaluation. +A failed scheduled refresh retains the existing five-minute durable claim. -| Source | Owning committed boundary | State | -| --- | --- | --- | -| Card create/config/archive/delete, compile, query/summary writeback, evaluation | status-card mutation transactions | wired; ordinary lifecycle exercised | -| Experimental feature flag | instance settings update transaction | wired; disable exercised | -| Company pause/archive/reactivate/delete | company owning transactions | wired; lifecycle acceptance remains | -| Ordinary issue create/update/remove | issue service transactions | wired; create/change exercised | -| Comments create/delete/tombstone | common insert and issue mutation transactions | wired; comment acceptance remains | -| Review state / chat-run milestones | issue-review-state notification boundary | wired; exact caller coverage audit remains | -| Execution-control writes/run settlement | execution-control transaction helpers | wired; exact caller coverage audit remains | -| Agent deletion/membership reassignment, issue tree transitions, title/prefix | owner transaction | wired; representative acceptance remains | -| Workspace selection/binding, plugin/provider workspace, native recovery/reclaim, active watchdog, retry | owner transactions where uncovered by shared notifications | partially wired; remaining per-write audit required | -| Raw workspace cleanup/routes, conversation activation/settlement, queue paths | direct writes and adjacent owner notifications | exact coverage incomplete; not acceptance-ready | -| Imports/independent writers/replicas | supported service writes and startup reconstruction | full ingress contract audit remains | +Once a generation exists, the normal issue assignment authority remains its +owner. Assignment failure cancels that issue and finalizes the card's failed +ledger entry. The user's ordinary refresh action can retry it. Restart recovery +must not repeat an already completed generation. No new timer table, automatic +write replay, execution-authority exemption or sleep-safety relaxation is added. -Process-local notifications cover writes using the installed database owner. They do not notify an unrelated SQL writer or another server process; those supported paths need explicit ingress or reconstruction coverage. No claim of cross-replica delivery, host suspension, or fleet savings is made by the current tests. +## Supported producer coverage -## Acceptance evidence and remaining gates +All hints are registered before the relevant write within its owning transaction. +Nested service writes share outer commit ownership. Company-scoped hints batch +reactive reconciliation; ordinary execution writes without known scope retain +all existing execution consumers without sending an instance-wide card hint. -The original status-card PostgreSQL suite passed25/25. New real PostgreSQL tests exercise ordinary card compilation/writeback, source-change debounce, persisted assignment at a controlled provider boundary, result persistence, quiet SQL after completion, earlier/cancelled deadlines, nested outer commit/rollback and drain/resume. The provider boundary is simulated; no model runs. Additional race, restart, failure, active-hour/rate/budget persistence, multi-company and producer-class coverage remain before PR delivery. +| Producer | Notification boundary and coverage | +| --- | --- | +| Card create, configuration, archive/restore, delete, compile, query/summary result, evaluation claim | Status-card mutation transactions; stalled issue finalization signals before releasing the card claim | +| Experimental status-card setting | Instance experimental-settings transaction; full reconstruction is appropriate for this instance-wide setting | +| Company pause/archive/reactivate/delete | Company lifecycle/deletion transactions and budget-owned company hard-stop/resume transactions | +| Issue create/update/delete and label changes | Ordinary issue transactions; label deletion signals its known company before cascades; title and prefix changes signal their existing owner transactions | +| Issue checkout/adoption/release/reset and environment-selection recency | Existing issue transaction, or a transaction around the previously standalone write; company comes from the existing issue/run read | +| Comment create, edit, delete, tombstone, import and recency | Common comment insert; outer root comment transaction; queued-comment and Telegram edit transactions; existing metadata-upgrade path; remove/tombstone transactions | +| Issue import, comment import, document import | One notification per affected company in the existing batch transaction; no per-row notification lookup | +| Document create/update/restore/delete and search membership | Document owner transactions; only the existing company is passed, without copying search predicates into producers | +| Review, interaction and execution transitions | Existing issue-review-state and native-status-reassessment boundaries; scoped execution-control helpers; interaction recency touches own a transaction when called after settlement | +| Run admission, retries, monitors, watchdog/recovery and native settlement/reclaim | Existing scoped review/milestone/status/execution-control hooks; explicit notifications at the uncovered issue execution bind, legacy claim, monitor claim/release and review-participant recovery transaction | +| Agent deletion/access reassignment and project deletion/privacy cascades | Existing company-scoped owner transactions before foreign-key or recursive issue changes | +| Workspace selection/binding, plugin/provider workspace and cleanup/audit touches | Existing owner transactions, including standalone cleanup and audit recency writes now enclosed with their intent | +| Skill-test harness expiry and task-watchdog fingerprint touch | Existing owner mutation transaction; no feature policy is moved into the writer | -Shared deadline regressions reproduced and corrected a settled-write admission hold during drain. The correction releases only settled holds, retains an in-memory resume wake, and leaves ambiguous transaction intent fenced by the database coordinator. Unit coverage includes first write during drain, failure entering drain, preserved future deadlines, warm standby and ambiguous commit settlement. +The audit includes direct issue/comment mutations, raw recursive privacy updates, +bulk insert helpers, label/project/agent cascades, linked document search content +and timestamp-dependent search membership. Metadata-only writes that do not +change the fingerprint, matching fields or visibility do not create an evaluation +obligation: examples include run identity linkage, workspace IDs without a +recency change, document locks and source-trust receipts without searched content +changes. Routine and status-summary documents that have no issue-document link +do not change issue search membership. + +This follows the existing [single-active-writer contract](../idle-sleep-safety.md). +Supported services, import/recovery writers, caller-owned transactions and startup +reconstruction participate. Independent SQL writers or another active server +process require explicit wake ingress; process-local hints do not establish +cross-replica notification delivery. The advisory lock supplies competing +scheduler exclusion, not cross-process event transport. Internal deadlines do +not establish a Cloud wake alarm or prove host/database suspension. + +## Verification + +Real PostgreSQL acceptance uses ordinary card create/compile and authorized +summarizer writeback. The assignment boundary is controlled: it persists a wake, +and the test explicitly performs the authorized result write; no model runs. +Checks cover empty/manual/reactive/disabled quiet SQL, first source change, +debounce and durable wake/result, earlier/cancelled deadlines, outer commit and +rollback, drain/resume, competing connections, busy locks, restart eligibility, +active hours/rate/budget gates, multiple companies, budget-owned company resume, +ignored activity, document/comment membership, relative query expiry including +an off-page candidate, failed assignment cleanup/manual retry and the persisted +five-minute internal-failure retry across restart/drain. + +Shared coordinator regressions also cover warm standby, ambiguous transaction +intent, failure entering drain and different domain/coordinator gates. The paused +worker releases only settled holds and retains a future in-memory resume wake. +An expired saved deadline cannot schedule an immediate skipped-attempt loop. +Existing startup, status-card, company-search, execution and monitor suites cover +the production wiring and unchanged consumers. Exact commands and results belong +in the PR verification record; this document does not claim deployed savings. diff --git a/server/src/__tests__/company-search-service.test.ts b/server/src/__tests__/company-search-service.test.ts index 5d9e53260fb..7dffe3d6a90 100644 --- a/server/src/__tests__/company-search-service.test.ts +++ b/server/src/__tests__/company-search-service.test.ts @@ -1,3 +1,4 @@ +import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js"; import { randomUUID } from "node:crypto"; import { sql } from "drizzle-orm"; import { afterAll, afterEach, beforeAll, describe, expect, it } from "vitest"; @@ -28,12 +29,13 @@ import { import { issueReadSqlCondition, projectReadSqlCondition } from "../services/authorization.js"; import { issueService } from "../services/issues.js"; -const embeddedPostgresSupport = await getEmbeddedPostgresTestSupport(); +const externalDatabaseUrl = resolveExternalTestDatabaseUrl("PAPERCLIP_COMPANY_SEARCH_TEST_DATABASE_URL"); +const embeddedPostgresSupport = externalDatabaseUrl ? { supported: true } : await getEmbeddedPostgresTestSupport(); const describeEmbeddedPostgres = embeddedPostgresSupport.supported ? describe : describe.skip; if (!embeddedPostgresSupport.supported) { console.warn( - `Skipping embedded Postgres company search tests on this host: ${embeddedPostgresSupport.reason ?? "unsupported environment"}`, + `Skipping embedded Postgres company search tests on this host: ${("reason" in embeddedPostgresSupport ? embeddedPostgresSupport.reason : "unsupported environment")}`, ); } @@ -80,8 +82,11 @@ describeEmbeddedPostgres("companySearchService", () => { let tempDb: Awaited> | null = null; beforeAll(async () => { - tempDb = await startEmbeddedPostgresTestDatabase("paperclip-company-search-"); - db = createDb(tempDb.connectionString); + if (!externalDatabaseUrl) tempDb = await startEmbeddedPostgresTestDatabase("paperclip-company-search-"); + db = createDb(externalDatabaseUrl ?? tempDb!.connectionString, { maxConnections: 6 }); + // External suites can reuse this dedicated database sequentially. Match + // the fresh embedded fixture before this suite creates its own rows. + if (externalDatabaseUrl) await db.execute(sql`TRUNCATE TABLE companies CASCADE`); svc = companySearchService(db); await db.execute(sql.raw("CREATE EXTENSION IF NOT EXISTS pg_trgm")); }, 20_000); @@ -100,6 +105,7 @@ describeEmbeddedPostgres("companySearchService", () => { }); afterAll(async () => { + await db?.$client.end({ timeout: 1 }); await tempDb?.cleanup(); }); diff --git a/server/src/__tests__/server-startup-feedback-export.test.ts b/server/src/__tests__/server-startup-feedback-export.test.ts index bbb020c865d..3008ee21f9c 100644 --- a/server/src/__tests__/server-startup-feedback-export.test.ts +++ b/server/src/__tests__/server-startup-feedback-export.test.ts @@ -1,3 +1,4 @@ +import { statusCardEvaluationService, registerStatusCardEvaluationWork } from "../services/status-card-evaluation.js"; import { nativeHarnessBackupExpirationService } from "../services/native-runtime/native-harness-backup-expiration.js"; import { skillVersionRetentionService } from "../services/skill-version-retention.js"; import { approvalQueueSeedingService } from "../services/approval-queue-seeding-work.js"; @@ -527,6 +528,13 @@ vi.mock("../services/native-runtime/native-question-bridge.js", () => ({ validateNativeQuestionResponseInput: vi.fn(), })); +vi.mock("../services/status-card-evaluation.js", () => ({ + statusCardEvaluationService: vi.fn(() => ({ run: vi.fn(async () => {}), next: vi.fn(async () => null), changed: vi.fn() })), + registerStatusCardEvaluationWork: vi.fn((_coordinator, service, canRun) => ({ + ready: canRun() ? service.run(new AbortController().signal) : Promise.resolve(), wake: vi.fn(), stop: vi.fn(async () => {}), + })), +})); + vi.mock("../services/secret-proposals.js", () => ({ createSecretProposalsService: vi.fn(() => ({ expiryDeadlines: vi.fn(async () => []), @@ -790,6 +798,28 @@ describe("startServer feedback export wiring", () => { expect(deliveryWorkErrors.filter(error => error.queue === DELIVERY_QUEUES.skillVersionRetention)).toEqual([]); }); + it.each([false, true])("registers status-card evaluation only with maintenance and never ticks it from the heartbeat loop (%s)", async enabled => { + loadConfigMock.mockReturnValue(buildTestConfig({ heartbeatSchedulerEnabled: enabled, heartbeatSchedulerIntervalMs: 1000 })); + let tick: (() => void) | undefined; + const timer = vi.spyOn(globalThis, "setInterval").mockImplementation(((callback: () => void, interval: number) => { + if (interval === 1000) tick = callback; + return 1 as unknown as ReturnType; + }) as typeof setInterval); + try { + await startServer(); await settleStartupHousekeeping(); + expect(registerStatusCardEvaluationWork).toHaveBeenCalledTimes(enabled ? 1 : 0); + expect(statusCardEvaluationService).toHaveBeenCalledTimes(enabled ? 1 : 0); + if (enabled) { + const service = vi.mocked(statusCardEvaluationService).mock.results.at(-1)!.value; + expect(service.run).toHaveBeenCalledOnce(); + expect(tick).toBeDefined(); + for (let i = 0; i < 3; i++) { tick!(); await settleStartupHousekeeping(); } + expect(service.run).toHaveBeenCalledOnce(); + } + expect(deliveryWorkErrors.filter(error => error.queue === DELIVERY_QUEUES.statusCardEvaluation)).toEqual([]); + } finally { timer.mockRestore(); } + }); + it.each([false, true])("registers response recovery and its shutdown subscription only when maintenance is enabled (%s)", async enabled => { loadConfigMock.mockReturnValue(buildTestConfig({ heartbeatSchedulerEnabled: enabled })); await startServer(); await settleStartupHousekeeping(); diff --git a/server/src/__tests__/status-card-evaluation-work.test.ts b/server/src/__tests__/status-card-evaluation-work.test.ts index 427ad3385eb..4f369cd76e9 100644 --- a/server/src/__tests__/status-card-evaluation-work.test.ts +++ b/server/src/__tests__/status-card-evaluation-work.test.ts @@ -1,7 +1,9 @@ +import { budgetService } from "../services/budgets.js"; +import { documentService } from "../services/documents.js"; import { randomUUID } from "node:crypto"; import { eq, sql } from "drizzle-orm"; import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; -import { activityLog, agentWakeupRequests, agents, companies, companyMemberships, createDb, documents, +import { activityLog, agentWakeupRequests, agents, approvals, budgetIncidents, budgetPolicies, costEvents, companies, companyMemberships, createDb, documents, heartbeatRuns, instanceSettings, issueComments, issues, statusCards, statusCardUpdates, documentRevisions } from "@paperclipai/db"; import { companySearchQuerySchema, statusCardRefreshPolicySchema } from "@paperclipai/shared"; import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js"; @@ -22,6 +24,7 @@ const support = external ? { supported: true } : await getEmbeddedPostgresTestSu let draining = false; let companyId: string, agentId: string; const errors: unknown[] = []; + let compiledQuery = companySearchQuerySchema.parse({ scope: "issues", status: ["blocked"] }); const actor = { agentId: null, userId: "board-user" }; const policy = (mode: "manual" | "reactive" | "interval" = "reactive") => statusCardRefreshPolicySchema.parse({ mode, debounceSeconds: 60, intervalMinutes: 15, @@ -29,7 +32,8 @@ const support = external ? { supported: true } : await getEmbeddedPostgresTestSu // Controlled provider boundary: assignment persists a queued wake, and tests // explicitly run the authorized summarizer writeback. No model/provider runs. const wakeup = vi.fn(async (id: string, options: { payload?: Record | null; reason?: string | null }) => { - return db.insert(agentWakeupRequests).values({ companyId, agentId: id, source: "assignment", + const [agent] = await db.select({ companyId: agents.companyId }).from(agents).where(eq(agents.id, id)); + return db.insert(agentWakeupRequests).values({ companyId: agent!.companyId, agentId: id, source: "assignment", reason: options.reason, payload: options.payload, status: "queued" }).returning(); }); @@ -38,7 +42,8 @@ const support = external ? { supported: true } : await getEmbeddedPostgresTestSu db = createDb(external ?? temporary!.connectionString, { maxConnections: 6, idleTimeoutSeconds: 0, maxLifetimeSeconds: 0 }); }, 30_000); beforeEach(async () => { - companyId = randomUUID(); agentId = randomUUID(); draining = false; errors.length = 0; wakeup.mockClear(); + companyId = randomUUID(); agentId = randomUUID(); draining = false; errors.length = 0; wakeup.mockReset(); + compiledQuery = companySearchQuerySchema.parse({ scope: "issues", status: ["blocked"] }); await db.insert(companies).values({ id: companyId, name: "Status", issuePrefix: `S${companyId.slice(0, 6)}` }); await db.insert(agents).values({ id: agentId, companyId, name: "Summarizer", role: "general", status: "idle", adapterType: "process", adapterConfig: {} }); await instanceSettingsService(db).updateExperimental({ enableStatusCards: true }); @@ -48,6 +53,7 @@ const support = external ? { supported: true } : await getEmbeddedPostgresTestSu afterEach(async () => { await coordinator?.stop(); coordinator = undefined; vi.useRealTimers(); vi.restoreAllMocks(); + await db.delete(budgetIncidents); await db.delete(budgetPolicies); await db.delete(approvals); await db.delete(costEvents); await db.delete(agentWakeupRequests); await db.delete(statusCardUpdates); await db.delete(statusCards); await db.delete(documentRevisions); await db.delete(documents); await db.delete(activityLog); await db.delete(issueComments); await db.delete(issues); await db.delete(heartbeatRuns); @@ -80,14 +86,14 @@ const support = external ? { supported: true } : await getEmbeddedPostgresTestSu await db.update(issues).set({ checkoutRunId: run!.id }).where(eq(issues.id, generationIssueId)); const writer = { agentId, runId: run!.id }; const cards = statusCardService(db); - if (compile) await cards.writeQuery(cardId, { queries: [companySearchQuerySchema.parse({ scope: "issues", status: ["blocked"] })], + if (compile) await cards.writeQuery(cardId, { queries: [compiledQuery], title: "Blockers", changeSummary: "Compiled", generationIssueId }, writer); await cards.writeSummary(cardId, { markdown: "Current blockers", changeSummary: "Updated", generationIssueId }, writer); await issueService(db).update(generationIssueId, { status: "done" }); } - async function card(mode: "manual" | "reactive" | "interval" = "reactive") { + async function card(mode: "manual" | "reactive" | "interval" = "reactive", refreshPolicy = policy(mode)) { const cards = statusCardService(db); - const created = await cards.create(companyId, { interestPrompt: "Blockers", titlePinned: false, agentId, refreshPolicy: policy(mode) }, actor); + const created = await cards.create(companyId, { interestPrompt: "Blockers", titlePinned: false, agentId, refreshPolicy }, actor); const compile = await cards.requestCompile(created.id, actor); await writeback(created.id, compile.generatingIssue.id, true); return cards.getById(created.id).then(row => row!); @@ -131,6 +137,19 @@ const support = external ? { supported: true } : await getEmbeddedPostgresTestSu expect(wakeup).not.toHaveBeenCalled(); expect(errors).toEqual([]); }); + it("restores overdue debt after the feature is disabled across its saved deadline", async () => { + await start(); const created = await card(); await quiet(); + await issueService(db).create(companyId, { title: "Disabled blocker", status: "blocked", priority: "medium" }); + await vi.waitFor(async () => expect((await statusCardService(db).getById(created.id))?.pendingChangeCount).toBeGreaterThan(0)); + const pending = (await statusCardService(db).getById(created.id))!; + await instanceSettingsService(db).updateExperimental({ enableStatusCards: false }); await quiet(); + expect(wakeup).not.toHaveBeenCalled(); + expect((await statusCardService(db).getById(created.id))!.nextEvalAt).toEqual(pending.nextEvalAt); + await instanceSettingsService(db).updateExperimental({ enableStatusCards: true }); + await vi.waitFor(() => expect(wakeup).toHaveBeenCalledOnce()); + await writeback(created.id, (await statusCardService(db).getById(created.id))!.generatingIssueId!); await quiet(); + }); + it("does not react to rolled-back or nested uncommitted card configuration", async () => { await start(); const created = await card("manual"); await quiet(); await expect(db.transaction(async tx => { @@ -169,6 +188,184 @@ const support = external ? { supported: true } : await getEmbeddedPostgresTestSu .toHaveLength(1); }); + it.each(["debounce", "hours", "rate", "budget"] as const)("restores pending %s eligibility after restart without advancing it or acting early", async gate => { + vi.setSystemTime(new Date("2026-11-02T18:00:00Z")); + const refreshPolicy = { ...policy(), ...(gate === "hours" ? { activeHours: { start: "09:00", end: "17:00", timezone: "UTC" } } : {}), + ...(gate === "rate" ? { maxUpdatesPerHour: 1 } : {}), ...(gate === "budget" ? { dailyTokenCap: 100 } : {}) }; + const created = await card("reactive", refreshPolicy); + if (gate === "rate" || gate === "budget") await db.insert(statusCardUpdates).values({ cardId: created.id, kind: "full", trigger: "reactive", + status: "ok", startedAt: new Date(Date.now() - (gate === "rate" ? 30 * 60_000 : 60_000)), finishedAt: new Date(), inputTokens: gate === "budget" ? 100 : 0 }); + await issueService(db).create(companyId, { title: "Saved blocker", status: "blocked", priority: "medium", createdByUserId: "board-user" }); + const service = statusCardEvaluationService(db, { wakeup }); + await service.run(new AbortController().signal); + const pending = (await statusCardService(db).getById(created.id))!; + expect(pending.pendingChangeCount).toBeGreaterThan(0); expect(pending.nextEvalAt!.getTime()).toBeGreaterThan(Date.now()); + const restarted = statusCardEvaluationService(db, { wakeup }); + await restarted.run(new AbortController().signal); + expect(await restarted.next()).toBe(pending.nextEvalAt!.getTime()); + expect((await statusCardService(db).getById(created.id))!.nextEvalAt).toEqual(pending.nextEvalAt); + vi.setSystemTime(pending.nextEvalAt!.getTime() - 1); await restarted.run(new AbortController().signal); + expect(wakeup).not.toHaveBeenCalled(); + vi.setSystemTime(pending.nextEvalAt!); await restarted.run(new AbortController().signal); + expect(wakeup).toHaveBeenCalledOnce(); + }); + + it("unions startup reconstruction with a scoped source change in another company", async () => { + const firstCompany = companyId; + const first = await card(); + await issueService(db).create(companyId, { title: "First blocker", status: "blocked", priority: "medium" }); + companyId = randomUUID(); agentId = randomUUID(); + await db.insert(companies).values({ id: companyId, name: "Other", issuePrefix: `X${companyId.slice(0, 6)}` }); + await db.insert(agents).values({ id: agentId, companyId, name: "Other summarizer", role: "general", status: "idle", adapterType: "process", adapterConfig: {} }); + const second = await card(); + await issueService(db).create(companyId, { title: "Second blocker", status: "blocked", priority: "medium" }); + const service = statusCardEvaluationService(db, { wakeup }); + service.changed(new Set([`changes:${companyId}`])); + await service.run(new AbortController().signal); + const cards = statusCardService(db); + expect((await cards.getById(first.id))?.companyId).toBe(firstCompany); + expect((await cards.getById(first.id))?.pendingChangeCount).toBeGreaterThan(0); + expect((await cards.getById(second.id))?.pendingChangeCount).toBeGreaterThan(0); + await vi.advanceTimersByTimeAsync(60_000); await service.run(new AbortController().signal); + expect(wakeup).toHaveBeenCalledTimes(2); + expect(new Set(await db.select().from(agentWakeupRequests).then(rows => rows.map(row => row.companyId)))) + .toEqual(new Set([firstCompany, companyId])); + }); + + it("keeps ignored issue activity quiet and reacts to human comments and document search membership", async () => { + await start(); await card(); await quiet(); + await issueService(db).create(companyId, { title: "Ignored todo", status: "todo", priority: "medium" }); + await quiet(); expect(wakeup).not.toHaveBeenCalled(); + compiledQuery = companySearchQuerySchema.parse({ scope: "all", q: "launch" }); + const searchCard = await card(); await quiet(); + const target = await issueService(db).create(companyId, { title: "Ordinary task", status: "todo", priority: "medium" }); + await quiet(); + await documentService(db).upsertIssueDocument({ issueId: target.id, key: "plan", format: "markdown", body: "launch evidence", createdByUserId: "board-user" }); + await vi.waitFor(async () => expect((await statusCardService(db).getById(searchCard.id))?.pendingChangeCount).toBeGreaterThan(0)); + await vi.advanceTimersByTimeAsync(60_000); await settled(); expect(wakeup).toHaveBeenCalledOnce(); + await writeback(searchCard.id, (await statusCardService(db).getById(searchCard.id))!.generatingIssueId!); await quiet(); + await issueService(db).addComment(target.id, "A human launch update", { userId: "board-user" }); + await vi.waitFor(async () => expect((await statusCardService(db).getById(searchCard.id))?.pendingChangeCount).toBeGreaterThan(0)); + }); + + it("retains relative query membership expiry across restart and emits the eventual removal", async () => { + const now = new Date(); + const target = await issueService(db).create(companyId, { title: "Expiring blocker", status: "blocked", priority: "medium" }); + await db.update(issues).set({ updatedAt: now }).where(eq(issues.id, target.id)); + compiledQuery = companySearchQuerySchema.parse({ scope: "issues", status: ["blocked"], updatedWithin: "1h" }); + const created = await card(); + const expires = now.getTime() + 3_600_001; + expect(created.nextEvalAt?.getTime()).toBe(expires); + const restarted = statusCardEvaluationService(db, { wakeup }); + await restarted.run(new AbortController().signal); + expect(await restarted.next()).toBe(expires); + vi.setSystemTime(expires - 1); await restarted.run(new AbortController().signal); expect(wakeup).not.toHaveBeenCalled(); + vi.setSystemTime(expires); await restarted.run(new AbortController().signal); + const pending = (await statusCardService(db).getById(created.id))!; + expect(pending.pendingChangeCount).toBe(1); expect(pending.nextEvalAt!.getTime()).toBe(expires + 60_000); + vi.setSystemTime(pending.nextEvalAt!); await restarted.run(new AbortController().signal); + expect(wakeup).toHaveBeenCalledOnce(); + }); + + it("uses expiry of an off-page candidate to refresh offset-ranked query membership", async () => { + const now = new Date(); + const tasks = await Promise.all((["critical", "high", "medium"] as const).map(priority => issueService(db).create(companyId, { + title: `${priority} blocker`, priority, status: "blocked", + }))); + await db.update(issues).set({ updatedAt: new Date(now.getTime() - 59 * 60_000) }).where(eq(issues.id, tasks[0]!.id)); + for (const task of tasks.slice(1)) await db.update(issues).set({ updatedAt: now }).where(eq(issues.id, task.id)); + compiledQuery = companySearchQuerySchema.parse({ scope: "issues", status: ["blocked"], updatedWithin: "1h", sort: "priority", limit: 1, offset: 1 }); + const created = await card(); + expect(Object.keys(created.fingerprint!)).toEqual([tasks[1]!.id]); + const expires = now.getTime() + 60_001; + expect(created.nextEvalAt?.getTime()).toBe(expires); + const service = statusCardEvaluationService(db, { wakeup }); + vi.setSystemTime(expires); await service.run(new AbortController().signal); + const pending = (await statusCardService(db).getById(created.id))!; + expect(pending.pendingChangeCount).toBe(2); expect(wakeup).not.toHaveBeenCalled(); + vi.setSystemTime(pending.nextEvalAt!); await service.run(new AbortController().signal); + expect(wakeup).toHaveBeenCalledOnce(); + const [update] = await db.select().from(statusCardUpdates).where(eq(statusCardUpdates.generationIssueId, (await statusCardService(db).getById(created.id))!.generatingIssueId!)); + expect(new Set(update!.changes.map(change => change.issueId))).toEqual(new Set([tasks[1]!.id, tasks[2]!.id])); + }); + + it("retains a scoped retry while another evaluator owns the card lock", async () => { + const created = await card(); + await issueService(db).create(companyId, { title: "Locked blocker", status: "blocked", priority: "medium" }); + const acquired = Promise.withResolvers(), release = Promise.withResolvers(); + const holder = db.transaction(async tx => { + await tx.execute(sql`select pg_advisory_xact_lock(hashtextextended(${created.id}, 0))`); + acquired.resolve(); await release.promise; + }); + await acquired.promise; + const service = statusCardEvaluationService(db, { wakeup }); + service.changed(new Set([`changes:${companyId}`])); + try { + await service.run(new AbortController().signal); + expect(await service.next()).toBeGreaterThan(Date.now()); + expect((await statusCardService(db).getById(created.id))!.nextEvalAt).toBeNull(); + expect(wakeup).not.toHaveBeenCalled(); + } finally { release.resolve(); await holder; } + await vi.advanceTimersByTimeAsync(30_000); await service.run(new AbortController().signal); + const pending = (await statusCardService(db).getById(created.id))!; + expect(pending.pendingChangeCount).toBeGreaterThan(0); + vi.setSystemTime(pending.nextEvalAt!); await service.run(new AbortController().signal); + expect(wakeup).toHaveBeenCalledOnce(); + }); + + it("retains the durable five-minute retry after evaluation failure across restart and drain", async () => { + const created = await card(); + await db.update(statusCards).set({ queries: [{ scope: "invalid" } as never], nextEvalAt: new Date() }).where(eq(statusCards.id, created.id)); + const service = statusCardEvaluationService(db, { wakeup }); + await service.run(new AbortController().signal); + const failed = (await statusCardService(db).getById(created.id))!; + const retryAt = failed.nextEvalAt!.getTime(); + expect(retryAt).toBeGreaterThan(Date.now() + 299_000); expect(wakeup).not.toHaveBeenCalled(); + await start(); expect(coordinator!.nextWakeAt()).toBe(retryAt); + draining = true; await vi.advanceTimersByTimeAsync(5 * 60_000); await settled(); expect(wakeup).not.toHaveBeenCalled(); + // Repair the controlled malformed stored-query fault; no tested result is precreated. + await db.update(statusCards).set({ queries: [compiledQuery] }).where(eq(statusCards.id, created.id)); + draining = false; await vi.advanceTimersByTimeAsync(30_000); await quiet(); + expect((await statusCardService(db).getById(created.id))!.nextEvalAt).toBeNull(); + }); + + it("cancels failed assignments without rerunning a completed generation and settles after retry", async () => { + await start(); const created = await card(); await quiet(); + wakeup.mockRejectedValueOnce(new Error("assignment unavailable")); + await issueService(db).create(companyId, { title: "Assignment blocker", status: "blocked", priority: "medium" }); + await vi.waitFor(async () => expect((await statusCardService(db).getById(created.id))?.pendingChangeCount).toBeGreaterThan(0)); + await vi.advanceTimersByTimeAsync(60_000); await settled(); + const failed = (await statusCardService(db).getById(created.id))!; + expect(failed).toMatchObject({ state: "error", generatingIssueId: null, nextEvalAt: null }); + expect(errors.length).toBeGreaterThan(0); + const history = await db.select().from(statusCardUpdates).where(eq(statusCardUpdates.cardId, created.id)); + const ledger = history.find(row => row.status === "failed")!; + expect(ledger.finishedAt).toBeInstanceOf(Date); + expect((await db.select().from(issues).where(eq(issues.id, ledger.generationIssueId!)))[0]?.status).toBe("cancelled"); + const count = wakeup.mock.calls.length; await quiet(); expect(wakeup).toHaveBeenCalledTimes(count); + // An explicit ordinary refresh is still the user's supported retry action. + const retry = await statusCardService(db).requestRefresh(created.id, { trigger: "manual", actor }); + expect(retry.generatingIssue).toBeTruthy(); await writeback(created.id, retry.generatingIssue!.id); await quiet(); + expect((await statusCardService(db).getById(created.id))!.state).toBe("active"); + }); + + it("budget-owned company resume reactivates reactive work committed while paused", async () => { + await start(); const created = await card(); await quiet(); + const budgets = budgetService(db); + await budgets.upsertPolicy(companyId, { scopeType: "company", scopeId: companyId, amount: 100, hardStopEnabled: true, notifyEnabled: false }, "board-user"); + const [cost] = await db.insert(costEvents).values({ companyId, agentId, provider: "openai", biller: "openai", billingType: "metered_api", model: "test", + costCents: 200, occurredAt: new Date() }).returning(); + await budgets.evaluateCostEvent(cost!); + expect((await db.select().from(companies).where(eq(companies.id, companyId)))[0]?.status).toBe("paused"); + await quiet(); + await issueService(db).create(companyId, { title: "Paused blocker", status: "blocked", priority: "medium" }); await quiet(); + expect(wakeup).not.toHaveBeenCalled(); + await budgets.upsertPolicy(companyId, { scopeType: "company", scopeId: companyId, amount: 1000 }, "board-user"); + expect((await db.select().from(companies).where(eq(companies.id, companyId)))[0]?.status).toBe("active"); + await vi.waitFor(async () => expect((await statusCardService(db).getById(created.id))?.pendingChangeCount).toBeGreaterThan(0)); + await vi.advanceTimersByTimeAsync(60_000); await settled(); expect(wakeup).toHaveBeenCalledOnce(); + }); + it("preserves the first write during drain until resume", async () => { await start(); const created = await card("manual"); await quiet(); draining = true; diff --git a/server/src/__tests__/status-card-notifications.test.ts b/server/src/__tests__/status-card-notifications.test.ts new file mode 100644 index 00000000000..0566ebdd394 --- /dev/null +++ b/server/src/__tests__/status-card-notifications.test.ts @@ -0,0 +1,31 @@ +import { expect, it, vi } from "vitest"; +import { installDatabaseWorkSignals, subscribeDatabaseWork } from "../../../packages/db/src/work-signals.js"; +import { DELIVERY_QUEUES } from "../services/delivery-work-notifications.js"; +import { notifyExecutionRunWork, writeExecutionControl } from "../services/execution-control-notifications.js"; + +it.each([false, true])("keeps execution queue consumers intact and emits only a known status-card company scope (%s)", async scoped => { + const owner = installDatabaseWorkSignals({ execute: vi.fn(async () => []), transaction: async (fn: (tx: object) => Promise) => + fn({ execute: async () => [{ xid: "1" }] }) }); + const queues = [DELIVERY_QUEUES.executionStatus, DELIVERY_QUEUES.executionFinalization, DELIVERY_QUEUES.executionReplacement, + DELIVERY_QUEUES.executionDisposition, DELIVERY_QUEUES.issueMonitors]; + const settled: Array<{ queue: string; hints?: string[] }> = []; + const unsubscribe = [...queues, DELIVERY_QUEUES.statusCardEvaluation, DELIVERY_QUEUES.workspaceRecovery].map(queue => subscribeDatabaseWork(owner, queue, (event, hints) => { + if (event === "settled") settled.push({ queue, hints: hints ? [...hints] : undefined }); + })); + try { + const scope = scoped ? { companyId: "changed-company", runId: "changed-run" } : undefined; + await owner.transaction(async tx => { + await notifyExecutionRunWork(tx, scope); + expect(settled).toEqual([]); + }); + expect(new Set(settled.map(row => row.queue))).toEqual(new Set([...queues, ...(scoped ? [DELIVERY_QUEUES.statusCardEvaluation] : [])])); + expect(settled.find(row => row.queue === DELIVERY_QUEUES.statusCardEvaluation)?.hints) + .toEqual(scoped ? ["changes:changed-company"] : undefined); + settled.length = 0; + const write = vi.fn(async () => "written"); + expect(await writeExecutionControl(owner as never, DELIVERY_QUEUES.workspaceRecovery, write, scope)).toBe("written"); + expect(write).toHaveBeenCalledOnce(); + expect(new Set(settled.map(row => row.queue))).toEqual(new Set([DELIVERY_QUEUES.issueMonitors, DELIVERY_QUEUES.workspaceRecovery, + ...(scoped ? [DELIVERY_QUEUES.statusCardEvaluation] : [])])); + } finally { for (const stop of unsubscribe) stop(); } +}); diff --git a/server/src/modules/wake-queue/adapters/postgres.ts b/server/src/modules/wake-queue/adapters/postgres.ts index b888ee51fb3..2b7f0a16e26 100644 --- a/server/src/modules/wake-queue/adapters/postgres.ts +++ b/server/src/modules/wake-queue/adapters/postgres.ts @@ -1,3 +1,4 @@ +import { notifyStatusCardIssueChange } from "../../../services/status-card-notifications.js"; import { notifyExternalConversationSettlement } from "../../../services/external-conversation-settlement-notifications.js"; import { notifyNativeStatusReassessment } from "../../../services/native-status-reassessment-notifications.js"; import { notifyUnsafeWorkspaceExportRecovery } from "../../../services/unsafe-workspace-export-notifications.js"; @@ -610,6 +611,7 @@ function buildTransaction(tx: Db, deps: WakeQueuePostgresAdapterDeps, db: Db, ru }, async queueReviewParticipantRecoveryRun({ companyId, issue, finishingRun, recoveryAgent, sessionBefore, now }) { + await notifyStatusCardIssueChange(tx, companyId); const executionState = parseIssueExecutionState(issue.executionState); const wakeupRequest = await tx .insert(agentWakeupRequests) diff --git a/server/src/modules/wake-queue/adapters/queued-comment-postgres.ts b/server/src/modules/wake-queue/adapters/queued-comment-postgres.ts index 184c799f82d..a10419f0c41 100644 --- a/server/src/modules/wake-queue/adapters/queued-comment-postgres.ts +++ b/server/src/modules/wake-queue/adapters/queued-comment-postgres.ts @@ -1,3 +1,4 @@ +import { notifyStatusCardIssueChange } from "../../../services/status-card-notifications.js"; import { notifyExternalAgentSettlement } from "../../external-agents/index.js"; import { notifyNativeStatusReassessment } from "../../../services/native-status-reassessment-notifications.js"; import { notifyUnsafeWorkspaceExportRecovery } from "../../../services/unsafe-workspace-export-notifications.js"; @@ -55,6 +56,7 @@ export type QueuedCommentQueuePostgresAdapterDeps = { function buildTransaction(tx: Db, companyId: string, issueId: string, deps: QueuedCommentQueuePostgresAdapterDeps): QueuedCommentQueueTransaction { return { async updateCommentBody({ issueId, commentId, body, updatedAt }) { + await notifyStatusCardIssueChange(tx, companyId); const updated = await tx .update(issueComments) .set({ body, updatedAt }) @@ -65,6 +67,7 @@ function buildTransaction(tx: Db, companyId: string, issueId: string, deps: Queu }, async touchIssueUpdatedAt({ issueId, updatedAt }) { + await notifyStatusCardIssueChange(tx, companyId); await tx.update(issues).set({ updatedAt }).where(and(eq(issues.id, issueId), eq(issues.companyId, companyId))); }, diff --git a/server/src/routes/execution-workspaces.ts b/server/src/routes/execution-workspaces.ts index e87fbbeea6b..7be8e505a94 100644 --- a/server/src/routes/execution-workspaces.ts +++ b/server/src/routes/execution-workspaces.ts @@ -1,3 +1,4 @@ +import { notifyStatusCardIssueChange } from "../services/status-card-notifications.js"; import { removePluginWorkspace } from "../services/plugin-workspaces.js"; import { canActorReadWorkspaceOperation } from "../services/heartbeat-run-privacy.js"; import { canActorReadExecutionWorkspace, executionWorkspaceReadSqlCondition } from "../services/authorization.js"; @@ -1245,7 +1246,9 @@ export function executionWorkspaceRoutes(db: Db, opts: { pluginWorkerManager?: P await runtimeLeases.release({ executionWorkspaceId: existing.id, force: true }); if (existing.mode === "shared_workspace") { - await db + await db.transaction(async tx => { + await notifyStatusCardIssueChange(tx, existing.companyId); + await tx .update(issues) .set({ executionWorkspaceId: null, @@ -1257,6 +1260,7 @@ export function executionWorkspaceRoutes(db: Db, opts: { pluginWorkerManager?: P eq(issues.executionWorkspaceId, existing.id), ), ); + }); } try { diff --git a/server/src/services/agent-conversations.ts b/server/src/services/agent-conversations.ts index af44a0355d8..6dc783568e2 100644 --- a/server/src/services/agent-conversations.ts +++ b/server/src/services/agent-conversations.ts @@ -1,3 +1,4 @@ +import { notifyStatusCardIssueChange } from "./status-card-notifications.js"; import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js"; import { notifyUnsafeWorkspaceExportRecovery } from "./unsafe-workspace-export-notifications.js"; import { FAST_RESPONSE_AGENT_GUIDANCE, fastResponseHistoryBody } from "@paperclipai/shared"; @@ -269,6 +270,7 @@ export async function prepareConversationTurn( context.conversationReplayThroughCommentId = replayThrough?.id ?? commentId; } } + await notifyStatusCardIssueChange(tx, issue.companyId); await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors); await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers); await tx @@ -389,6 +391,7 @@ export async function settleConversationTurn( issue.conversationState === conversationState ) return true; + await notifyStatusCardIssueChange(tx, issue.companyId); await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors); await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers); if (status === "in_review") await notifyUnsafeWorkspaceExportRecovery(tx, { companyId: issue.companyId, issueId: issue.id }); diff --git a/server/src/services/budgets.ts b/server/src/services/budgets.ts index 6777124ee90..baf667c43c9 100644 --- a/server/src/services/budgets.ts +++ b/server/src/services/budgets.ts @@ -1,3 +1,4 @@ +import { notifyStatusCardIssueChange } from "./status-card-notifications.js"; import { notifyApprovalWork } from "./approval-work-notifications.js"; import { notifyCompanyRetentionWork } from "./retention-work-notifications.js"; import { addCents, compareCents, normalizeCents } from "@paperclipai/shared"; @@ -301,6 +302,7 @@ export function budgetServiceInTransaction(db: Db, publications: ActivityPublica const scope = await resolveScopeRecord(db, "company", policy.scopeId); if (scope.paused) return; await notifyCompanyRetentionWork(db, policy.companyId); + await notifyStatusCardIssueChange(db, policy.companyId); await notifyDeliveryWork(db, DELIVERY_QUEUES.agentTimers); await notifyDeliveryWork(db, DELIVERY_QUEUES.routineSchedules); await notifyDeliveryWork(db, DELIVERY_QUEUES.issueMonitors); @@ -361,6 +363,7 @@ export function budgetServiceInTransaction(db: Db, publications: ActivityPublica const scope = await resolveScopeRecord(db, "company", policy.scopeId); if (scope.pauseReason !== "budget") return; await notifyCompanyRetentionWork(db, policy.companyId); + await notifyStatusCardIssueChange(db, policy.companyId); await notifyDeliveryWork(db, DELIVERY_QUEUES.agentTimers); await notifyDeliveryWork(db, DELIVERY_QUEUES.routineSchedules); await notifyDeliveryWork(db, DELIVERY_QUEUES.issueMonitors); diff --git a/server/src/services/chat-channels/telegram/attachment-failure.ts b/server/src/services/chat-channels/telegram/attachment-failure.ts index 58d5d4a2677..49e2497a055 100644 --- a/server/src/services/chat-channels/telegram/attachment-failure.ts +++ b/server/src/services/chat-channels/telegram/attachment-failure.ts @@ -1,3 +1,4 @@ +import { notifyStatusCardIssueChange } from "../../status-card-notifications.js"; import { createProviderEffects, type ProviderEffectTarget } from "../provider-effects.js"; import type { EndpointRow, RuntimeContext, InboundRuntimeContext } from "../runtime-types.js"; @@ -82,6 +83,7 @@ export function createTelegramAttachmentFailure(db: Db, options: TelegramAttachm // Paperclip-visible failure. The message link stays authoritative for // exact provider redelivery, while no agent wakeup is enqueued for an // empty turn that contains no usable bytes. + await notifyStatusCardIssueChange(tx, input.endpoint.companyId); await tx .update(issueComments) .set({ body: visibleFailure, updatedAt: new Date() }) diff --git a/server/src/services/company-search.ts b/server/src/services/company-search.ts index bcf2a550d3f..e6595beebd8 100644 --- a/server/src/services/company-search.ts +++ b/server/src/services/company-search.ts @@ -548,7 +548,7 @@ export function companySearchService(db: Db) { search: async ( companyId: string, query: CompanySearchQuery, - options?: { issueReadCondition?: SQL; projectReadCondition?: SQL }, + options?: { issueReadCondition?: SQL; projectReadCondition?: SQL; onNextMembershipChange?: (at: Date | null) => void }, ): Promise => { const taskSearch = parseTaskSearch(query.q); const normalizedQuery = taskSearch.normalizedQuery; @@ -559,6 +559,7 @@ export function companySearchService(db: Db) { const limit = query.limit; const offset = query.offset; if (!hasSearchText && !issueOnlyFiltersActive(query)) { + options?.onNextMembershipChange?.(null); return { query: query.q, normalizedQuery, @@ -727,6 +728,13 @@ export function companySearchService(db: Db) { } } + const membershipWindow = companySearchUpdatedWithinMs(query.updatedWithin); + if (options?.onNextMembershipChange && membershipWindow !== null && scopeIncludesIssues(scope)) { + // Includes rows outside the visible page: their expiry can move an + // offset/priority-ranked result into the watched set without a write. + branches.push(sql`SELECT 'membership-expiry' AS kind, min(m.updated_at)::text AS value, + 0::int AS count ${countTail} FROM matched m ${branchWhere([scopeCond, ...facetsAll])}`); + } const resultRows = await db.execute(sql` ${taskSearchCtes(companyId, taskSearch, scope !== "issues", and(...issueFilters), options?.issueReadCondition)} ${sql.join(branches, sql` UNION ALL `)} @@ -738,6 +746,7 @@ export function companySearchService(db: Db) { totals: { current: 0, unfiltered: 0, omit: {} }, }; const issueRowsRaw: IssueSearchRow[] = []; + let nextMembershipChange: Date | null = null; for (const row of resultRows) { if (row.kind === "result") { issueRowsRaw.push({ @@ -762,6 +771,9 @@ export function companySearchService(db: Db) { }); continue; } + if (row.kind === "membership-expiry" && row.value && membershipWindow !== null) { + nextMembershipChange = new Date(new Date(row.value).getTime() + membershipWindow + 1); + } const count = Number(row.count ?? 0); if (row.kind === "type:issue") aggregates.typeCounts.issue = count; else if (row.kind === "type:comment") aggregates.typeCounts.comment = count; @@ -782,6 +794,7 @@ export function companySearchService(db: Db) { aggregates.totals.omit[row.kind.slice("total:omit:".length) as CompanySearchIssueFilterKey] = count; } } + options?.onNextMembershipChange?.(nextMembershipChange); return { rows: await enrichIssueSnippets(issueRowsRaw), aggregates }; } diff --git a/server/src/services/documents.ts b/server/src/services/documents.ts index 493c495e007..79e8e14309c 100644 --- a/server/src/services/documents.ts +++ b/server/src/services/documents.ts @@ -1,3 +1,4 @@ +import { notifyStatusCardIssueChange } from "./status-card-notifications.js"; import { randomUUID } from "node:crypto"; import { and, asc, desc, eq, sql } from "drizzle-orm"; import type { Db } from "@paperclipai/db"; @@ -238,6 +239,7 @@ export function documentService(db: Db) { for (let attempt = 0; attempt < maxAttempts; attempt += 1) { try { return await db.transaction(async (tx) => { + await notifyStatusCardIssueChange(tx, issue.companyId); const now = new Date(); const existing = await tx .select({ @@ -592,6 +594,7 @@ export function documentService(db: Db) { }); } await db.transaction(async (tx) => { + for (const companyId of new Set(rows.map(row => row.companyId))) await notifyStatusCardIssueChange(tx, companyId); await insertRowsInChunks(tx, documents, documentRows); await insertRowsInChunks(tx, documentRevisions, revisionRows); await insertRowsInChunks(tx, issueDocuments, linkRows); @@ -662,6 +665,7 @@ export function documentService(db: Db) { }) .returning(); + await notifyStatusCardIssueChange(tx, existing.companyId); await tx .update(documents) .set({ @@ -818,6 +822,7 @@ export function documentService(db: Db) { } await tx.delete(issueDocuments).where(eq(issueDocuments.documentId, existing.id)); + await notifyStatusCardIssueChange(tx, existing.companyId); await tx.delete(documents).where(eq(documents.id, existing.id)); return { diff --git a/server/src/services/execution-workspaces.ts b/server/src/services/execution-workspaces.ts index caaae186ad8..15ff32f292e 100644 --- a/server/src/services/execution-workspaces.ts +++ b/server/src/services/execution-workspaces.ts @@ -2530,13 +2530,16 @@ export function executionWorkspaceService(db: Db, opts: ExecutionWorkspaceServic forceWorktreeRemoval: false, }); if (cleanup.cleaned && workspace.mode === "shared_workspace") { - await db + await db.transaction(async tx => { + await notifyStatusCardIssueChange(tx, workspace.companyId); + await tx .update(issues) .set({ executionWorkspaceId: null, updatedAt: now() }) .where(and( eq(issues.companyId, workspace.companyId), eq(issues.executionWorkspaceId, workspace.id), )); + }); } const cleanupReason = [ISSUE_TERMINAL_WORKSPACE_CLEANUP_REASON, ...cleanup.warnings].join(" | "); if (!cleanup.cleaned || cleanup.warnings.length > 0) { diff --git a/server/src/services/heartbeat/queue.ts b/server/src/services/heartbeat/queue.ts index a41b486b647..4d1e423d014 100644 --- a/server/src/services/heartbeat/queue.ts +++ b/server/src/services/heartbeat/queue.ts @@ -1,3 +1,4 @@ +import { notifyStatusCardIssueChange } from "../status-card-notifications.js"; import { DELIVERY_QUEUES, notifyDeliveryWork } from "../delivery-work-notifications.js"; import { notifyExternalAgentSettlement } from "../../modules/external-agents/index.js"; import { notifyNativeStatusReassessment } from "../native-status-reassessment-notifications.js"; @@ -1267,6 +1268,7 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende } async function bindClaimedIssueExecution(tx: Db, ownsIssue: boolean, claimedRun: typeof heartbeatRuns.$inferSelect | null | undefined) { if (!claimedRun || !issueId || !ownsIssue) return; + await notifyStatusCardIssueChange(tx, run.companyId); await tx.update(issues).set({ executionRunId: claimedRun.id, executionAgentNameKey: normalizeAgentNameKey(agent.name), @@ -3758,6 +3760,7 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende .from(agents) .where(eq(agents.id, legacyRun.agentId)) .then((rows) => rows[0] ?? null); + await notifyStatusCardIssueChange(tx, issue.companyId); await tx .update(issues) .set({ diff --git a/server/src/services/heartbeat/scheduling.ts b/server/src/services/heartbeat/scheduling.ts index dece8b41ae9..b0e7f308297 100644 --- a/server/src/services/heartbeat/scheduling.ts +++ b/server/src/services/heartbeat/scheduling.ts @@ -1,3 +1,4 @@ +import { notifyStatusCardIssueChange } from "../status-card-notifications.js"; import { DELIVERY_QUEUES, notifyDeliveryWork } from "../delivery-work-notifications.js"; import { notifyIssueReviewState } from "../issue-review-state-notifications.js"; import { insertIssueComment } from "../issue-comment-writes.js"; @@ -595,6 +596,7 @@ export function createHeartbeatScheduling( try { await db.transaction(async (tx) => { await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors); + await notifyStatusCardIssueChange(tx, claimed.companyId); const released = await tx.update(issues) .set({ monitorWakeRequestedAt: null, updatedAt: input.now }) .where(issueMonitorClaimCondition(claimed)).returning({ id: issues.id }); @@ -628,6 +630,7 @@ export function createHeartbeatScheduling( // failed, and never clobber a monitor that was replaced in the meantime. await db.transaction(async tx => { await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors); + await notifyStatusCardIssueChange(tx, claimed.companyId); await tx.update(issues).set({ monitorWakeRequestedAt: null, updatedAt: input.now }) .where(issueMonitorClaimCondition(claimed)); }); @@ -827,6 +830,7 @@ export function createHeartbeatScheduling( } const consumed = await db.transaction(async tx => { + await notifyStatusCardIssueChange(tx, claimed.companyId); await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors); return tx.update(issues) .set({ @@ -977,6 +981,7 @@ export function createHeartbeatScheduling( const staleClaimThreshold = new Date(now.getTime() - 5 * 60 * 1000); const claimed = await db.transaction(async (tx) => { + await notifyStatusCardIssueChange(tx, issue.companyId); await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors); const [updated] = await tx .update(issues) @@ -1050,6 +1055,7 @@ export function createHeartbeatScheduling( for (const due of dueMonitors) { if (!canContinue()) break; const claimed = await db.transaction(async (tx) => { + await notifyStatusCardIssueChange(tx, due.companyId); await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors); const [updated] = await tx .update(issues) diff --git a/server/src/services/issue-thread-interactions.ts b/server/src/services/issue-thread-interactions.ts index 59c14618342..f4d5abbc5e2 100644 --- a/server/src/services/issue-thread-interactions.ts +++ b/server/src/services/issue-thread-interactions.ts @@ -1,3 +1,4 @@ +import { notifyStatusCardIssueChange } from "./status-card-notifications.js"; import { notifyNativeStatusReassessment } from "./native-status-reassessment-notifications.js"; import { notifyIssueReviewState } from "./issue-review-state-notifications.js"; import { notifyChatConfirmationResolution } from "./chat-confirmation-notifications.js"; @@ -376,7 +377,7 @@ type ResolvedInteractionResult = { }; type IssueThreadInteractionRow = typeof issueThreadInteractions.$inferSelect; -type IssueTouchDb = Pick; +type IssueTouchDb = Pick; function isNativeCompletionReview( row: Pick, @@ -801,11 +802,11 @@ function hydrateInteraction( } } -async function touchIssue(db: IssueTouchDb, issueId: string) { - await db - .update(issues) - .set({ updatedAt: new Date() }) - .where(eq(issues.id, issueId)); +async function touchIssue(db: IssueTouchDb, issueId: string, companyId: string) { + await db.transaction(async tx => { + await notifyStatusCardIssueChange(tx, companyId); + await tx.update(issues).set({ updatedAt: new Date() }).where(eq(issues.id, issueId)); + }); } function isTerminalIssueStatus(status: string) { @@ -2013,7 +2014,7 @@ async function expireStaleRequestConfirmationTarget( "Interaction has already been resolved", ); } - await touchIssue(tx, args.row.issueId); + await touchIssue(tx, args.row.issueId, args.row.companyId); const expired = hydrateInteraction(updated); await enqueueTerminalIssueInteractionChatPublications(tx, expired); return expired; @@ -2468,7 +2469,7 @@ export function issueThreadInteractionService( }; } } else { - await touchIssue(tx, args.issue.id); + await touchIssue(tx, args.issue.id, args.issue.companyId); } // Snapshot the continuation target only after every lifecycle mutation @@ -2682,7 +2683,7 @@ export function issueThreadInteractionService( tx, ); } else { - await touchIssue(tx, args.issue.id); + await touchIssue(tx, args.issue.id, args.issue.companyId); } await enqueueTerminalIssueInteractionChatPublications( tx as unknown as Db, @@ -2864,7 +2865,7 @@ export function issueThreadInteractionService( await enqueueTerminalIssueInteractionChatPublications(tx as unknown as Db, hydrateInteraction(previous)); } } - await touchIssue(tx, issue.id); + await touchIssue(tx, issue.id, issue.companyId); inserted = true; await enqueueIssueInteractionChatPublications(tx as unknown as Db, hydrateInteraction(row)); await notifyDeliveryWork(tx, DELIVERY_QUEUES.connection); @@ -2908,7 +2909,7 @@ export function issueThreadInteractionService( ) .returning(); if (!updated) throw interactionAlreadyResolvedError(); - await touchIssue(db, issue.id); + await touchIssue(db, issue.id, issue.companyId); return hydrateInteraction(updated) as ConnectionIntentInteraction; }, resolveConnectionIntent: async ( @@ -2972,7 +2973,7 @@ export function issueThreadInteractionService( return row; }); if (!updated) throw interactionAlreadyResolvedError(); - await touchIssue(db, issue.id); + await touchIssue(db, issue.id, issue.companyId); const interaction = hydrateInteraction( updated, ) as ConnectionIntentInteraction; @@ -3351,7 +3352,7 @@ export function issueThreadInteractionService( }) .where(eq(issueThreadInteractions.id, current.id)) .returning(); - await touchIssue(tx, issue.id); + await touchIssue(tx, issue.id, issue.companyId); return row; }); return hydrateInteraction(updated); @@ -3414,10 +3415,8 @@ export function issueThreadInteractionService( if (updated) cancelled.push(updated); } - for (const issueId of new Set( - cancelled.map((interaction) => interaction.issueId), - )) { - await touchIssue(db, issueId); + for (const [issueId, companyId] of new Map(cancelled.map(interaction => [interaction.issueId, interaction.companyId]))) { + await touchIssue(db, issueId, companyId); } await emitResolvedInteractionsTelemetry(db, cancelled); return cancelled; @@ -3506,10 +3505,8 @@ export function issueThreadInteractionService( } if (expired.length > 0) { - for (const issueId of new Set( - expired.map((interaction) => interaction.issueId), - )) { - await touchIssue(db, issueId); + for (const [issueId, companyId] of new Map(expired.map(interaction => [interaction.issueId, interaction.companyId]))) { + await touchIssue(db, issueId, companyId); } await emitResolvedInteractionsTelemetry(db, expired); } @@ -4052,7 +4049,7 @@ export function issueThreadInteractionService( return interaction; } - await touchIssue(db, issue.id); + await touchIssue(db, issue.id, issue.companyId); if (superseded.length > 0) { await emitResolvedInteractionsTelemetry( db, @@ -4357,7 +4354,7 @@ export function issueThreadInteractionService( .where(eq(issueThreadInteractions.id, interactionId)) .returning(); - await touchIssue(tx, issue.id); + await touchIssue(tx, issue.id, issue.companyId); await mutationOptions.afterResolveInTransaction?.(tx, hydrateInteraction(updated)); await enqueueTerminalIssueInteractionChatPublications(tx as unknown as Db, hydrateInteraction(updated)); current.status = updated.status; @@ -4549,7 +4546,7 @@ export function issueThreadInteractionService( throw interactionAlreadyResolvedError(); } - await touchIssue(tx, issue.id); + await touchIssue(tx, issue.id, issue.companyId); await mutationOptions.afterResolveInTransaction?.(tx, hydrateInteraction(updated)); if (complete) await enqueueTerminalIssueInteractionChatPublications(tx as unknown as Db, hydrateInteraction(updated)); return { @@ -4622,7 +4619,7 @@ export function issueThreadInteractionService( throw interactionAlreadyResolvedError(); } - await touchIssue(db, issue.id); + await touchIssue(db, issue.id, issue.companyId); const rejected = hydrateInteraction(updated); await emitInteractionResolvedTelemetry(db, rejected); return rejected; @@ -4720,7 +4717,7 @@ export function issueThreadInteractionService( } if (expired.length > 0) { - await touchIssue(db, issue.id); + await touchIssue(db, issue.id, issue.companyId); await emitResolvedInteractionsTelemetry(db, expired); } return expired; @@ -4952,7 +4949,7 @@ export function issueThreadInteractionService( } if (expired.length > 0) { - await touchIssue(db, issue.id); + await touchIssue(db, issue.id, issue.companyId); await emitResolvedInteractionsTelemetry(db, expired); } return expired; @@ -5049,7 +5046,7 @@ export function issueThreadInteractionService( } if (expired.length > 0) { - await touchIssue(db, issue.id); + await touchIssue(db, issue.id, issue.companyId); await emitResolvedInteractionsTelemetry(db, expired); } return expired; @@ -5189,7 +5186,7 @@ export function issueThreadInteractionService( if (updated) expired.push(hydrateInteraction(updated)); } if (expired.length > 0) { - await touchIssue(db, issue.id); + await touchIssue(db, issue.id, issue.companyId); await emitResolvedInteractionsTelemetry(db, expired); } return expired; @@ -5294,7 +5291,7 @@ export function issueThreadInteractionService( return row; }); - await touchIssue(db, issue.id); + await touchIssue(db, issue.id, issue.companyId); const withdrawn = hydrateInteraction(updated); await emitInteractionResolvedTelemetry(db, withdrawn); return withdrawn; @@ -5417,7 +5414,7 @@ export function issueThreadInteractionService( return row; }); - await touchIssue(db, issue.id); + await touchIssue(db, issue.id, issue.companyId); const answered = hydrateInteraction(updated); await emitInteractionResolvedTelemetry(db, answered); return answered; @@ -5553,7 +5550,7 @@ export function issueThreadInteractionService( return row; }); - await touchIssue(db, issue.id); + await touchIssue(db, issue.id, issue.companyId); const skipped = hydrateInteraction(updated); await emitInteractionResolvedTelemetry(db, skipped); return skipped; @@ -5630,7 +5627,7 @@ export function issueThreadInteractionService( return row; }); - await touchIssue(db, issue.id); + await touchIssue(db, issue.id, issue.companyId); const cancelled = hydrateInteraction(updated); await emitInteractionResolvedTelemetry(db, cancelled); return cancelled; diff --git a/server/src/services/issues.ts b/server/src/services/issues.ts index 8787ac867a8..f58d0e62789 100644 --- a/server/src/services/issues.ts +++ b/server/src/services/issues.ts @@ -7730,6 +7730,7 @@ export function issueService(db: Db) { } const now = new Date(); + await notifyStatusCardIssueChange(tx, lockedIssue.companyId); const adopted = await tx .update(issues) .set({ @@ -7783,13 +7784,14 @@ export function issueService(db: Db) { sql`select ${heartbeatRuns.id} from ${heartbeatRuns} where ${heartbeatRuns.id} = ${input.actorRunId} for update`, ); const actorRun = await tx - .select({ status: heartbeatRuns.status }) + .select({ status: heartbeatRuns.status, companyId: heartbeatRuns.companyId }) .from(heartbeatRuns) .where(eq(heartbeatRuns.id, input.actorRunId)) .then((rows) => rows[0] ?? null); if (!actorRun || TERMINAL_HEARTBEAT_RUN_STATUSES.has(actorRun.status)) return null; + await notifyStatusCardIssueChange(tx, actorRun.companyId); const now = new Date(); const adopted = await tx .update(issues) @@ -10892,6 +10894,7 @@ export function issueService(db: Db) { } } + await notifyStatusCardIssueChange(tx, companyId); await insertRowsInChunks(tx, issues, issueRows); await insertRowsInChunks(tx, issueLabels, labelRows); }); @@ -10908,6 +10911,7 @@ export function issueService(db: Db) { const censorUsernameInLogs = (await instanceSettings.getGeneral()) .censorUsernameInLogs; await db.transaction(async (tx) => { + for (const companyId of new Set(rows.map(row => row.companyId))) await notifyStatusCardIssueChange(tx, companyId); const commentRows = rows.map((row) => { const createdAt = row.createdAt ? new Date(row.createdAt) : null; return { @@ -11922,7 +11926,9 @@ export function issueService(db: Db) { ); if (settings?.environmentId !== environmentId) continue; - await db + await db.transaction(async tx => { + await notifyStatusCardIssueChange(tx, companyId); + await tx .update(issues) .set({ executionWorkspaceSettings: { @@ -11932,6 +11938,7 @@ export function issueService(db: Db) { updatedAt: new Date(), }) .where(eq(issues.id, row.id)); + }); cleared += 1; } @@ -12088,6 +12095,7 @@ export function issueService(db: Db) { ) : isNull(issues.executionRunId); const updated = await db.transaction(async tx => { + await notifyStatusCardIssueChange(tx, issueCompany.companyId); await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors); await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers); return tx @@ -12140,7 +12148,9 @@ export function issueService(db: Db) { current.executionRunId === checkoutRunId) && checkoutRunId ) { - const adopted = await db + const adopted = await db.transaction(async tx => { + await notifyStatusCardIssueChange(tx, issueCompany.companyId); + return tx .update(issues) .set({ checkoutRunId, @@ -12161,6 +12171,7 @@ export function issueService(db: Db) { ) .returning() .then((rows) => rows[0] ?? null); + }); if (adopted) return adopted; } @@ -12217,6 +12228,7 @@ export function issueService(db: Db) { } const staleExecutionRunId = current.executionRunId; const adopted = await db.transaction(async tx => { + await notifyStatusCardIssueChange(tx, issueCompany.companyId); await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors); await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers); return tx @@ -12524,6 +12536,7 @@ export function issueService(db: Db) { patch.assigneeAgentId = null; } + await notifyStatusCardIssueChange(tx, existing.companyId); await notifyTerminalWorkspaceWork(tx, `issue:${id}`); if (existing.checkoutRunId || existing.executionRunId || options.clearAssignee) { await notifyExternalConversationSettlement(tx, { companyId: existing.companyId, issueId: id }); @@ -12581,12 +12594,12 @@ export function issueService(db: Db) { return created; }, - deleteLabel: async (id: string) => - db - .delete(labels) - .where(eq(labels.id, id)) - .returning() - .then((rows) => rows[0] ?? null), + deleteLabel: async (id: string) => db.transaction(async tx => { + const [label] = await tx.select({ companyId: labels.companyId }).from(labels).where(eq(labels.id, id)); + if (!label) return null; + await notifyStatusCardIssueChange(tx, label.companyId); + return tx.delete(labels).where(eq(labels.id, id)).returning().then(rows => rows[0] ?? null); + }), listComments: async ( issueId: string, @@ -12838,7 +12851,7 @@ export function issueService(db: Db) { }, dbOrTx: any = db, ): Promise { - if (dbOrTx === db && (actor.runId || actor.userId)) { + if (dbOrTx === db) { const append = () => measureRunTransaction(db, "response", async (tx) => { // Serialize run-authored comments on the issue so a provider retry @@ -13081,6 +13094,7 @@ export function issueService(db: Db) { currentUserRedactionOptions.enabled, ); } + await notifyStatusCardIssueChange(dbOrTx, issue.companyId); comment = shouldUpgradeExternalAuthorization || shouldUpgradeAttachmentAuthorization diff --git a/server/src/services/native-status-reassessment-notifications.ts b/server/src/services/native-status-reassessment-notifications.ts index fb845523bb4..33cf199736f 100644 --- a/server/src/services/native-status-reassessment-notifications.ts +++ b/server/src/services/native-status-reassessment-notifications.ts @@ -1,8 +1,10 @@ +import { notifyStatusCardIssueChange } from "./status-card-notifications.js"; import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js"; export type NativeStatusReassessmentScope = { companyId: string; issueId?: string; runId?: string; agentId?: string }; -export function notifyNativeStatusReassessment(tx: object, scope: NativeStatusReassessmentScope) { +export async function notifyNativeStatusReassessment(tx: object, scope: NativeStatusReassessmentScope) { + await notifyStatusCardIssueChange(tx, scope.companyId); return notifyDeliveryWork(tx, DELIVERY_QUEUES.nativeStatusReassessment, JSON.stringify({ companyId: scope.companyId, issueId: scope.issueId, runId: scope.runId, agentId: scope.agentId })); } diff --git a/server/src/services/projects.ts b/server/src/services/projects.ts index 28cae661e9b..45891a218a6 100644 --- a/server/src/services/projects.ts +++ b/server/src/services/projects.ts @@ -1,3 +1,4 @@ +import { notifyStatusCardIssueChange } from "./status-card-notifications.js"; import { notifyChatInboundWakeNotices } from "./chat-inbound-wake-notice-notifications.js"; import { listProjectWorkspaceProviders, resolveProjectWorkspaceProvider } from "./project-workspace-provider.js"; import { retireRemovedPluginWorkspaceDependents } from "./plugin-workspaces.js"; @@ -1013,6 +1014,7 @@ export function projectService(db: Db, options: { captureWorkspaceUpdates?: bool const row = await db.transaction(async (tx) => { if (updates.visibility !== undefined) { + await notifyStatusCardIssueChange(tx, existingProject.companyId); await notifyChatInboundWakeNotices(tx, { companyId: existingProject.companyId }); await tx.execute(sql`select pg_advisory_xact_lock(hashtextextended(${`issue-privacy-tree:${existingProject.companyId}`}, 0))`); } @@ -1086,6 +1088,7 @@ export function projectService(db: Db, options: { captureWorkspaceUpdates?: bool await tx.select({ id: projectWorkspaces.id }).from(projectWorkspaces).where(eq(projectWorkspaces.projectId, id)).for("update"); await recordProjectLifecycleEvent(tx as unknown as Db, project.companyId, id, "delete"); await notifyDeliveryWork(tx, DELIVERY_QUEUES.skillSourceSync); + await notifyStatusCardIssueChange(tx, project.companyId); const [row] = await tx.delete(projects).where(eq(projects.id, id)).returning(); return row ? { ...row, urlKey: deriveProjectUrlKey(row.name, row.id) } : null; }), diff --git a/server/src/services/status-card-evaluation.ts b/server/src/services/status-card-evaluation.ts index 143df32344a..3e33c0d0885 100644 --- a/server/src/services/status-card-evaluation.ts +++ b/server/src/services/status-card-evaluation.ts @@ -18,17 +18,18 @@ export function statusCardEvaluationService(db: Db, heartbeat: IssueAssignmentWa let reconstructReactive = true; let allChanges = false; const changes = new Set(); - let enabled = false; + let enabled: boolean | null = null; + let retryAt: number | null = null; return { changed(hints?: ReadonlySet) { for (const hint of hints ?? ["*"]) { - if (hint === "*" || hint === "changes:*") allChanges = true; + if (hint === "*" || hint === "changes:*") { allChanges = true; if (hint === "*") enabled = null; } else if (hint.startsWith("changes:")) changes.add(hint.slice("changes:".length)); } }, async run(signal: AbortSignal) { - enabled = (await settings.getExperimental()).enableStatusCards === true; + if (enabled === null) enabled = (await settings.getExperimental()).enableStatusCards === true; if (!enabled || signal.aborted) return; const reconstruct = reconstructReactive; const evaluateAllChanges = allChanges; @@ -37,7 +38,9 @@ export function statusCardEvaluationService(db: Db, heartbeat: IssueAssignmentWa reconstructReactive = false; allChanges = false; try { - const result = await cards.tickDueStatusCards(new Date(), evaluateAllChanges ? "changed" : reconstruct && companiesChanged.size === 0 ? "all" : companiesChanged); + const result = await cards.tickDueStatusCards(new Date(), evaluateAllChanges ? "changed" : companiesChanged, reconstruct); + retryAt = result.deferredCompanyIds.size ? Date.now() + 30_000 : null; + for (const companyId of result.deferredCompanyIds) changes.add(companyId); const assignments = await Promise.allSettled(result.enqueued.map(async ({ cardId, generatingIssue }) => { // Once generated, dispatch or cancel even if shutdown begins: leaving // an unassigned claimed card is not a completed sweep. @@ -74,7 +77,8 @@ export function statusCardEvaluationService(db: Db, heartbeat: IssueAssignmentWa .where(and(eq(companies.status, "active"), isNull(statusCards.archivedAt), isNull(statusCards.generatingIssueId), isNotNull(statusCards.nextEvalAt))) .orderBy(asc(statusCards.nextEvalAt)).limit(1); - return next?.at?.getTime() ?? null; + const deadline = next?.at?.getTime() ?? null; + return retryAt === null ? deadline : Math.min(deadline ?? Infinity, retryAt); }, }; } diff --git a/server/src/services/status-cards.ts b/server/src/services/status-cards.ts index 84bbf7b2cd4..19e7ac75140 100644 --- a/server/src/services/status-cards.ts +++ b/server/src/services/status-cards.ts @@ -25,7 +25,7 @@ import { conflict, forbidden, notFound, unprocessable } from "../errors.js"; import { logger } from "../middleware/logger.js"; import { readBuiltInAgentMarker } from "../lib/built-in-agent-metadata.js"; import { builtInAgentService } from "./built-in-agents.js"; -import { companySearchService, companySearchUpdatedWithinMs } from "./company-search.js"; +import { companySearchService } from "./company-search.js"; import { issueService } from "./issues.js"; import { SUMMARIZER_BUILT_IN_KEY } from "./summary-slots.js"; import { @@ -47,7 +47,6 @@ import { type StatusCardActor = { agentId: string | null; userId: string | null }; type StatusCardWriter = { agentId: string | null; runId: string | null }; type StatusCardRow = typeof statusCards.$inferSelect; -type WatchedIssue = CompanySearchIssueSummary & { scopeExpiresAt?: Date | null }; type StatusCardReadOptions = { issueReadCondition?: SQL }; const TERMINAL_ISSUE_STATUSES = new Set(["done", "cancelled"]); @@ -590,22 +589,15 @@ export function statusCardService( return loadIssueSummaries(card.companyId, card.mentionedIssueIds ?? [], options); } - async function executeQueries(card: StatusCardRow, options: StatusCardReadOptions = {}) { - const issueMap = new Map(); + async function executeQueries(card: StatusCardRow, options: StatusCardReadOptions = {}, onNextMembershipChange?: (at: Date | null) => void) { + const issueMap = new Map(); for (const storedQuery of card.queries) { const query = companySearchQuerySchema.parse(storedQuery); const response = await searchSvc.search(card.companyId, query, { - issueReadCondition: options.issueReadCondition, + issueReadCondition: options.issueReadCondition, onNextMembershipChange, }); for (const result of response.results) { - if (result.type !== "issue" || !result.issue) continue; - const duration = companySearchUpdatedWithinMs(query.updatedWithin); - const expiry = duration === null ? null : new Date(new Date(result.issue.updatedAt).getTime() + duration + 1); - const previous = issueMap.get(result.issue.id); - issueMap.set(result.issue.id, { ...result.issue, scopeExpiresAt: previous - ? previous.scopeExpiresAt == null || expiry === null ? null - : new Date(Math.max(previous.scopeExpiresAt.getTime(), expiry.getTime())) - : expiry }); + if (result.type === "issue" && result.issue) issueMap.set(result.issue.id, result.issue); } } // Issues mentioned in the latest summary join the watched set alongside the @@ -615,11 +607,7 @@ export function statusCardService( (card.mentionedIssueIds ?? []).filter((issueId) => !issueMap.has(issueId)), options, ); - for (const issue of mentioned) issueMap.set(issue.id, { ...issue, scopeExpiresAt: null }); - for (const id of card.mentionedIssueIds ?? []) { - const issue = issueMap.get(id); - if (issue) issue.scopeExpiresAt = null; - } + for (const issue of mentioned) issueMap.set(issue.id, issue); const snapshot = [...issueMap.values()]; if (snapshot.length === 0) return snapshot; const latestHumanComments = await db @@ -643,11 +631,10 @@ export function statusCardService( return snapshot.map((issue) => ({ ...issue, latestHumanCommentAt: commentByIssueId.get(issue.id) ?? null })); } - function nextEvaluationAt(card: StatusCardRow, snapshot: WatchedIssue[], now: Date) { + function nextEvaluationAt(card: StatusCardRow, membershipAt: Date | null, now: Date) { const interval = nextStatusCardEvaluationAt(card.refreshPolicy, now); - if (card.refreshPolicy.mode !== "reactive" || !card.refreshPolicy.triggers.membershipChanges) return interval; - const expiries = snapshot.flatMap(issue => issue.scopeExpiresAt && issue.scopeExpiresAt > now ? [issue.scopeExpiresAt.getTime()] : []); - return expiries.length ? new Date(Math.min(...expiries)) : null; + return card.refreshPolicy.mode === "reactive" && (card.refreshPolicy.triggers.membershipChanges || card.refreshPolicy.triggers.anyUpdate) + ? membershipAt : interval; } async function requestRefresh(cardId: string, input: { @@ -670,13 +657,14 @@ export function statusCardService( } const now = input.now ?? new Date(); - const snapshot = await executeQueries(card); + let membershipAt: Date | null = null; + const snapshot = await executeQueries(card, {}, at => { if (at && (!membershipAt || at < membershipAt)) membershipAt = at; }); const fingerprint = buildStatusCardFingerprint(snapshot); const allChanges = diffStatusCardFingerprint(card.fingerprint as StatusCardFingerprint | null, fingerprint); const changes = filterStatusCardChanges(allChanges, card.refreshPolicy); const trigger = input.trigger ?? "manual"; const forceRun = trigger === "manual" || trigger === "restore"; - const nextEvalAt = nextEvaluationAt(card, snapshot, now); + const nextEvalAt = nextEvaluationAt(card, membershipAt, now); if (!forceRun && changes.length === 0) { const [next] = await mutate(card.companyId, tx => tx.update(statusCards).set({ pendingChangeCount: 0, @@ -797,9 +785,12 @@ export function statusCardService( return { card: next, generatingIssue: generationIssue!, alreadyGenerating: deduplicated, enqueued: true, kind, changes }; } - async function tickDueStatusCards(now = new Date(), reactiveCompanies?: ReadonlySet | "all" | "changed") { - const reactiveScope = reactiveCompanies === "all" ? isNull(statusCards.nextEvalAt) : reactiveCompanies === "changed" ? sql`true` : reactiveCompanies?.size + async function tickDueStatusCards(now = new Date(), reactiveCompanies?: ReadonlySet | "changed", reconstruct = false) { + const changedScope = reactiveCompanies === "changed" ? sql`true` : reactiveCompanies?.size ? inArray(statusCards.companyId, [...reactiveCompanies]) : sql`false`; + // Startup restores every disarmed reactive scope. A concurrent committed + // source hint can additionally preempt its company's saved future deadline. + const reactiveScope = or(reconstruct ? isNull(statusCards.nextEvalAt) : undefined, changedScope); const due = await db.select({ card: statusCards }).from(statusCards) .innerJoin(companies, eq(companies.id, statusCards.companyId)) .where(and(eq(companies.status, "active"), isNull(statusCards.archivedAt), isNull(statusCards.generatingIssueId), @@ -808,6 +799,7 @@ export function statusCardService( .then(rows => rows.map(row => row.card)); const enqueued: Array<{ cardId: string; generatingIssue: typeof issues.$inferSelect }> = []; let evaluated = 0; + const deferredCompanyIds = new Set(); for (const candidate of due) { // nextEvalAt is both a retry lease and an evaluation deadline. A source // event may preempt a genuine future deadline, so serialize scheduler @@ -817,7 +809,7 @@ export function statusCardService( const [lock] = await lockTx.execute<{ acquired: boolean }>( sql`select pg_try_advisory_xact_lock(hashtextextended(${candidate.id}, 0)) as acquired`, ); - if (!lock?.acquired) return; + if (!lock?.acquired) { deferredCompanyIds.add(candidate.companyId); return; } const claimUntil = new Date(now.getTime() + 5 * 60 * 1000); const [claimed] = await mutate(candidate.companyId, tx => tx.update(statusCards).set({ nextEvalAt: claimUntil }) .where(and(eq(statusCards.id, candidate.id), isNull(statusCards.archivedAt), isNull(statusCards.generatingIssueId), @@ -836,7 +828,7 @@ export function statusCardService( } }); } - return { evaluated, enqueued }; + return { evaluated, enqueued, deferredCompanyIds }; } async function writeSummary(cardId: string, input: WriteStatusCardSummary, actor: StatusCardWriter) { @@ -866,7 +858,8 @@ export function statusCardService( const mentionedIssueIds = await resolveMentionedIssueIds(current.companyId, input.markdown); // Current watched membership: compiled-query matches plus the issues this // summary mentions. - const watchedSnapshot = await executeQueries({ ...current, mentionedIssueIds }); + let membershipAt: Date | null = null; + const watchedSnapshot = await executeQueries({ ...current, mentionedIssueIds }, {}, at => { if (at && (!membershipAt || at < membershipAt)) membershipAt = at; }); const watchedNow = buildStatusCardFingerprint(watchedSnapshot); let snapshot: StatusCardFingerprint; if (payloadFingerprint) { @@ -940,7 +933,7 @@ export function statusCardService( lastChangeAt: null, nextEvalAt: current.refreshPolicy.mode === "reactive" && filterStatusCardChanges(diffStatusCardFingerprint(snapshot, watchedNow), current.refreshPolicy).length > 0 - ? now : nextEvaluationAt(current, watchedSnapshot, now), + ? now : nextEvaluationAt(current, membershipAt, now), updatedAt: now, }).where(and(eq(statusCards.id, current.id), eq(statusCards.generatingIssueId, input.generationIssueId))).returning(); if (!next) throw conflict("Status-card generation was superseded by a newer task"); diff --git a/server/src/services/task-watchdogs.ts b/server/src/services/task-watchdogs.ts index cf8d92d39df..84c7f92a8a6 100644 --- a/server/src/services/task-watchdogs.ts +++ b/server/src/services/task-watchdogs.ts @@ -1,3 +1,4 @@ +import { notifyStatusCardIssueChange } from "./status-card-notifications.js"; import { createHash } from "node:crypto"; import { and, asc, eq, gte, inArray, isNull, or, sql } from "drizzle-orm"; import type { Db } from "@paperclipai/db"; @@ -1398,10 +1399,13 @@ export function taskWatchdogService(db: Db, deps: TaskWatchdogServiceDeps = {}) }) ?? fallback : fallback; if (!shouldReopen && watchdogIssue.originFingerprint !== input.classification.stopFingerprint) { - await db + await db.transaction(async tx => { + await notifyStatusCardIssueChange(tx, input.watchdog.companyId); + await tx .update(issues) .set({ originFingerprint: input.classification.stopFingerprint, updatedAt: new Date() }) .where(and(eq(issues.companyId, input.watchdog.companyId), eq(issues.id, watchdogIssue.id))); + }); watchdogIssue.originFingerprint = input.classification.stopFingerprint; } await issuesSvc.addComment( diff --git a/server/src/services/workspace-runtime.ts b/server/src/services/workspace-runtime.ts index 600d5e32a42..2bb9239eaf3 100644 --- a/server/src/services/workspace-runtime.ts +++ b/server/src/services/workspace-runtime.ts @@ -1,3 +1,4 @@ +import { notifyStatusCardIssueChange } from "./status-card-notifications.js"; import { insertIssueComment } from "./issue-comment-writes.js"; import { restoreGitWorkspaceContents } from "./git-workspace-contents.js"; import { savedGitWorkspaceRestoration, ensureSavedGitCommit, GitCacheRestorationFailure } from "./git-workspace-restoration.js"; @@ -1636,10 +1637,11 @@ async function writeDirtyQuarantineAuditComments(input: { let sourceAuditCommentId: string | null = null; let claimantAuditCommentId: string | null = null; const now = new Date(); - if (input.evidence.sourceIssueId) { + const sourceIssueId = input.evidence.sourceIssueId; + if (sourceIssueId) { const [sourceComment] = await insertIssueComment(input.db, { companyId: input.companyId, - issueId: input.evidence.sourceIssueId, + issueId: sourceIssueId, authorAgentId: null, authorUserId: null, authorType: "system", @@ -1647,10 +1649,10 @@ async function writeDirtyQuarantineAuditComments(input: { body, }); sourceAuditCommentId = sourceComment?.id ?? null; - await input.db - .update(issues) - .set({ updatedAt: now }) - .where(eq(issues.id, input.evidence.sourceIssueId)); + await input.db.transaction(async tx => { + await notifyStatusCardIssueChange(tx, input.companyId); + await tx.update(issues).set({ updatedAt: now }).where(eq(issues.id, sourceIssueId)); + }); } const claimantIssueId = input.evidence.contention?.claimedByIssueId ?? null; @@ -1665,10 +1667,10 @@ async function writeDirtyQuarantineAuditComments(input: { body, }); claimantAuditCommentId = claimantComment?.id ?? null; - await input.db - .update(issues) - .set({ updatedAt: now }) - .where(eq(issues.id, claimantIssueId)); + await input.db.transaction(async tx => { + await notifyStatusCardIssueChange(tx, input.companyId); + await tx.update(issues).set({ updatedAt: now }).where(eq(issues.id, claimantIssueId)); + }); } return { sourceAuditCommentId, claimantAuditCommentId }; From 097ac579da3a19154d28ba980906d3a23080363e Mon Sep 17 00:00:00 2001 From: Dotta Date: Fri, 30 Oct 2026 17:59:04 -0500 Subject: [PATCH 40/74] perf(server): batch remaining attention retention page evidence Load company-partitioned decision/recovery links, run/agent visibility and newer-run probes once per page; share one blocker graph reconstruction and exact SQL timestamps across page entries. Keep fresh per-entry archive locks and revalidation. Add full100-source query budgets and late visibility/supersession safety controls. Co-Authored-By: Paperclip --- .../2026-10-30-attention-retention-sleep.md | 2 +- .../attention-retention-work.test.ts | 85 +++++++++++++++ server/src/services/attention-retention.ts | 103 ++++++++++++++---- 3 files changed, 169 insertions(+), 21 deletions(-) diff --git a/doc/plans/2026-10-30-attention-retention-sleep.md b/doc/plans/2026-10-30-attention-retention-sleep.md index 1b71dc71261..2e0373fc975 100644 --- a/doc/plans/2026-10-30-attention-retention-sleep.md +++ b/doc/plans/2026-10-30-attention-retention-sleep.md @@ -25,7 +25,7 @@ The feed and worker share confirmation collapse, newer-run probes, issue visibil ## Existing mutation boundaries reactivate work -Committed hints identify a company, source kind, and source ID where those values are known. Retention's own writes use exact source hints; blocker retention changes invalidate the company/source because multiple blocked roots can project one terminal subject. Reconstruction pages and due batches remain bounded. Review attention and interaction invokability are computed in company batches per candidate page; a full 100-candidate review page uses at most 20 statements and 100 confirmations use at most six. Ordinary execution, decision and budget mutations pass their affected company; exact decision sources are included. +Committed hints identify a company, source kind, and source ID where those values are known. Retention's own writes use exact source hints; blocker retention changes invalidate the company/source because multiple blocked roots can project one terminal subject. Reconstruction pages and due batches remain bounded. Review attention and interaction invokability are computed in company batches per candidate page; a full 100-candidate review page uses at most 20 statements and 100 confirmations use at most six. Decisions, recovery source links, runs, eligible agents, issue visibility, newer-run probes and blocker graphs also load in company batches per page. A 100-source decision or recovery page uses at most three statements; 100 failed runs use at most six; 100 blocker roots use at most 30 graph/source statements plus retention-state reads for their emitted entries. These evidence maps exist only for one load. Archive checks still lock and freshly re-read sources, terminal subjects and related eligibility. Ordinary execution, decision and budget mutations pass their affected company; exact decision sources are included. Producer coverage includes decision creation and changes; interaction creation, resolution, expiry and supersession; issue creation, updates, blocker relations, checkout, adoption and release; agent hire, configuration, lifecycle, error clearing and deletion; budget incidents and company pause/resume; exhausted-run events and retry-run creation; native status, review, finalization and reconstruction; external conversation transitions; and imported blocker relations. Existing service transactions retain outer commit ownership. Savepoint release does not publish work before outer commit. diff --git a/server/src/__tests__/attention-retention-work.test.ts b/server/src/__tests__/attention-retention-work.test.ts index 889030d2c22..fd1a01e8582 100644 --- a/server/src/__tests__/attention-retention-work.test.ts +++ b/server/src/__tests__/attention-retention-work.test.ts @@ -171,6 +171,91 @@ const delay = DEFAULT_DECISION_ARCHIVE_DAYS * 86_400_000; expect(statements.filter(query => query.includes('from "agents"'))).toHaveLength(1); }); + it("batches decision and recovery source links and rechecks a late hidden origin", async () => { + const origins = await db.insert(issues).values(Array.from({ length: 100 }, () => ({ companyId, + title: randomUUID(), status: "todo", updatedAt: old() }))).returning(); + const [run] = await db.insert(heartbeatRuns).values({ companyId, agentId, status: "succeeded" }).returning(); + const decisionRows = await db.insert(decisions).values(origins.map(origin => ({ companyId, + originAgentId: agentId, originIssueId: origin.id, originRunId: run.id, title: "Decision", body: "Choose", + options: [], signedSpec: "fixture", targetSnapshots: {}, expiresAt: new Date(Date.now() + 86_400_000), updatedAt: old() }))).returning(); + const recoveryRows = await db.insert(issueRecoveryActions).values(origins.map(origin => ({ companyId, + sourceIssueId: origin.id, kind: "execution", ownerType: "board", cause: "fixture", fingerprint: "fixture", + nextAction: "Choose", updatedAt: old() }))).returning(); + const service = attentionRetentionService(db); + const pages = []; + for (const sourceKind of ["decision", "recovery_action"] as const) { + statements.length = 0; + const page = await service.load({ companyId, sourceKind }); + expect(page.rows).toHaveLength(100); expect(page.complete).toBe(false); + expect(statements.length).toBeLessThanOrEqual(3); + pages.push(page.rows!); + } + await db.update(issues).set({ hiddenAt: new Date() }).where(eq(issues.id, origins[99].id)); + const hiddenDecision = pages[0].find(entry => entry.sourceId === decisionRows[99].id)!; + const hiddenRecovery = pages[1].find(entry => entry.sourceId === recoveryRows[99].id)!; + expect(await service.check(hiddenDecision)).toBeNull(); expect(await service.check(hiddenRecovery)).toBeNull(); + expect(await state("decision", hiddenDecision.sourceId)).toBeNull(); expect(await state("recovery_action", hiddenRecovery.sourceId)).toBeNull(); + expect(await db.select().from(decisionArchiveNotificationOutbox)).toHaveLength(0); + }); + + it("batches 100 failed runs and rechecks late visibility and supersession before archive", async () => { + const origins = await db.insert(issues).values(Array.from({ length: 100 }, () => ({ companyId, + title: randomUUID(), status: "todo", updatedAt: old() }))).returning(); + const runs = await db.insert(heartbeatRuns).values(origins.map(origin => ({ companyId, agentId, + status: "failed", contextSnapshot: { issueId: origin.id }, createdAt: old(), finishedAt: old() }))).returning(); + await db.insert(heartbeatRunEvents).values(runs.map(run => ({ companyId, agentId, runId: run.id, + seq: 1, eventType: "lifecycle", message: "Bounded retry exhausted: batch fixture" }))); + const service = attentionRetentionService(db), scope = { companyId, sourceKind: "failed_run" as const }; + statements.length = 0; + const page = await service.load(scope); + expect(page.rows).toHaveLength(100); expect(page.complete).toBe(false); + expect(statements.length).toBeLessThanOrEqual(6); + const hiddenRun = runs[98], supersededRun = runs[99]; + const hidden = page.rows!.find(entry => entry.sourceId === hiddenRun.id)!; + const superseded = page.rows!.find(entry => entry.sourceId === supersededRun.id)!; + await db.update(issues).set({ hiddenAt: new Date() }).where(eq(issues.id, origins[98].id)); + await db.insert(heartbeatRuns).values({ companyId, agentId, status: "succeeded", + contextSnapshot: { issueId: origins[99].id }, createdAt: new Date(old().getTime() + 60_000), finishedAt: old() }); + expect(await service.check(hidden)).toBeNull(); expect(await service.check(superseded)).toBeNull(); + expect(await state("failed_run", hiddenRun.id)).toBeNull(); expect(await state("failed_run", supersededRun.id)).toBeNull(); + expect(await db.select().from(decisionArchiveNotificationOutbox)).toHaveLength(0); + expect((await service.load(scope)).complete).toBe(true); + const refreshed = await service.load(scope); + expect(refreshed.rows).toHaveLength(98); + expect(refreshed.rows!.some(entry => [hiddenRun.id, supersededRun.id].includes(entry.sourceId))).toBe(false); + }); + + it("batches 100 blocker graphs with exact terminal timestamps and fresh terminal visibility", async () => { + const roots = await db.insert(issues).values(Array.from({ length: 100 }, () => ({ companyId, + title: randomUUID(), status: "blocked", updatedAt: old(), blockedTransitionAt: old(), + unblockDescriptor: { owner: "board", action: "Pick an owner" } }))).returning(); + const terminals = await db.insert(issues).values(Array.from({ length: 100 }, () => ({ companyId, + title: randomUUID(), status: "backlog", assigneeAgentId: agentId, updatedAt: old() }))).returning(); + await db.insert(issueRelations).values(roots.map((root, index) => ({ companyId, + issueId: terminals[index].id, relatedIssueId: root.id, type: "blocks" }))); + await db.execute(sql`update issues set updated_at = updated_at + interval '0.000321 seconds' where id = ${terminals[99].id}::uuid`); + const service = attentionRetentionService(db), scope = { companyId, sourceKind: "blocker_attention" as const }; + statements.length = 0; + const page = await service.load(scope); + const queries = [...statements]; + expect(page.rows).toHaveLength(200); expect(page.complete).toBe(false); + const retentionReads = queries.filter(query => query.includes('from "decision_retention"')); + expect(retentionReads).toHaveLength(200); + expect(queries.length - retentionReads.length).toBeLessThanOrEqual(30); + const terminal = page.rows!.find(entry => entry.rootId === roots[99].id && entry.variant === "terminal")!; + expect(terminal.sourceId).toBe(terminals[99].id); + const [expected] = await db.execute(sql`select updated_at::text as at, + ceil(extract(epoch from updated_at) * 1000 + ${delay}) as due from issues where id = ${terminals[99].id}::uuid`); + expect(terminal.activityAt).toBe(expected.at); expect(terminal.dueAt).toBe(Number(expected.due)); + await db.update(issues).set({ hiddenAt: new Date() }).where(eq(issues.id, terminals[99].id)); + expect(await service.check(terminal)).toBeNull(); + expect(await state("blocker_attention", terminals[99].id)).toBeNull(); + expect(await db.select().from(decisionArchiveNotificationOutbox)).toHaveLength(0); + expect((await service.load(scope)).complete).toBe(true); + const refreshed = await service.load(scope); + expect(refreshed.rows!.some(entry => entry.sourceId === terminals[99].id)).toBe(false); + }); + it("scopes ordinary execution-control writes to their company", async () => { const otherCompany = randomUUID(); await db.insert(companies).values({ id: otherCompany, name: "Other", issuePrefix: "OTHER" }); diff --git a/server/src/services/attention-retention.ts b/server/src/services/attention-retention.ts index 85e27a0d941..c72947e5349 100644 --- a/server/src/services/attention-retention.ts +++ b/server/src/services/attention-retention.ts @@ -1,4 +1,4 @@ -import { and, eq, inArray, isNull, sql, type SQL } from "drizzle-orm"; +import { and, eq, getTableColumns, inArray, isNull, sql, type SQL } from "drizzle-orm"; import { agents, decisionRetention, decisions, heartbeatRuns, issueApprovals, issueRecoveryActions, issueThreadInteractions, issues, approvals, type Db } from "@paperclipai/db"; import type { AttentionSourceKind } from "@paperclipai/shared"; @@ -23,6 +23,18 @@ type Candidate = { sourceKind: Kind; sourceId: string; companyId: string; activi type Entry = Candidate & { id: string; dueAt: number; rootId: string; variant?: "owner" | "terminal" }; type Cursor = { sourceKind: string; sourceId: string }; +type PageIssue = typeof issues.$inferSelect & { activityAt: string; deadlineAt: number; + blockedActivityAt: string | null; blockedDeadlineAt: number | null }; +type PageEvidence = { + decisions: Map; + recoveries: Map; + runs: Map; + agents: Set; + visibleIssues: Map; + newerRuns: Set; + blockers: Awaited["listBlockerAttention"]>>; +}; + const sourceTables: Record = { decision: sql`decisions`, issue_thread_interaction: sql`issue_thread_interactions`, recovery_action: sql`issue_recovery_actions`, blocker_attention: sql`issues`, review: sql`issues`, @@ -137,56 +149,106 @@ export function attentionRetentionService(db: Db) { return result; } - async function eligible(database: Db, candidate: Candidate, variant?: "owner" | "terminal", lock = false, reviewAttention?: Map, interactionAttention?: Map): Promise { + async function pageEvidence(database: Db, candidates: Candidate[]) { + const byCompany = new Map(); + for (const candidate of candidates) { + if (!["decision", "recovery_action", "failed_run", "blocker_attention"].includes(candidate.sourceKind)) continue; + const rows = byCompany.get(candidate.companyId) ?? []; + rows.push(candidate); + byCompany.set(candidate.companyId, rows); + } + const result = new Map(); + for (const [companyId, rows] of byCompany) { + const ids = (kind: Kind) => rows.filter(row => row.sourceKind === kind).map(row => row.sourceId); + const decisionIds = ids("decision"), recoveryIds = ids("recovery_action"), runIds = ids("failed_run"), blockerIds = ids("blocker_attention"); + const decisionRows = decisionIds.length ? await database.select({ id: decisions.id, originIssueId: decisions.originIssueId }) + .from(decisions).where(and(eq(decisions.companyId, companyId), inArray(decisions.id, decisionIds), eq(decisions.status, "open"))) : []; + const recoveryRows = recoveryIds.length ? await database.select().from(issueRecoveryActions).where(and( + eq(issueRecoveryActions.companyId, companyId), inArray(issueRecoveryActions.id, recoveryIds))) : []; + const runRows = runIds.length ? await database.select().from(heartbeatRuns).where(and( + eq(heartbeatRuns.companyId, companyId), inArray(heartbeatRuns.id, runIds))) : []; + const runAgents = runRows.length ? await database.select({ id: agents.id }).from(agents).where(and( + eq(agents.companyId, companyId), inArray(agents.id, [...new Set(runRows.map(row => row.agentId))]), + sql`${agents.status} <> 'terminated'`)) : []; + const readIssues = async (issueIds: string[]) => issueIds.length ? database.select({ ...getTableColumns(issues), + activityAt: sql`${issues.updatedAt}::text`, + deadlineAt: sql`ceil(extract(epoch from ${issues.updatedAt}) * 1000 + ${archiveDelay})`.mapWith(Number), + blockedActivityAt: sql`${issues.blockedTransitionAt}::text`, + blockedDeadlineAt: sql`ceil(extract(epoch from ${issues.blockedTransitionAt}) * 1000 + ${archiveDelay})`.mapWith(Number), + }).from(issues).where(and(eq(issues.companyId, companyId), inArray(issues.id, [...new Set(issueIds)]), + attentionIssueCondition(false))) : []; + const issueIds = [...blockerIds, ...decisionRows.map(row => row.originIssueId), ...recoveryRows.map(row => row.sourceIssueId), + ...runRows.map(row => readRunIssueId(row.contextSnapshot))].filter((id): id is string => Boolean(id)); + const visibleRows = await readIssues(issueIds); + const visible = new Map(visibleRows.map(row => [row.id, row])); + const blockerRoots = visibleRows.filter(row => blockerIds.includes(row.id)); + const blockers: PageEvidence["blockers"] = blockerRoots.length ? await issueService(database).listBlockerAttention(companyId, blockerRoots) : new Map(); + const terminalIds = [...blockers.values()].filter(terminalBlockerRequiresAttention) + .map(attention => attention.terminalBlockerIssueId).filter((id): id is string => id != null && !visible.has(id)); + for (const row of await readIssues(terminalIds)) visible.set(row.id, row); + result.set(companyId, { + decisions: new Map(decisionRows.map(row => [row.id, row])), recoveries: new Map(recoveryRows.map(row => [row.id, row])), + runs: new Map(runRows.map(row => [row.id, row])), agents: new Set(runAgents.map(row => row.id)), visibleIssues: visible, + newerRuns: await listFailedRunsWithNewerRun(database, companyId, + runRows.map(run => ({ ...run, issueId: readRunIssueId(run.contextSnapshot) }))), blockers, + }); + } + return result; + } + + async function eligible(database: Db, candidate: Candidate, variant?: "owner" | "terminal", lock = false, reviewAttention?: Map, interactionAttention?: Map, evidence?: PageEvidence): Promise { let sourceId = candidate.sourceId; let activityAt = candidate.activityAt; let dueAt = candidate.deadlineAt; if (candidate.sourceKind === "decision") { - const [row] = await database.select({ originIssueId: decisions.originIssueId }).from(decisions) - .where(and(eq(decisions.companyId, candidate.companyId), eq(decisions.id, sourceId), eq(decisions.status, "open"))); - if (!row || row.originIssueId && !await visibleIssue(database, candidate.companyId, row.originIssueId)) return null; + const row = evidence ? evidence.decisions.get(sourceId) : (await database.select({ originIssueId: decisions.originIssueId }).from(decisions) + .where(and(eq(decisions.companyId, candidate.companyId), eq(decisions.id, sourceId), eq(decisions.status, "open"))))[0]; + if (!row || row.originIssueId && !(evidence ? evidence.visibleIssues.has(row.originIssueId) + : await visibleIssue(database, candidate.companyId, row.originIssueId))) return null; } else if (candidate.sourceKind === "issue_thread_interaction") { const attention = interactionAttention ?? await interactionEligibility(database, [candidate]); if (!attention.get(sourceId)) return null; } else if (candidate.sourceKind === "recovery_action") { - const [row] = await database.select().from(issueRecoveryActions).where(and(eq(issueRecoveryActions.companyId, candidate.companyId), eq(issueRecoveryActions.id, sourceId))); - if (!row || !await visibleIssue(database, candidate.companyId, row.sourceIssueId)) return null; + const row = evidence ? evidence.recoveries.get(sourceId) : (await database.select().from(issueRecoveryActions).where(and(eq(issueRecoveryActions.companyId, candidate.companyId), eq(issueRecoveryActions.id, sourceId))))[0]; + if (!row || !(evidence ? evidence.visibleIssues.has(row.sourceIssueId) + : await visibleIssue(database, candidate.companyId, row.sourceIssueId))) return null; } else if (candidate.sourceKind === "review") { const attention = reviewAttention ?? await reviewEligibility(database, [candidate]); if (!attention.get(sourceId)) return null; } else if (candidate.sourceKind === "blocker_attention") { - const row = await visibleIssue(database, candidate.companyId, sourceId); + const row = evidence ? evidence.visibleIssues.get(sourceId) : await visibleIssue(database, candidate.companyId, sourceId); if (!row) return null; if (variant === "owner") { if (row.unblockDescriptor?.owner !== "board" || !isProspectiveBlockedTransition(row)) return null; - const [activity] = await database.execute(sql`select blocked_transition_at::text as at, + const activity = evidence ? { at: evidence.visibleIssues.get(sourceId)!.blockedActivityAt!, due: evidence.visibleIssues.get(sourceId)!.blockedDeadlineAt! } : (await database.execute(sql`select blocked_transition_at::text as at, ceil(extract(epoch from blocked_transition_at) * 1000 + ${archiveDelay}) as due from issues - where company_id = ${candidate.companyId}::uuid and id = ${row.id}::uuid`) as unknown as Array<{ at: string; due: string }>; + where company_id = ${candidate.companyId}::uuid and id = ${row.id}::uuid`) as unknown as Array<{ at: string; due: string }>)[0]; activityAt = activity.at; dueAt = Number(activity.due); } else { - const attention = (await issueService(database).listBlockerAttention(candidate.companyId, [row])).get(row.id); + const attention = evidence ? evidence.blockers.get(row.id) : (await issueService(database).listBlockerAttention(candidate.companyId, [row])).get(row.id); if (!terminalBlockerRequiresAttention(attention)) return null; sourceId = attention.terminalBlockerIssueId ?? row.id; if (lock && sourceId !== candidate.sourceId) await database.execute(sql`select id from issues where company_id = ${candidate.companyId}::uuid and id = ${sourceId}::uuid for update`); - const terminal = await visibleIssue(database, candidate.companyId, sourceId); + const terminal = evidence ? evidence.visibleIssues.get(sourceId) : await visibleIssue(database, candidate.companyId, sourceId); if (!terminal) return null; - const [activity] = await database.execute(sql`select updated_at::text as at, + const activity = evidence ? { at: evidence.visibleIssues.get(sourceId)!.activityAt, due: evidence.visibleIssues.get(sourceId)!.deadlineAt } : (await database.execute(sql`select updated_at::text as at, ceil(extract(epoch from updated_at) * 1000 + ${archiveDelay}) as due from issues - where company_id = ${candidate.companyId}::uuid and id = ${sourceId}::uuid`) as unknown as Array<{ at: string; due: string }>; + where company_id = ${candidate.companyId}::uuid and id = ${sourceId}::uuid`) as unknown as Array<{ at: string; due: string }>)[0]; activityAt = activity.at; dueAt = Number(activity.due); } const [retention] = await database.select().from(decisionRetention).where(and(eq(decisionRetention.companyId, candidate.companyId), eq(decisionRetention.sourceKind, candidate.sourceKind), eq(decisionRetention.sourceId, sourceId))); if (retention?.keep || retention?.archivedAt) return null; } else if (candidate.sourceKind === "failed_run") { - const [run] = await database.select().from(heartbeatRuns).where(and(eq(heartbeatRuns.companyId, candidate.companyId), eq(heartbeatRuns.id, sourceId))); + const run = evidence ? evidence.runs.get(sourceId) : (await database.select().from(heartbeatRuns).where(and(eq(heartbeatRuns.companyId, candidate.companyId), eq(heartbeatRuns.id, sourceId))))[0]; if (!run) return null; - const [agent] = await database.select({ id: agents.id }).from(agents).where(and(eq(agents.companyId, candidate.companyId), - eq(agents.id, run.agentId), sql`${agents.status} <> 'terminated'`)); + const agent = evidence ? evidence.agents.has(run.agentId) : (await database.select({ id: agents.id }).from(agents).where(and(eq(agents.companyId, candidate.companyId), + eq(agents.id, run.agentId), sql`${agents.status} <> 'terminated'`)))[0]; const issueId = readRunIssueId(run.contextSnapshot); - if (!agent || issueId && !await visibleIssue(database, candidate.companyId, issueId)) return null; - if ((await listFailedRunsWithNewerRun(database, candidate.companyId, [{ ...run, issueId }])).has(run.id)) return null; + if (!agent || issueId && !(evidence ? evidence.visibleIssues.has(issueId) + : await visibleIssue(database, candidate.companyId, issueId))) return null; + if ((evidence ? evidence.newerRuns : await listFailedRunsWithNewerRun(database, candidate.companyId, [{ ...run, issueId }])).has(run.id)) return null; } return { ...candidate, sourceId, activityAt, rootId: candidate.sourceId, variant, id: `${candidate.sourceKind}:${candidate.sourceId}:${variant ?? "source"}`, dueAt }; @@ -198,10 +260,11 @@ export function attentionRetentionService(db: Db) { const entries: Entry[] = []; const reviews = await reviewEligibility(db, rows); const interactions = await interactionEligibility(db, rows); + const evidence = await pageEvidence(db, rows); for (const row of rows) { if (signal?.aborted) return { rows: null, complete: true }; for (const variant of row.sourceKind === "blocker_attention" ? ["owner", "terminal"] as const : [undefined]) { - const entry = await eligible(db, row, variant, false, reviews, interactions); + const entry = await eligible(db, row, variant, false, reviews, interactions, evidence.get(row.companyId)); if (entry) entries.push(entry); } } From 259a3ae4dfd5d837c0307615e97a51272eecd5a1 Mon Sep 17 00:00:00 2001 From: Dotta Date: Fri, 30 Oct 2026 18:01:35 -0500 Subject: [PATCH 41/74] fix(server): scope backup invalidation to retained session references Co-Authored-By: Paperclip --- .../2026-10-30-backup-expiration-sleep.md | 15 +- .../native-harness-backup-expiration.test.ts | 160 +++++++++++++++++- server/src/services/environments.ts | 4 +- server/src/services/issues.ts | 2 +- ...ve-environment-settlement-notifications.ts | 4 +- ...harness-backup-expiration-notifications.ts | 11 +- .../native-harness-backup-expiration-work.ts | 111 ++++++++---- .../native-harness-backup-expiration.ts | 76 +++++++-- .../native-runtime/native-session-executor.ts | 11 +- ...ed-native-session-cleanup-notifications.ts | 11 +- 10 files changed, 335 insertions(+), 70 deletions(-) diff --git a/doc/plans/2026-10-30-backup-expiration-sleep.md b/doc/plans/2026-10-30-backup-expiration-sleep.md index 91cf1e9ec0a..e5b3132fbc8 100644 --- a/doc/plans/2026-10-30-backup-expiration-sleep.md +++ b/doc/plans/2026-10-30-backup-expiration-sleep.md @@ -14,9 +14,13 @@ must invalidate the source when it changes. A busy process or failed I/O leaves real unresolved cleanup and uses the shared retry backoff. Dry-run reports an eligible source once and waits for source invalidation or restart. -Startup pages existing disk scopes and indexes company ownership, including -protected backups. Company notifications inspect only that index; local backup -publication adds newly created roots. Roots without backup or cleanup tombstones +Startup pages existing disk scopes and indexes each backup's actual session, +run, coalesced native-task/task, agent and lease references, including protected +backups. Ordinary task, run, lease and process-release notifications retain +these identities and inspect only affected roots. New run or lease references +resolve through the owner's database relationships. Local publication retains +its exact digest. Explicit company or instance configuration changes can still +reconstruct their whole affected scope. Roots without backup or cleanup tombstones perform no backup SQL. After discovery and all obligations settle, there is no recurring SQL or filesystem scan. Startup reconstructs future work; inspecting a source never changes its business timestamps. Runtime task close/reopen and @@ -31,6 +35,11 @@ manual disk edits require reconstruction on restart. Database writer ingress continues to use the existing transaction-aware work signals. This change adds no permanent database listener and does not add a polling table or migration. +Read-only reconstruction does not lock issue rows. Transient identity I/O errors +retain a retry obligation; missing or malformed state remains unrecognized. +Byte accounting occurs before the retirement transaction. The current backup +age is rechecked inside the transaction before the rename. + The existing issue share locks now remain held through the retirement rename. This fences task reopen while the backup becomes unavailable. A failed database transaction restores `.retiring-*`; a process interruption is recovered from diff --git a/server/src/__tests__/native-harness-backup-expiration.test.ts b/server/src/__tests__/native-harness-backup-expiration.test.ts index afadf5d1d63..9dddca536f3 100644 --- a/server/src/__tests__/native-harness-backup-expiration.test.ts +++ b/server/src/__tests__/native-harness-backup-expiration.test.ts @@ -1,3 +1,6 @@ +import { notifyNativeEnvironmentSettlement } from "../services/native-environment-settlement-notifications.js"; +import { notifyRetainedNativeSessionCleanup, onRetainedNativeSessionCleanupRelease, releaseRetainedNativeSessionCleanup } from "../services/retained-native-session-cleanup-notifications.js"; +import { environmentService } from "../services/environments.js"; import { agentService } from "../services/agents.js"; import { publishNativeHarnessBackup } from "../services/native-runtime/native-harness-backup-publication.js"; import { registerNativeHarnessBackupExpirationWork } from "../services/native-runtime/native-harness-backup-expiration-work.js"; @@ -14,7 +17,7 @@ import os from "node:os"; import path from "node:path"; import { promises as fs } from "node:fs"; import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest"; -import { eq } from "drizzle-orm"; +import { eq, sql } from "drizzle-orm"; import { agents, agentTaskSessions, @@ -84,6 +87,7 @@ describeEmbeddedPostgres("native harness backup expiration", () => { }); async function seedSession(input: { + companyId?: string; agentId?: string; issueStatus?: string; closedDaysAgo?: number; runStatus?: string; @@ -93,9 +97,10 @@ describeEmbeddedPostgres("native harness backup expiration", () => { cleanupActive?: boolean; } = {}) { if (!stateBase) stateBase = await fs.mkdtemp(path.join(os.tmpdir(), "paperclip-durable-sessions-")); - const companyId = randomUUID(); - await db.insert(companies).values({ id: companyId, name: "Backups", issuePrefix: `B${companyId.slice(0, 6)}` }); - const [agent] = await db.insert(agents).values({ companyId, name: "Coder" }).returning(); + const companyId = input.companyId ?? randomUUID(); + if (!input.companyId) await db.insert(companies).values({ id: companyId, name: "Backups", issuePrefix: `B${companyId.slice(0, 6)}` }); + const [agent] = input.agentId ? [{ id: input.agentId }] + : await db.insert(agents).values({ companyId, name: "Coder" }).returning(); const issueStatus = input.issueStatus ?? "done"; const closedAt = new Date(now.getTime() - (input.closedDaysAgo ?? 10) * DAY_MS); const [issue] = await db.insert(issues).values({ @@ -733,6 +738,153 @@ describeEmbeddedPostgres("native harness backup expiration", () => { await quiet(); }); + it("coalesces repeated task writes to one retained root in a large company with the same historical agent", async () => { + const own = await seedSession({ issueStatus: "todo" }); + const unrelated = []; + for (let index = 0; index < 80; index++) unrelated.push(await seedSession({ companyId: own.companyId, + agentId: own.agentId, issueStatus: index % 2 ? "todo" : "done", closedDaysAgo: 2, + runStatus: index % 3 ? "succeeded" : "running" })); + const due = now.getTime() + 5 * DAY_MS + 1; + await startBackups({ isSessionBusy: digest => unrelated.some((entry, index) => index % 4 === 0 && entry.scope === digest) }); + await settled(due); await quiet(60_000); + const reads = vi.spyOn(fs, "readFile"); + const queries: Array<{ query: string; parameters: unknown[] }> = []; + const previous = db.$client.options.debug; + db.$client.options.debug = (_connection, query, parameters) => { queries.push({ query, parameters }); }; + try { + await db.transaction(async tx => { + await issueService(db).update(own.issueId, { status: "done" }, tx); + for (let index = 0; index < 12; index++) await issueService(db).update(own.issueId, + { completedAt: new Date(now.getTime() - DAY_MS) }, tx); + }); + const writes = queries.length; + await settled(due); + expect(reads.mock.calls.map(call => String(call[0]))).toEqual([path.join(own.root, "control-plane", "control-plane-state.json")]); + expect(queries.length - writes).toBeLessThanOrEqual(20); + const otherIds = unrelated.flatMap(entry => [entry.issueId, entry.runId, entry.sessionId]); + expect(queries.filter(query => query.parameters.some(parameter => otherIds.some(id => String(parameter).includes(id))))).toEqual([]); + } finally { reads.mockRestore(); db.$client.options.debug = previous; } + await issueService(db).update(own.issueId, { status: "todo" }); await settled(due); + await quiet(); expect(await exists(own.current)).toBe(true); + }); + + it("indexes protected session relations so a new run, historical task close and exact process release reactivate only its backup", async () => { + const own = await seedSession({ closedDaysAgo: 2 }); + const other = await seedSession({ companyId: own.companyId, agentId: own.agentId, issueStatus: "todo" }); + const [linked] = await db.insert(issues).values({ companyId: own.companyId, title: "New linked task", status: "todo" }).returning(); + const due = now.getTime() + 5 * DAY_MS + 1; + await startBackups(); await settled(due); await quiet(60_000); + const reads = vi.spyOn(fs, "readFile"); + const runId = randomUUID(); + try { + await db.transaction(async tx => { + await notifyNativeEnvironmentSettlement(tx, { companyId: own.companyId, runId, issueId: linked!.id }); + await tx.insert(heartbeatRuns).values({ id: runId, companyId: own.companyId, agentId: own.agentId, + nativeSessionId: own.sessionId, nativeIssueId: linked!.id, issueId: other.issueId, status: "running" }); + }); + await settled(); + expect(reads.mock.calls.every(call => String(call[0]).startsWith(own.root))).toBe(true); + expect(reads).toHaveBeenCalled(); reads.mockClear(); + await issueService(db).update(linked!.id, { status: "done" }); await settled(); + expect(reads.mock.calls.every(call => String(call[0]).startsWith(own.root))).toBe(true); reads.mockClear(); + await db.transaction(async tx => { + await notifyRetainedNativeSessionCleanup(tx, { companyId: own.companyId, runId, agentId: own.agentId }); + await tx.update(heartbeatRuns).set({ status: "succeeded", finishedAt: new Date() }).where(eq(heartbeatRuns.id, runId)); + }); + const [closed] = await db.select().from(issues).where(eq(issues.id, linked!.id)); + const later = new Date(closed!.completedAt!).getTime() + 7 * DAY_MS + 1; + await settled(later); reads.mockClear(); + const cleanupScope = { companyId: own.companyId, agentId: own.agentId }; + const observed = vi.fn(); const stop = onRetainedNativeSessionCleanupRelease(db, observed); + releaseRetainedNativeSessionCleanup(db, cleanupScope, { companyId: own.companyId, runId }); + expect(observed).toHaveBeenCalledWith(cleanupScope, { companyId: own.companyId, runId }); stop(); + await settled(later); + expect(reads.mock.calls.every(call => String(call[0]).startsWith(own.root))).toBe(true); + expect(reads).toHaveBeenCalled(); + await issueService(db).update(linked!.id, { status: "todo" }); await settled(); + } finally { reads.mockRestore(); } + expect(await exists(other.current)).toBe(true); await quiet(); + }); + + it("ordinary lease release without a run uses its source identity instead of other retained roots", async () => { + const own = await seedSession({ leaseStatus: "retained" }); + const other = await seedSession({ companyId: own.companyId, agentId: own.agentId, issueStatus: "todo" }); + await startBackups(); await settled(); await quiet(60_000); + const [lease] = await db.select().from(environmentLeases).where(eq(environmentLeases.companyId, own.companyId)); + const reads = vi.spyOn(fs, "readFile"); + try { + await environmentService(db).releaseLease(lease!.id, "expired"); await settled(); + expect(reads).toHaveBeenCalled(); + expect(reads.mock.calls.every(call => String(call[0]).startsWith(own.root))).toBe(true); + } finally { reads.mockRestore(); } + expect(await exists(own.current)).toBe(false); expect(await exists(other.current)).toBe(true); await quiet(); + }); + + it("retains the future deadline when a lease metadata edit removes its last backup reference", async () => { + const own = await seedSession({ leaseStatus: "retained", closedDaysAgo: 2 }); + const other = await seedSession({ companyId: own.companyId, agentId: own.agentId, issueStatus: "todo" }); + await startBackups(); await settled(); + const [lease] = await db.select().from(environmentLeases).where(eq(environmentLeases.companyId, own.companyId)); + const reads = vi.spyOn(fs, "readFile"); const due = now.getTime() + 5 * DAY_MS + 1; + try { + await environmentService(db).updateLeaseMetadata(lease!.id, {}); await settled(due); + expect(reads).toHaveBeenCalled(); expect(reads.mock.calls.every(call => String(call[0]).startsWith(own.root))).toBe(true); + } finally { reads.mockRestore(); } + await vi.advanceTimersByTimeAsync(due - Date.now()); await settled(); + expect(await exists(own.current)).toBe(false); expect(await exists(other.current)).toBe(true); await quiet(); + }); + + it("allows an ordinary task reopen while backup byte counting is held outside the retirement lock", async () => { + const own = await seedSession(); const lstat = fs.lstat; + const entered = Promise.withResolvers(), release = Promise.withResolvers(); + const held = vi.spyOn(fs, "lstat").mockImplementation(async (...args) => { + if (String(args[0]) === path.join(own.current, "codex-home", "history.jsonl")) { + entered.resolve(); await release.promise; + } + return lstat(...args); + }); + const startup = startBackups(); await entered.promise; + try { + await db.transaction(async tx => { + await tx.execute(sql`set local lock_timeout = '500ms'`); + await issueService(db).update(own.issueId, { status: "todo" }, tx); + }); + } finally { release.resolve(); await startup; held.mockRestore(); } + await settled(); expect(await exists(own.current)).toBe(true); expect(backupErrors).toEqual([]); await quiet(); + }); + + it.each(["EIO", "EACCES"])("retains a failed identity read as real retry debt (%s)", async code => { + const own = await seedSession(); const read = fs.readFile; + const fault = vi.spyOn(fs, "readFile").mockImplementation(async (...args) => { + if (String(args[0]) === path.join(own.root, "control-plane", "control-plane-state.json")) { + throw Object.assign(new Error("controlled identity I/O failure"), { code }); + } + return read(...args); + }); + await startBackups(); expect(backupErrors).toHaveLength(1); + const retry = coordinator!.nextWakeAt()!; expect(retry - Date.now()).toBeGreaterThanOrEqual(29_000); + expect(await exists(own.current)).toBe(true); fault.mockRestore(); await quiet(5000); + await vi.advanceTimersByTimeAsync(retry - Date.now()); await settled(); + expect(await exists(own.current)).toBe(false); expect(await taskSessionCount(own.agentId)).toBe(0); await quiet(); + }); + + it("reconstructs future retention without issue row locks", async () => { + await seedSession({ closedDaysAgo: 2 }); const queries: string[] = []; + const previous = db.$client.options.debug; + db.$client.options.debug = (_connection, query) => { queries.push(query); }; + try { + await startBackups(); await settled(now.getTime() + 5 * DAY_MS + 1); + expect(queries.some(query => /for\s+share/i.test(query))).toBe(false); + } finally { db.$client.options.debug = previous; } + await quiet(); + }); + + it("rejects destructive backup scopes outside the controller digest namespace", async () => { + await emptyStateBase(); const remove = vi.spyOn(fs, "rm"); + await expect(service().expireBackup("../outside")).rejects.toThrow("Invalid native harness backup scope"); + expect(remove).not.toHaveBeenCalled(); remove.mockRestore(); + }); + it.each([false, true])("finishes cleanup journals after ordinary agent deletion (restart %s)", async restart => { const session = await seedSession(); const remove = fs.rm; const fault = vi.spyOn(fs, "rm").mockImplementation(async (...args) => { diff --git a/server/src/services/environments.ts b/server/src/services/environments.ts index ea72243e464..831ab854f04 100644 --- a/server/src/services/environments.ts +++ b/server/src/services/environments.ts @@ -1701,7 +1701,7 @@ export function environmentService(db: Db) { cleanupStatus: environmentLeases.cleanupStatus, releasedAt: environmentLeases.releasedAt, }).from(environmentLeases).where(condition).for("update"); if (!previous) return null; - await notifyEnvironmentCleanup(tx, { companyId: previous.companyId, runId: previous.heartbeatRunId ?? undefined }); + await notifyEnvironmentCleanup(tx, { companyId: previous.companyId, runId: previous.heartbeatRunId ?? undefined, leaseId: id }); if (previous.heartbeatRunId && (previous.status !== status || (options?.cleanupStatus !== undefined && previous.cleanupStatus !== options.cleanupStatus) || (status !== "retained" && !previous.releasedAt))) { @@ -1821,7 +1821,7 @@ export function environmentService(db: Db) { const [lease] = await tx.select({ companyId: environmentLeases.companyId, heartbeatRunId: environmentLeases.heartbeatRunId, metadata: environmentLeases.metadata }) .from(environmentLeases).where(eq(environmentLeases.id, id)).for("update"); if (!lease) return null; - await notifyEnvironmentCleanup(tx, { companyId: lease.companyId, runId: lease.heartbeatRunId ?? undefined }); + await notifyEnvironmentCleanup(tx, { companyId: lease.companyId, runId: lease.heartbeatRunId ?? undefined, leaseId: id }); return tx.update(environmentLeases) .set({ metadata: preserveReusableLeaseHandoff(lease.metadata, withoutCleanupVerification(metadata)), diff --git a/server/src/services/issues.ts b/server/src/services/issues.ts index fdb02c5ea9e..6b654a701f8 100644 --- a/server/src/services/issues.ts +++ b/server/src/services/issues.ts @@ -11488,7 +11488,7 @@ export function issueService(db: Db) { patch.statusVersion = sql`${issues.statusVersion} + 1` as unknown as number; } if (patch.status !== undefined || patch.completedAt !== undefined || patch.cancelledAt !== undefined) { - await notifyNativeHarnessBackupExpiration(tx, { companyId: receiptExisting.companyId }); + await notifyNativeHarnessBackupExpiration(tx, { companyId: receiptExisting.companyId, issueId: id }); } if ((patch.status !== undefined && patch.status !== receiptExisting.status) || patch.completedAt !== undefined || patch.cancelledAt !== undefined diff --git a/server/src/services/native-environment-settlement-notifications.ts b/server/src/services/native-environment-settlement-notifications.ts index 427c43522b7..0c55810b6a8 100644 --- a/server/src/services/native-environment-settlement-notifications.ts +++ b/server/src/services/native-environment-settlement-notifications.ts @@ -2,9 +2,9 @@ import { notifyNativeHarnessBackupExpiration } from "./native-harness-backup-exp import { notifySkillVersionRetention } from "./skill-version-retention-notifications.js"; import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js"; -export type NativeEnvironmentSettlementScope = { companyId: string; runId?: string; issueId?: string }; +export type NativeEnvironmentSettlementScope = { companyId: string; runId?: string; issueId?: string; leaseId?: string }; export async function notifyNativeEnvironmentSettlement(tx: object, scope?: NativeEnvironmentSettlementScope) { - await notifyNativeHarnessBackupExpiration(tx, scope && { companyId: scope.companyId }); + await notifyNativeHarnessBackupExpiration(tx, scope); if (scope) await notifySkillVersionRetention(tx, scope.companyId); await notifyDeliveryWork(tx, DELIVERY_QUEUES.nativeEnvironmentSettlement, scope ? JSON.stringify({ companyId: scope.companyId, runId: scope.runId, issueId: scope.issueId }) : "*"); diff --git a/server/src/services/native-harness-backup-expiration-notifications.ts b/server/src/services/native-harness-backup-expiration-notifications.ts index 602a7063acd..e334487a82c 100644 --- a/server/src/services/native-harness-backup-expiration-notifications.ts +++ b/server/src/services/native-harness-backup-expiration-notifications.ts @@ -1,9 +1,16 @@ import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js"; -export type NativeHarnessBackupExpirationScope = { companyId: string; scopeDigest?: string }; +export type NativeHarnessBackupExpirationScope = { + companyId: string; scopeDigest?: string; leaseId?: string; runId?: string; issueId?: string; agentId?: string; sessionId?: string; +}; + +export function nativeHarnessBackupScopeHint(scope: NativeHarnessBackupExpirationScope) { + return JSON.stringify({ companyId: scope.companyId, scopeDigest: scope.scopeDigest, leaseId: scope.leaseId, + runId: scope.runId, issueId: scope.issueId, agentId: scope.agentId, sessionId: scope.sessionId }); +} export function notifyNativeHarnessBackupExpiration(tx: object, scope?: NativeHarnessBackupExpirationScope) { - return notifyDeliveryWork(tx, DELIVERY_QUEUES.nativeHarnessBackupExpiration, scope ? JSON.stringify(scope) : "*"); + return notifyDeliveryWork(tx, DELIVERY_QUEUES.nativeHarnessBackupExpiration, scope ? nativeHarnessBackupScopeHint(scope) : "*"); } const listeners = new WeakMap void>>(); diff --git a/server/src/services/native-runtime/native-harness-backup-expiration-work.ts b/server/src/services/native-runtime/native-harness-backup-expiration-work.ts index f0dafe195e0..174f804b3a5 100644 --- a/server/src/services/native-runtime/native-harness-backup-expiration-work.ts +++ b/server/src/services/native-runtime/native-harness-backup-expiration-work.ts @@ -2,12 +2,14 @@ import type { Db } from "@paperclipai/db"; import { logger } from "../../middleware/logger.js"; import type { createDeliveryWorkCoordinator } from "../delivery-work-coordinator.js"; import { DELIVERY_QUEUES } from "../delivery-work-notifications.js"; -import { onNativeHarnessBackupChange, type NativeHarnessBackupExpirationScope } from "../native-harness-backup-expiration-notifications.js"; +import { nativeHarnessBackupScopeHint, onNativeHarnessBackupChange, type NativeHarnessBackupExpirationScope } from "../native-harness-backup-expiration-notifications.js"; import { onRetainedNativeSessionCleanupRelease } from "../retained-native-session-cleanup-notifications.js"; import { registerScopedDeadlineWork } from "../scoped-deadline-work.js"; -import { nativeHarnessBackupExpirationService, type NativeHarnessBackupExpirationOptions } from "./native-harness-backup-expiration.js"; +import { nativeHarnessBackupExpirationService, type NativeHarnessBackupDependencies, type NativeHarnessBackupExpirationOptions } from "./native-harness-backup-expiration.js"; -type Entry = { id: string; companyId: string; agentId: string; dueAt: number }; +type Dependencies = { companyId: string; agentId: string; sessionId: string; runIds: Set; issueIds: Set; leaseIds: Set }; +type Entry = { id: string; companyId: string; agentId: string; dueAt: number; dependencies: Dependencies }; +type Reference = "company" | "agent" | "session" | "run" | "issue" | "lease"; export function registerNativeHarnessBackupExpirationWork( coordinator: ReturnType, db: Db, @@ -15,46 +17,94 @@ export function registerNativeHarnessBackupExpirationWork( ) { const service = nativeHarnessBackupExpirationService(db, { ...options, duty: "backups" }); const cursors = new Map(); - const roots = new Map>(); - const owners = new Map(); - function remember(id: string, companyId?: string) { - const previous = owners.get(id); - if (previous && previous !== companyId) { - const companyRoots = roots.get(previous)!; - companyRoots.delete(id); - if (!companyRoots.size) roots.delete(previous); - owners.delete(id); + const sources = new Map(); + const index = new Map>(); + const key = (companyId: string, kind: Reference, id = "") => JSON.stringify([companyId, kind, id]); + const sessionKey = (agentId: string, sessionId: string) => JSON.stringify([agentId, sessionId]); + function references(source: Dependencies) { + return [key(source.companyId, "company"), key(source.companyId, "agent", source.agentId), + key(source.companyId, "session", sessionKey(source.agentId, source.sessionId)), + ...[...source.runIds].map(id => key(source.companyId, "run", id)), + ...[...source.issueIds].map(id => key(source.companyId, "issue", id)), + ...[...source.leaseIds].map(id => key(source.companyId, "lease", id))]; + } + function add(reference: string, id: string) { + let roots = index.get(reference); + if (!roots) { roots = new Set(); index.set(reference, roots); } + roots.add(id); + } + function remember(id: string, current?: NativeHarnessBackupDependencies) { + const previous = sources.get(id); + if (previous) for (const reference of references(previous)) { + const roots = index.get(reference)!; + roots.delete(id); if (!roots.size) index.delete(reference); + } + if (!current) { sources.delete(id); return undefined; } + const source = { companyId: current.companyId, agentId: current.agentId, + sessionId: current.sessionId, runIds: new Set(current.runIds), issueIds: new Set(current.issueIds), leaseIds: new Set(current.leaseIds) }; + sources.set(id, source); + for (const reference of references(source)) add(reference, id); + return source; + } + function selected(scope: NativeHarnessBackupExpirationScope): Set { + if (scope.scopeDigest) return new Set([scope.scopeDigest]); + const reference = scope.leaseId ? key(scope.companyId, "lease", scope.leaseId) + : scope.runId ? key(scope.companyId, "run", scope.runId) + : scope.sessionId && scope.agentId ? key(scope.companyId, "session", sessionKey(scope.agentId, scope.sessionId)) + : scope.issueId ? key(scope.companyId, "issue", scope.issueId) + : scope.agentId ? key(scope.companyId, "agent", scope.agentId) : key(scope.companyId, "company"); + return new Set(index.get(reference) ?? []); + } + async function discoverReferences(scope: NativeHarnessBackupExpirationScope) { + if (scope.scopeDigest || (!scope.runId && !scope.leaseId)) return; + const resolved = await service.resolveBackupScope(scope); + const roots = selected(scope); + if (resolved.scopeDigest) roots.add(resolved.scopeDigest); + const related = resolved.sessionId && resolved.agentId + ? selected({ companyId: scope.companyId, sessionId: resolved.sessionId, agentId: resolved.agentId }) + : resolved.issueId ? selected({ companyId: scope.companyId, issueId: resolved.issueId }) : []; + for (const id of related) roots.add(id); + for (const id of roots) { + const source = sources.get(id); + if (!source || source.companyId !== scope.companyId) continue; + if (scope.runId) { source.runIds.add(scope.runId); add(key(scope.companyId, "run", scope.runId), id); } + if (scope.leaseId) { source.leaseIds.add(scope.leaseId); add(key(scope.companyId, "lease", scope.leaseId), id); } } - if (!companyId) return; - let companyRoots = roots.get(companyId); - if (!companyRoots) { companyRoots = new Set(); roots.set(companyId, companyRoots); } - companyRoots.add(id); owners.set(id, companyId); } async function reconcile(id: string): Promise { const source = await service.reconcileBackup(id); - remember(id, source?.companyId); - return source && source.dueAt !== null ? { ...source, dueAt: source.dueAt } : null; + const dependencies = remember(id, source?.dependencies); + return source && dependencies && source.dueAt !== null ? { ...source, dependencies, dueAt: source.dueAt } : null; + } + function matches(entry: Entry, scope?: NativeHarnessBackupExpirationScope) { + if (!scope) return true; + if (entry.companyId !== scope.companyId) return false; + if (scope.scopeDigest) return entry.id === scope.scopeDigest; + if (scope.leaseId) return entry.dependencies.leaseIds.has(scope.leaseId); + if (scope.runId) return entry.dependencies.runIds.has(scope.runId); + if (scope.sessionId) return entry.dependencies.sessionId === scope.sessionId + && (!scope.agentId || entry.agentId === scope.agentId); + if (scope.issueId) return entry.dependencies.issueIds.has(scope.issueId); + return !scope.agentId || entry.agentId === scope.agentId; } const worker = registerScopedDeadlineWork(coordinator, { queue: DELIVERY_QUEUES.nativeHarnessBackupExpiration, batchSize: options.maxRemovals ?? 25, decodeScope: (hint: string): NativeHarnessBackupExpirationScope | undefined => hint === "*" ? undefined : JSON.parse(hint), - matches: (entry: Entry, scope?: NativeHarnessBackupExpirationScope) => !scope || entry.companyId === scope.companyId - && (!scope.scopeDigest || entry.id === scope.scopeDigest), + matches, load: async (scope, signal) => { - const key = scope ? JSON.stringify(scope) : "*"; + const hint = scope ? nativeHarnessBackupScopeHint(scope) : "*"; + if (scope && !cursors.has(hint)) await discoverReferences(scope); const limit = options.maxEntries ?? 100; - const indexed = scope ? [...roots.get(scope.companyId) ?? []] - .filter(id => !cursors.get(key) || id > cursors.get(key)!).sort() : null; - const page = scope?.scopeDigest ? { names: [scope.scopeDigest], complete: true } - : indexed ? { names: indexed.slice(0, limit), complete: indexed.length <= limit } - : await service.backupScopes(cursors.get(key), limit); + const indexed = scope ? [...selected(scope)].filter(id => !cursors.get(hint) || id > cursors.get(hint)!).sort() : null; + const page = indexed ? { names: indexed.slice(0, limit), complete: indexed.length <= limit } + : await service.backupScopes(cursors.get(hint), limit); const rows: Entry[] = []; for (const name of page.names) { if (signal?.aborted || !canRun()) throw new Error("Backup expiration discovery paused"); const entry = await reconcile(name); if (entry && (!scope || entry.companyId === scope.companyId)) rows.push(entry); } - if (page.complete) cursors.delete(key); else cursors.set(key, page.names.at(-1)!); + if (page.complete) cursors.delete(hint); else cursors.set(hint, page.names.at(-1)!); return { rows, complete: page.complete }; }, check: async (entry, signal) => { @@ -71,10 +121,7 @@ export function registerNativeHarnessBackupExpirationWork( return remaining; }, }, canRun); - const unsubscribe = onNativeHarnessBackupChange(db, scope => { - if (scope.scopeDigest) remember(scope.scopeDigest, scope.companyId); - worker.invalidate(JSON.stringify(scope)); - }); - const release = onRetainedNativeSessionCleanupRelease(db, scope => worker.invalidate(JSON.stringify({ companyId: scope.companyId }))); + const unsubscribe = onNativeHarnessBackupChange(db, scope => worker.invalidate(nativeHarnessBackupScopeHint(scope))); + const release = onRetainedNativeSessionCleanupRelease(db, (_scope, source) => worker.invalidate(nativeHarnessBackupScopeHint(source))); return { ...worker, async stop() { unsubscribe(); release(); await worker.stop(); } }; } diff --git a/server/src/services/native-runtime/native-harness-backup-expiration.ts b/server/src/services/native-runtime/native-harness-backup-expiration.ts index ad9ac18cfb2..b1d8c7c6f1d 100644 --- a/server/src/services/native-runtime/native-harness-backup-expiration.ts +++ b/server/src/services/native-runtime/native-harness-backup-expiration.ts @@ -17,6 +17,7 @@ import { nativeHarnessStateBase } from "./native-harness-backup-stamp.js"; import { nativeSessionScopeDigestIsBusy } from "./native-session-executor.js"; import type { createDeliveryWorkCoordinator } from "../delivery-work-coordinator.js"; import { DELIVERY_QUEUES } from "../delivery-work-notifications.js"; +import type { NativeHarnessBackupExpirationScope } from "../native-harness-backup-expiration-notifications.js"; export const NATIVE_HARNESS_BACKUP_EXPIRATION_INTERVAL_MS = 60 * 60 * 1000; export const NATIVE_HARNESS_BACKUP_EXPIRATION_STARTUP_DELAY_MS = 2 * 60 * 1000; @@ -146,7 +147,9 @@ async function readSessionIdentity(root: string): Promise MAX_CONTROL_PLANE_STATE_BYTES) return null; const state = JSON.parse(await fs.readFile(statePath, "utf8")) as { identity?: Record }; + if (!state || typeof state !== "object" || Array.isArray(state)) return null; const identity = state.identity ?? {}; + if (!identity || typeof identity !== "object" || Array.isArray(identity)) return null; const runId = identity.runId; const normalizedSessionId = identity.normalizedSessionId; const environmentLeaseId = identity.environmentLeaseId; @@ -160,16 +163,21 @@ async function readSessionIdentity(root: string): Promise[0]>[0]; async function sessionRetentionBlocker( tx: Tx, - input: { scopeDigest: string | null; identity: SessionIdentity; closedBefore: Date; checkpoint?: boolean }, + input: { scopeDigest: string | null; identity: SessionIdentity; closedBefore: Date; checkpoint?: boolean; lock?: boolean }, ): Promise<{ reason: string } | { companyId: string; agentId: string; runIds: string[]; issueIds: string[]; closedAt: number }> { const [owner] = await tx .select({ companyId: heartbeatRuns.companyId, agentId: heartbeatRuns.agentId }) @@ -197,7 +205,7 @@ async function sessionRetentionBlocker( if (input.checkpoint && runs.some(run => run.finishedAt! >= input.closedBefore)) return { reason: "recent_run" }; const issueIds = [...new Set(runs.map((run) => run.issueId).filter((id): id is string => Boolean(id)))]; if (issueIds.length === 0) return { reason: "issue_unknown" }; - const issueRows = await tx + const issueQuery = tx .select({ id: issues.id, status: issues.status, @@ -205,8 +213,8 @@ async function sessionRetentionBlocker( closedAt: sql`coalesce(${issues.completedAt}, ${issues.cancelledAt})`, }) .from(issues) - .where(and(eq(issues.companyId, owner.companyId), inArray(issues.id, issueIds))) - .for("share"); + .where(and(eq(issues.companyId, owner.companyId), inArray(issues.id, issueIds))); + const issueRows = await (input.lock === false ? issueQuery : issueQuery.for("share")); if (issueRows.length !== issueIds.length) return { reason: "issue_unknown" }; for (const issue of issueRows) { if (!CLOSED_ISSUE_STATUSES.includes(issue.status)) return { reason: "issue_open" }; @@ -246,7 +254,7 @@ async function sessionRetentionBlocker( .where(and(inArray(environmentLeases.status, LIVE_LEASE_STATUSES), or(...leaseReferences))) .limit(1); if (lease) return { reason: "live_lease" }; - return { ...owner, runIds, issueIds, closedAt: Math.max(...issueRows.map(issue => new Date(issue.closedAt!).getTime())) }; + return { ...owner, runIds, issueIds, closedAt: issueRows.reduce((latest, issue) => Math.max(latest, new Date(issue.closedAt!).getTime()), Number.NEGATIVE_INFINITY) }; } async function hasLiveRun(db: Db, owner: { companyId: string; agentId: string }, sessionId: string) { @@ -379,6 +387,7 @@ export function nativeHarnessBackupExpirationService( const identity = await readSessionIdentity(root); if (!identity) return skip("identity_unknown"); + const bytes = await allocatedBytes(current); const id = randomUUID(); const retiring = path.join(backupRoot, `${RETIRING_PREFIX}${id}`); const trash = path.join(backupRoot, `${EXPIRED_PREFIX}${id}`); @@ -390,7 +399,7 @@ export function nativeHarnessBackupExpirationService( scopeDigest, identity, closedBefore: options.closedBefore, }); if ("reason" in blocker) return blocker; - const bytes = await allocatedBytes(current); + if ((await fs.stat(current)).mtime >= options.closedBefore) return { reason: "recent_backup" }; if (options.dryRun) return { cleared: 0, bytes }; if (options.isSessionBusy(scopeDigest)) return { reason: "session_active" }; await notifyExternalConversationSettlement(tx, { companyId: blocker.companyId }); @@ -739,6 +748,43 @@ export function nativeHarnessBackupExpirationService( }); } + async function backupDependencies(scopeDigest: string, identity: SessionIdentity): Promise { + const [owner] = await db.select({ companyId: heartbeatRuns.companyId, agentId: heartbeatRuns.agentId, + issueId: sql`coalesce(${heartbeatRuns.nativeIssueId}, ${heartbeatRuns.issueId})`, + }).from(heartbeatRuns).where(eq(heartbeatRuns.id, identity.runId)); + if (!owner) return null; + const runs = await db.select({ id: heartbeatRuns.id, + issueId: sql`coalesce(${heartbeatRuns.nativeIssueId}, ${heartbeatRuns.issueId})`, + }).from(heartbeatRuns).where(and(eq(heartbeatRuns.companyId, owner.companyId), + eq(heartbeatRuns.agentId, owner.agentId), eq(heartbeatRuns.nativeSessionId, identity.normalizedSessionId))); + const runIds = [...new Set([identity.runId, ...runs.map(run => run.id)])]; + const issueIds = [...new Set([owner.issueId, ...runs.map(run => run.issueId)].filter((id): id is string => Boolean(id)))]; + const leases = await db.select({ id: environmentLeases.id }).from(environmentLeases).where(and( + eq(environmentLeases.companyId, owner.companyId), or(inArray(environmentLeases.heartbeatRunId, runIds), + sql`${environmentLeases.metadata}->'nativeHarnessBackup'->>'sessionScopeSha256' = ${scopeDigest}`, + identity.environmentLeaseId ? eq(environmentLeases.id, identity.environmentLeaseId) : undefined))); + return { companyId: owner.companyId, agentId: owner.agentId, sessionId: identity.normalizedSessionId, + runIds, issueIds, leaseIds: leases.map(lease => lease.id) }; + } + + async function resolveBackupScope(scope: NativeHarnessBackupExpirationScope) { + let runId = scope.runId; + let issueId = scope.issueId; + let scopeDigest = scope.scopeDigest; + if (scope.leaseId) { + const [lease] = await db.select({ runId: environmentLeases.heartbeatRunId, issueId: environmentLeases.issueId, + scopeDigest: sql`${environmentLeases.metadata}->'nativeHarnessBackup'->>'sessionScopeSha256'`, + }).from(environmentLeases).where(and(eq(environmentLeases.companyId, scope.companyId), eq(environmentLeases.id, scope.leaseId))); + runId = lease?.runId ?? runId; issueId = lease?.issueId ?? issueId; + scopeDigest = lease?.scopeDigest && SCOPE_PATTERN.test(lease.scopeDigest) ? lease.scopeDigest : scopeDigest; + } + const [run] = runId ? await db.select({ agentId: heartbeatRuns.agentId, sessionId: heartbeatRuns.nativeSessionId, + issueId: sql`coalesce(${heartbeatRuns.nativeIssueId}, ${heartbeatRuns.issueId})`, + }).from(heartbeatRuns).where(and(eq(heartbeatRuns.companyId, scope.companyId), eq(heartbeatRuns.id, runId))) : []; + return { runId, issueId: run?.issueId ?? issueId, agentId: run?.agentId ?? scope.agentId, + sessionId: run?.sessionId ?? scope.sessionId, scopeDigest }; + } + async function reconcileBackup(scopeDigest: string) { const base = path.resolve(defaults.stateBase ?? nativeHarnessStateBase()); const root = path.join(base, scopeDigest); @@ -751,19 +797,18 @@ export function nativeHarnessBackupExpirationService( if (!stat?.isDirectory() || stat.isSymbolicLink()) return null; const identity = await readSessionIdentity(root); if (!identity) return null; - const [owner] = await db.select({ companyId: heartbeatRuns.companyId, agentId: heartbeatRuns.agentId }) - .from(heartbeatRuns).where(eq(heartbeatRuns.id, identity.runId)); - if (!owner) return null; - const source = { id: scopeDigest, ...owner }; + const dependencies = await backupDependencies(scopeDigest, identity); + if (!dependencies) return null; + const source = { id: scopeDigest, companyId: dependencies.companyId, agentId: dependencies.agentId, dependencies }; if (await fs.lstat(path.join(root, "cleanup-activation.json")).then(() => true, () => false)) { return { ...source, dueAt: null }; } const blocker = await db.transaction(tx => sessionRetentionBlocker(tx, { - scopeDigest, identity, closedBefore: new Date(8_000_000_000_000_000), + scopeDigest, identity, closedBefore: new Date(8_000_000_000_000_000), lock: false, })); if ("reason" in blocker) return { ...source, dueAt: null }; const retentionMs = Math.max(1, defaults.closedForDays ?? NATIVE_HARNESS_BACKUP_EXPIRATION_DEFAULT_DAYS) * DAY_MS; - return { id: scopeDigest, ...owner, dueAt: Math.max(stat!.mtime.getTime(), blocker.closedAt) + retentionMs + 1 }; + return { ...source, dueAt: Math.max(stat!.mtime.getTime(), blocker.closedAt) + retentionMs + 1 }; } async function backupScopes(after?: string, limit = 100) { @@ -776,6 +821,7 @@ export function nativeHarnessBackupExpirationService( } async function expireBackup(scopeDigest: string) { + if (!SCOPE_PATTERN.test(scopeDigest)) throw new Error("Invalid native harness backup scope"); const result = emptyResult(); const root = path.join(path.resolve(defaults.stateBase ?? nativeHarnessStateBase()), scopeDigest); const dryRun = defaults.dryRun ?? false; @@ -788,5 +834,5 @@ export function nativeHarnessBackupExpirationService( } return { tick, noteEvidence, nextEvidenceAt, tickCheckpoints, registerCheckpointExpiration, - reconcileBackup, backupScopes, expireBackup }; + reconcileBackup, backupScopes, expireBackup, resolveBackupScope }; } diff --git a/server/src/services/native-runtime/native-session-executor.ts b/server/src/services/native-runtime/native-session-executor.ts index 99365030f3d..2424f60d26c 100644 --- a/server/src/services/native-runtime/native-session-executor.ts +++ b/server/src/services/native-runtime/native-session-executor.ts @@ -8092,7 +8092,8 @@ export async function executePaperclipNativeSession(input: { } releaseRetainedNativeSessionCleanup(input.db, { companyId: input.execution.binding.companyId, agentId: input.execution.binding.agentId, - }); + }, { companyId: input.execution.binding.companyId, runId: input.execution.binding.runId, + issueId: input.execution.binding.issueId }); } } @@ -9474,7 +9475,8 @@ async function executePaperclipNativeSessionWithinScope( warmNativeSessions.set(warmSessionId, { onReleased: () => releaseRetainedNativeSessionCleanup(input.db, { companyId: input.execution.binding.companyId, agentId: input.execution.binding.agentId, - }), + }, { companyId: input.execution.binding.companyId, runId: input.execution.binding.runId, + issueId: input.execution.binding.issueId }), pluginTaskId: usesPluginTask && input.runnerExecutionTarget?.kind === "remote" && input.runnerExecutionTarget.transport === "sandbox" ? input.runnerExecutionTarget.sandboxLeaseAcquisition?.providerLeaseId : undefined, runId: input.execution.binding.runId, @@ -11815,7 +11817,8 @@ export async function createRunnerdBackend(input: { initializingSessionToolAuthorities.delete(sessionScopeId); releaseRetainedNativeSessionCleanup(input.db, { companyId: input.execution.binding.companyId, agentId: input.execution.binding.agentId, - }); + }, { companyId: input.execution.binding.companyId, runId: input.execution.binding.runId, + issueId: input.execution.binding.issueId }); } } @@ -12860,7 +12863,7 @@ async function createRunnerdBackendWithinSessionClaim( }) .where(eq(environmentLeases.id, leaseId)) .returning({ id: environmentLeases.id }) - .then((rows) => rows[0] ?? null), { companyId: leaseRow.companyId, runId: leaseRow.heartbeatRunId ?? undefined }); + .then((rows) => rows[0] ?? null), { companyId: leaseRow.companyId, runId: leaseRow.heartbeatRunId ?? undefined, leaseId }); if (!updated) throw new Error("runner_harness_backup_lease_missing"); }; diff --git a/server/src/services/retained-native-session-cleanup-notifications.ts b/server/src/services/retained-native-session-cleanup-notifications.ts index d5aaff7c31a..e29665b3171 100644 --- a/server/src/services/retained-native-session-cleanup-notifications.ts +++ b/server/src/services/retained-native-session-cleanup-notifications.ts @@ -10,15 +10,15 @@ export type RetainedNativeSessionCleanupScope = { }; export async function notifyRetainedNativeSessionCleanup(tx: object, scope: RetainedNativeSessionCleanupScope) { - await notifyNativeHarnessBackupExpiration(tx, { companyId: scope.companyId }); + await notifyNativeHarnessBackupExpiration(tx, scope); await notifySkillVersionRetention(tx, scope.companyId); await notifyDeliveryWork(tx, DELIVERY_QUEUES.retainedNativeSessionCleanup, JSON.stringify({ companyId: scope.companyId, runId: scope.runId, issueId: scope.issueId, agentId: scope.agentId })); } -const listeners = new WeakMap void>>(); -export function onRetainedNativeSessionCleanupRelease(db: object, listener: (scope: RetainedNativeSessionCleanupScope) => void) { +const listeners = new WeakMap void>>(); +export function onRetainedNativeSessionCleanupRelease(db: object, listener: (scope: RetainedNativeSessionCleanupScope, source: RetainedNativeSessionCleanupScope) => void) { let owned = listeners.get(db); if (!owned) { owned = new Set(); listeners.set(db, owned); } owned.add(listener); @@ -27,6 +27,7 @@ export function onRetainedNativeSessionCleanupRelease(db: object, listener: (sco if (!owned.size && listeners.get(db) === owned) listeners.delete(db); }; } -export function releaseRetainedNativeSessionCleanup(db: object, scope: RetainedNativeSessionCleanupScope) { - for (const listener of listeners.get(db) ?? []) listener(scope); +export function releaseRetainedNativeSessionCleanup(db: object, scope: RetainedNativeSessionCleanupScope, + source: RetainedNativeSessionCleanupScope = scope) { + for (const listener of listeners.get(db) ?? []) listener(scope, source); } From d86188c90233be87f670a209dc88804a662f412f Mon Sep 17 00:00:00 2001 From: Dotta Date: Fri, 30 Oct 2026 17:47:09 -0500 Subject: [PATCH 42/74] test: prove timer deadlines survive settled drain holds Expect committed producers to release idle admission while retaining a resume deadline, then verify actual dispatch and final quiet after resume. Co-Authored-By: Paperclip --- server/src/__tests__/agent-timer-deadlines.test.ts | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/server/src/__tests__/agent-timer-deadlines.test.ts b/server/src/__tests__/agent-timer-deadlines.test.ts index 4c4c0d324a9..f925fb8dfd7 100644 --- a/server/src/__tests__/agent-timer-deadlines.test.ts +++ b/server/src/__tests__/agent-timer-deadlines.test.ts @@ -597,8 +597,10 @@ describe.skipIf(!support.supported)("agent timer owned deadlines", () => { await agentService(db).update(f.agent.id, { runtimeConfig: { heartbeat: { enabled: true, intervalSec: 600 } } }); await vi.advanceTimersByTimeAsync(2000); - - expect(idleWorkSnapshot().active).toBe(1); + // The producer committed: retain its resume deadline without keeping the + // instance busy throughout the drain. Uncertain writes still hold admission. + expect(idleWorkSnapshot().active).toBe(0); + expect(coordinator!.nextWakeAt()).toBeGreaterThan(Date.now()); const select = vi.spyOn(db, "select"), execute = vi.spyOn(db, "execute"), transaction = vi.spyOn(db, "transaction"); try { await vi.advanceTimersByTimeAsync(60_000); @@ -608,6 +610,11 @@ describe.skipIf(!support.supported)("agent timer owned deadlines", () => { stopTaskDrain(); await vi.advanceTimersByTimeAsync(1000); await vi.waitFor(() => expect(coordinator!.nextWakeAt()).toBe(now.getTime() + 600_000), { timeout: 3000 }); + await vi.advanceTimersByTimeAsync(coordinator!.nextWakeAt()! - Date.now()); + await vi.waitFor(() => expect(f.enqueueWakeup).toHaveBeenCalledOnce()); + expect(f.enqueueWakeup).toHaveBeenCalledWith(f.agent.id, expect.objectContaining({ source: "timer", reason: "heartbeat_timer" })); + await agentService(db).update(f.agent.id, { runtimeConfig: { heartbeat: { enabled: false } } }); + await quiet(); }); it("does not inspect timer rows during startup drain, then reconstructs them on resume", async () => { From 18b1637b6371a6a6adade857e2a73654ee626fef Mon Sep 17 00:00:00 2001 From: Dotta <34892728+cryppadotta@users.noreply.github.com> Date: Fri, 30 Oct 2026 18:05:05 -0500 Subject: [PATCH 43/74] fix(runner): accept late Antigravity login URL during drain (#18254) ## Thinking Path > - Paperclip manages work performed by AI agents. > - The native Runner owns Antigravity subscription sign-in. > - Sign-in receives completion on stdout and the browser URL on stderr. > - The two pipes can deliver the completion result before the valid URL. > - The owner rejected that URL after it entered the completed phase. > - This pull request accepts one validated URL while the same child drains. > - A real child regression now proves the order and the failure controls. ## Linked Issues or Issue Description Refs #18188, #18189, and #18190. Related qualification work: #18345 and #18356. The duplicate search found this existing PR, so this update preserves its history and adds the real child proof here. **What happened?** A valid sign-in can fail when the authenticate response arrives before the authorization URL on the other pipe. The completed phase then rejects the valid URL. The same existing success case failed in [current follow-up CI](https://github.com/paperclipai/paperclip/actions/runs/41138618869/job/124347859445). That log has no stream-order trace, so it does not prove its inner failure branch. **Expected behavior** Both valid delivery orders must succeed. Invalid or duplicate URLs, unexpected protocol requests, failed retirement, and failed credential verification must still fail. **Steps to reproduce** 1. Run a synthetic native child that completes authenticate on stdout. 2. Wait for stdin EOF before it emits the authorization URL on stderr. EOF proves that the parent accepted the result and began retirement. 3. The valid case fails on frozen master `45a3a92e6dc52d20923b9aa34a22fee2ec5443f9`. The invalid and duplicate controls pass there. 4. Run the same regression with this PR's existing production fix. All three cases pass. **Paperclip version or commit** Existing reviewed PR parent: `646f5cdf37898ad54e1fa1cdfe39f59844da3e7c`. Frozen actual master for the real child RED: `45a3a92e6dc52d20923b9aa34a22fee2ec5443f9`. ## What Changed - Accept one fully validated authorization URL in the authentication or completed-drain phase. Keep duplicate rejection and all existing URL checks. - Retain the ten existing in-memory controls for both valid orders and eight failure paths. - Add three real child cases for valid, invalid, and duplicate late URLs. Use stdin EOF to order the streams without sleeps or a mocked retirement handler. - Check exact protocol requests, child exit, close-once behavior, Google-call counts, and persisted refresh rotation. This update changes only the existing test file; it adds no production delta to the reviewed parent. ## Verification - Frozen actual master: the new valid real child case fails, while both negative controls pass. The original eight-control parent calibration also remains recorded in this PR's history. - Existing fixed source plus the new tests: 24 passed. This includes 11 existing authentication tests, ten stream controls, and three real child cases. - Late invalid and duplicate URLs fail with zero Google calls. The valid case makes two mocked identity calls and saves the rotated synthetic refresh token. All three retire the real child and close the command lease once. - Scoped owning TypeScript `noEmit` passes for the auth module and both test files. The temporary configuration is not committed. Diff and credential-pattern checks pass. - The tests use real child pipes and private synthetic files. Google responses and the command installation are mocked. No provider account, network request, model run, app, or QA deployment was used. - Full monorepo typecheck, tests, and build were not run locally. Fresh [normal CI](https://github.com/paperclipai/paperclip/actions/runs/41140757050) passes on `1744c9f3374f78b10c2f15c42f3cf5d3af8c3119`: 56 successful checks, two skipped, and one neutral. Typecheck, build, Rust, static, Runner, server and browser lanes pass. Greptile gives that exact head a fresh 5/5 with zero unresolved threads. The PR is mergeable and clean. The earlier parent's passing CI remains historical evidence only. ## Risks The change accepts a URL while the same authenticated child drains. It still validates the exact OAuth URL and rejects a second URL. Success still requires the authenticate result, verified credentials, and proven process retirement. No provider pins, contracts, timeout bounds, retries, or public APIs change. Existing documentation already specifies these sign-in semantics; no documentation change is needed. ## Model Used OpenAI Codex, based on GPT-6, with tool use and code execution. The exact runtime model ID and context window were not exposed in this session. ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above - [x] I have either (a) linked existing issues with `Fixes: #` / `Closes #` / `Refs #` OR (b) described the issue in-PR following the relevant issue template - [x] I have not referenced internal/instance-local Paperclip issues or links (only public GitHub `#NNN` / `github.com/paperclipai/paperclip` URLs) - [x] My branch name describes the change (e.g. `docs/...`, `fix/...`) and contains no internal Paperclip ticket id or instance-derived details - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [x] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [x] All Paperclip CI gates are green - [x] Greptile is 5/5 with no open P2s, recommendations, or follow-ups - [x] I will address all Greptile and reviewer comments before requesting merge --------- Co-authored-by: Paperclip --- .../antigravity-auth-stream-order.test.ts | 172 ++++++++++++++++++ .../src/drivers/acpx/antigravity-auth.ts | 2 +- 2 files changed, 173 insertions(+), 1 deletion(-) create mode 100644 packages/paperclip-runner/src/drivers/acpx/antigravity-auth-stream-order.test.ts diff --git a/packages/paperclip-runner/src/drivers/acpx/antigravity-auth-stream-order.test.ts b/packages/paperclip-runner/src/drivers/acpx/antigravity-auth-stream-order.test.ts new file mode 100644 index 00000000000..e44bb0ade13 --- /dev/null +++ b/packages/paperclip-runner/src/drivers/acpx/antigravity-auth-stream-order.test.ts @@ -0,0 +1,172 @@ +import { spawn, type ChildProcess } from "node:child_process"; +import { EventEmitter } from "node:events"; +import { mkdir, mkdtemp, readFile, realpath, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { PassThrough, Writable } from "node:stream"; +import { afterEach, describe, expect, it, vi } from "vitest"; + +const setup = vi.hoisted(() => ({ verify: vi.fn(), spawn: vi.fn(), close: vi.fn(), retire: vi.fn() })); +vi.mock("./antigravity-installation.js", () => ({ antigravityNativeDistributionSpec: () => ({ pinned: true }) })); +vi.mock("./installation-integrity.js", () => ({ verifyNativeAcpxInstallation: setup.verify })); +vi.mock("./antigravity-process.js", async importOriginal => ({ + ...await importOriginal(), retireAntigravityProcess: setup.retire, +})); +import { startAntigravityBrowserLogin } from "./antigravity-auth.js"; + +const clientId = "1071006060591-tmhssin2h21lcre235vtolojh4g403ep.apps.googleusercontent.com"; +const scopes = ["https://www.googleapis.com/auth/cloud-platform", "https://www.googleapis.com/auth/userinfo.email", "https://www.googleapis.com/auth/aicode"]; +const authorizationUrl = "https://accounts.google.com/o/oauth2/v2/auth?" + new URLSearchParams({ client_id: clientId, response_type: "code", + redirect_uri: "http://127.0.0.1:18973/", scope: scopes.join(" "), state: "fixture_state_0123456789", code_challenge: "a".repeat(43), + code_challenge_method: "S256", access_type: "offline", prompt: "consent" }); +const roots: string[] = []; +const physicalChildren: ChildProcess[] = []; +afterEach(async () => { + for (const child of physicalChildren.splice(0)) { + if (child.exitCode === null && child.signalCode === null) { + try { process.kill(-child.pid!, "SIGKILL"); } catch {} + } + } + vi.unstubAllGlobals(); vi.clearAllMocks(); + await Promise.all(roots.splice(0).map(root => rm(root, { recursive: true, force: true }))); +}); + +type Mode = "url-first" | "result-first" | "wrong-url" | "duplicate-url" | "unexpected-prompt" | "retirement-failure" | "credential-failure" | "auth-error" | "late-wrong-url" | "late-duplicate-url"; +async function fixture(mode: Mode) { + const home = await realpath(await mkdtemp(join(tmpdir(), "paperclip-antigravity-stream-test-"))); roots.push(home); + await mkdir(join(home, "antigravity-acp"), { mode: 0o700 }); + await writeFile(join(home, "antigravity-acp/acp_token.json"), JSON.stringify({ client_id: clientId, client_secret: "synthetic-client-secret", + refresh_token: "synthetic-refresh-token", token_uri: "https://oauth2.googleapis.com/token", scopes }), { mode: 0o600 }); + const stdout = new PassThrough(), stderr = new PassThrough(), requests: string[] = []; + const child = Object.assign(new EventEmitter(), { stdout, stderr, stdin: new Writable({ write(chunk, _encoding, done) { + const request = JSON.parse(String(chunk)); requests.push(request.method); + queueMicrotask(() => { + if (request.method === "initialize") { + stdout.write(JSON.stringify({ jsonrpc: "2.0", id: request.id, result: { protocolVersion: 1, + agentInfo: { name: "antigravity-acp", version: "1.4.3" }, agentCapabilities: { _meta: { antigravity: { paperclipAdmission: 1 } } }, + authMethods: [{ id: "oauth-personal" }] } }) + "\n"); + } else if (request.method === "authenticate") { + const url = mode === "wrong-url" || mode === "late-wrong-url" ? "https://example.invalid/synthetic-private-marker" : authorizationUrl; + const emitUrl = () => stderr.write("Open the following link to authenticate the ACP server: " + url + "\n"); + const resultFirst = ["result-first", "late-wrong-url", "late-duplicate-url"].includes(mode); + if (!resultFirst) emitUrl(); + if (mode === "duplicate-url") emitUrl(); + if (mode === "unexpected-prompt") stdout.write(JSON.stringify({ jsonrpc: "2.0", id: 9, method: "session/prompt", params: {} }) + "\n"); + stdout.write(JSON.stringify(mode === "auth-error" + ? { jsonrpc: "2.0", id: request.id, error: { code: -32000, message: "synthetic-private-marker" } } + : { jsonrpc: "2.0", id: request.id, result: {} }) + "\n"); + if (resultFirst) queueMicrotask(() => { emitUrl(); if (mode === "late-duplicate-url") emitUrl(); }); + } else throw new Error("Unexpected session/model request"); + }); + done(); + } }) }); + setup.close.mockResolvedValue(undefined); + setup.retire.mockImplementation(async (_child, closed: Promise) => { + await Promise.resolve(); child.emit("close", 0); await closed; + }); + if (mode === "retirement-failure") setup.retire.mockImplementationOnce(async () => { throw new Error("synthetic-private-marker"); }); + setup.spawn.mockReturnValue(child); + setup.verify.mockResolvedValue({ openCommand: async () => ({ spawn: setup.spawn, close: setup.close }) }); + const fetch = vi.fn(async (url: string) => { + if (mode === "credential-failure") return new Response(null, { status: 401 }); + if (url === "https://oauth2.googleapis.com/token") return Response.json({ access_token: "synthetic-access-token", token_type: "Bearer" }); + if (url === "https://www.googleapis.com/oauth2/v3/userinfo") return Response.json({ email: "person@example.test", email_verified: true }); + throw new Error("Unexpected network request"); + }); + vi.stubGlobal("fetch", fetch); + return { home, requests, fetch }; +} + +describe("Antigravity login cross-stream completion", () => { + it.each(["url-first", "result-first"] as const)("accepts the same authenticated exchange with %s delivery", async mode => { + const { home, requests, fetch } = await fixture(mode); + const login = startAntigravityBrowserLogin(home); + await vi.waitFor(() => expect(login.outcome).toBeDefined()); + expect(login.outcome).toBe("success"); + expect(login.authorizationUrl).toBe(authorizationUrl); + expect(requests).toEqual(["initialize", "authenticate"]); + expect(setup.retire).toHaveBeenCalledOnce(); expect(setup.close).toHaveBeenCalledOnce(); + expect(fetch).toHaveBeenCalledTimes(2); + await login.abort(); + }); + + it.each(["wrong-url", "duplicate-url", "unexpected-prompt", "retirement-failure", "credential-failure", "auth-error", "late-wrong-url", "late-duplicate-url"] as const)("keeps %s failures closed and private", async mode => { + const { home, requests, fetch } = await fixture(mode); + const login = startAntigravityBrowserLogin(home); + await vi.waitFor(() => expect(login.outcome).toBeDefined()); + expect(login.outcome).toBe("failure"); expect(login.error).toBe("Google sign-in did not complete. Start sign-in again."); + expect(login.error).not.toContain("synthetic-private-marker"); + await login.abort(); expect(setup.close).toHaveBeenCalledOnce(); + expect(requests).toEqual(["initialize", "authenticate"]); + expect(fetch).toHaveBeenCalledTimes(mode === "credential-failure" ? 1 : 0); + if (mode === "wrong-url" || mode === "late-wrong-url") expect(login.authorizationUrl).toBeUndefined(); + }); + + it.each(["valid", "invalid", "duplicate"] as const)("handles a %s URL from a real child after authenticate completion and stdin EOF", async mode => { + const home = await realpath(await mkdtemp(join(tmpdir(), "paperclip-antigravity-real-stream-"))); roots.push(home); + const peer = join(home, "peer.cjs"); + await writeFile(peer, ` +const fs = require('node:fs'); +const path = require('node:path'); +const readline = require('node:readline'); +const home = process.argv[2], mode = process.argv[3], requests = [], events = []; +const reader = readline.createInterface({input:process.stdin}); +reader.on('line', line => { + const request = JSON.parse(line); requests.push(request.method); + fs.writeFileSync(path.join(home,'requests.json'),JSON.stringify(requests)); + if(request.method==='initialize') { + process.stdout.write(JSON.stringify({jsonrpc:'2.0',id:request.id,result:{protocolVersion:1, + agentInfo:{name:'antigravity-acp',version:'1.4.3'},agentCapabilities:{_meta:{antigravity:{paperclipAdmission:1}}}, + authMethods:[{id:'oauth-personal'}]}})+'\\n'); + } else if(request.method==='authenticate') { + fs.mkdirSync(path.join(home,'antigravity-acp'),{mode:0o700}); + fs.writeFileSync(path.join(home,'antigravity-acp/acp_token.json'),${JSON.stringify(JSON.stringify({ client_id: clientId, + client_secret: "synthetic-client-secret", refresh_token: "synthetic-refresh-token", token_uri: "https://oauth2.googleapis.com/token", scopes }))},{mode:0o600}); + events.push('authenticate-result'); + process.stdout.write(JSON.stringify({jsonrpc:'2.0',id:request.id,result:{}})+'\\n'); + } else throw new Error('Unexpected session/model request'); +}); +reader.on('close',()=>{ + events.push('stdin-eof'); + const url = mode==='invalid' ? 'https://example.invalid/synthetic-private-marker' : ${JSON.stringify(authorizationUrl)}; + const message = 'Open the following link to authenticate the ACP server: '+url+'\\n'; + process.stderr.write(mode==='duplicate' ? message+message : message,()=>{ + events.push('url-flushed');fs.writeFileSync(path.join(home,'events.json'),JSON.stringify(events));process.exit(0); + }); +}); +`); + setup.spawn.mockImplementation((_args, options) => { + const child = spawn(process.execPath, [peer, home, mode], { ...options, detached: true }); + physicalChildren.push(child); return child; + }); + setup.close.mockResolvedValue(undefined); + setup.verify.mockResolvedValue({ openCommand: async () => ({ spawn: setup.spawn, close: setup.close }) }); + const realProcess = await vi.importActual("./antigravity-process.js"); + setup.retire.mockImplementation(realProcess.retireAntigravityProcess); + const fetch = vi.fn(async (url: string) => { + if (url === "https://oauth2.googleapis.com/token") return Response.json({ access_token: "synthetic-access-token", + refresh_token: "synthetic-rotated-refresh", token_type: "Bearer" }); + if (url === "https://www.googleapis.com/oauth2/v3/userinfo") return Response.json({ email: "person@example.test", email_verified: true }); + throw new Error("Unexpected network request"); + }); + vi.stubGlobal("fetch", fetch); + const login = startAntigravityBrowserLogin(home); + await vi.waitFor(() => expect(login.outcome).toBeDefined(), { timeout: 5_000 }); + expect(login.outcome).toBe(mode === "valid" ? "success" : "failure"); + expect(JSON.parse(await readFile(join(home, "events.json"), "utf8"))).toEqual(["authenticate-result", "stdin-eof", "url-flushed"]); + expect(JSON.parse(await readFile(join(home, "requests.json"), "utf8"))).toEqual(["initialize", "authenticate"]); + expect(physicalChildren.at(-1)!.exitCode).toBe(0); + expect(setup.retire).toHaveBeenCalledOnce(); expect(setup.close).toHaveBeenCalledOnce(); + expect(fetch).toHaveBeenCalledTimes(mode === "valid" ? 2 : 0); + if (mode === "valid") { + expect(login.authorizationUrl).toBe(authorizationUrl); + expect(JSON.parse(await readFile(join(home, "antigravity-acp", "acp_token.json"), "utf8")).refresh_token).toBe("synthetic-rotated-refresh"); + } else { + expect(login.error).toBe("Google sign-in did not complete. Start sign-in again."); + expect(login.error).not.toContain("synthetic-private-marker"); + if (mode === "invalid") expect(login.authorizationUrl).toBeUndefined(); + } + await login.abort(); + expect(setup.close).toHaveBeenCalledOnce(); + }, 15_000); +}); diff --git a/packages/paperclip-runner/src/drivers/acpx/antigravity-auth.ts b/packages/paperclip-runner/src/drivers/acpx/antigravity-auth.ts index 2921fd141db..7f4b3bf54bd 100644 --- a/packages/paperclip-runner/src/drivers/acpx/antigravity-auth.ts +++ b/packages/paperclip-runner/src/drivers/acpx/antigravity-auth.ts @@ -203,7 +203,7 @@ export function startAntigravityBrowserLogin(home: string): AntigravityBrowserLo if (!line.startsWith(AUTH_PREFIX)) continue; try { const url = validateAntigravityAuthorizationUrl(line.slice(AUTH_PREFIX.length)); - if (phase !== 2 || state.authorizationUrl !== undefined) throw new Error(LOGIN_ERROR); + if ((phase !== 2 && phase !== 3) || state.authorizationUrl !== undefined) throw new Error(LOGIN_ERROR); state.authorizationUrl = url; } catch { fail(); } } From 4e97daafa06a54f02027335437c4f48e50b84236 Mon Sep 17 00:00:00 2001 From: Dotta Date: Fri, 30 Oct 2026 18:09:41 -0500 Subject: [PATCH 44/74] refactor: remove obsolete external object batch refresh entrypoints Exercise lease, selection, and recovery behavior through the registered scheduler owner. Co-Authored-By: Paperclip --- .../2026-10-30-external-objects-sleep.md | 8 ++- .../external-object-refresh-work.test.ts | 8 --- .../external-objects-service.test.ts | 66 +++++++++++-------- .../server-startup-feedback-export.test.ts | 4 -- server/src/services/external-objects.ts | 52 --------------- 5 files changed, 43 insertions(+), 95 deletions(-) diff --git a/doc/plans/2026-10-30-external-objects-sleep.md b/doc/plans/2026-10-30-external-objects-sleep.md index 61c7d8dded1..41bf59440d1 100644 --- a/doc/plans/2026-10-30-external-objects-sleep.md +++ b/doc/plans/2026-10-30-external-objects-sleep.md @@ -106,9 +106,11 @@ result; it releases only its own claim and cannot publish that late result. Startup does not await provider work. Shutdown closes the active GitHub HTTP request, and its saved due row can resume immediately after restart. -The two explicit `refreshDueObjects*` helpers remain manual diagnostic/test entry -points. They have no production scheduler caller. Their regression fixtures also -preserve the prior selection/lease semantics. The scheduled check retains its +The old `refreshDueObjects*` batch helpers are removed. Existing selection, +lease, concurrent manual refresh, terminal-history and not-found recovery tests +now enter through the registered scheduler owner. The removed baseline control +recorded six SQL reads over three empty aggregate ticks; the current quiet-work +acceptance measures zero recurring SQL after settlement. The scheduled check retains its post-refresh scoped read to revalidate current company, reference and resolver eligibility and recover an authoritative lease after a competing writer; all such reads target one object rather than rebuilding a company. diff --git a/server/src/__tests__/external-object-refresh-work.test.ts b/server/src/__tests__/external-object-refresh-work.test.ts index 754725c7d80..4c1817b9722 100644 --- a/server/src/__tests__/external-object-refresh-work.test.ts +++ b/server/src/__tests__/external-object-refresh-work.test.ts @@ -100,14 +100,6 @@ const active: ExternalObjectResolveResult = { ok: true, snapshot: { }); } - it("captures the old configured-empty aggregate SQL cost", async () => { - const svc = service(); await fixture(); - const selects = vi.spyOn(db, "select"); - for (let i = 0; i < 3; i++) await svc.refreshDueObjectsForActiveCompanies(); - expect(selects).toHaveBeenCalledTimes(6); - expect(provider).not.toHaveBeenCalled(); - }); - it("activates saved work when the ordinary feature and company settings enable it", async () => { clock(); const settings = instanceSettingsService(db); await settings.updateExperimental({ enableExternalObjects: true }); diff --git a/server/src/__tests__/external-objects-service.test.ts b/server/src/__tests__/external-objects-service.test.ts index bc268d0ba6a..e6de9780444 100644 --- a/server/src/__tests__/external-objects-service.test.ts +++ b/server/src/__tests__/external-objects-service.test.ts @@ -1,4 +1,5 @@ import { randomUUID } from "node:crypto"; +import { createDeliveryWorkCoordinator } from "../services/delivery-work-coordinator.js"; import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest"; import { eq } from "drizzle-orm"; import { @@ -733,6 +734,18 @@ describeEmbeddedPostgres("externalObjectService", () => { await tempDb?.cleanup(); }); + async function runRefreshWork(service: ReturnType) { + const errors: unknown[] = []; + const coordinator = createDeliveryWorkCoordinator({ owner: db, canRun: () => true, + onError: error => errors.push(error) }); + try { + await service.registerRefreshWork(coordinator, () => true).ready; + expect(errors).toEqual([]); + } finally { + await coordinator.stop(); + } + } + async function createIssue(companyId = randomUUID()) { const issueId = randomUUID(); await db.insert(companies).values({ @@ -882,9 +895,9 @@ describeEmbeddedPostgres("externalObjectService", () => { expect(object.nextRefreshAt).toBeInstanceOf(Date); - const refreshed = await svc.refreshDueObjects(companyId, 50, new Date(Date.now() + 1_000)); + await runRefreshWork(svc); - expect(refreshed).toHaveLength(1); + expect((await db.select().from(externalObjects))[0]?.lastResolvedAt).toBeInstanceOf(Date); expect(resolve).toHaveBeenCalledTimes(1); }); @@ -921,15 +934,13 @@ describeEmbeddedPostgres("externalObjectService", () => { await svc.syncIssue(issueId); const object = await db.select().from(externalObjects).then((rows) => rows[0]!); - const dueRefresh = svc.refreshDueObjects(companyId, 50, new Date(Date.now() + 1_000)); + const dueRefresh = runRefreshWork(svc); await resolverStarted; const manualRefresh = svc.refreshObject(object.id, { companyId, force: true }); finishResolver(); - const [dueResults, manualResult] = await Promise.all([dueRefresh, manualRefresh]); + const [, manualResult] = await Promise.all([dueRefresh, manualRefresh]); - expect(dueResults).toHaveLength(1); - expect(dueResults[0]?.refreshed).toBe(true); expect(manualResult.refreshed).toBe(true); expect(manualResult.object.statusLabel).toBe("Open"); expect(resolve).toHaveBeenCalledTimes(1); @@ -969,14 +980,12 @@ describeEmbeddedPostgres("externalObjectService", () => { await scheduledService.syncIssue(issueId); const object = await db.select().from(externalObjects).then((rows) => rows[0]!); - const dueRefresh = scheduledService.refreshDueObjects(companyId, 50, new Date(Date.now() + 1_000)); + const dueRefresh = runRefreshWork(scheduledService); await resolverStarted; const manualRefresh = await manualService.refreshObject(object.id, { companyId, force: true }); finishResolver(); - const dueResults = await dueRefresh; + await dueRefresh; - expect(dueResults).toHaveLength(1); - expect(dueResults[0]?.refreshed).toBe(true); expect(manualRefresh.refreshed).toBe(false); expect(manualRefresh.reason).toBe("refresh_in_progress"); expect(manualRefresh.object).not.toHaveProperty("refreshToken"); @@ -1032,7 +1041,7 @@ describeEmbeddedPostgres("externalObjectService", () => { const object = await db.select().from(externalObjects).then((rows) => rows[0]!); const claimedAt = new Date(Date.now() + 1_000); - const dueRefresh = scheduledService.refreshDueObjects(companyId, 50, claimedAt); + const dueRefresh = runRefreshWork(scheduledService); await resolverStarted; const replacementResult = await replacementService.refreshObject(object.id, { companyId, @@ -1040,11 +1049,10 @@ describeEmbeddedPostgres("externalObjectService", () => { now: new Date(claimedAt.getTime() + 301_000), }); finishResolver(); - const dueResults = await dueRefresh; + await dueRefresh; const finalObject = await db.select().from(externalObjects).then((rows) => rows[0]!); expect(replacementResult.refreshed).toBe(true); - expect(dueResults[0]).toMatchObject({ refreshed: false, reason: "refresh_superseded" }); expect(finalObject.statusLabel).toBe("Closed"); expect(finalObject.refreshStartedAt).toBeNull(); expect(finalObject.refreshToken).toBeNull(); @@ -1078,9 +1086,7 @@ describeEmbeddedPostgres("externalObjectService", () => { await svc.syncIssue(active.issueId); await svc.syncIssue(paused.issueId); - const result = await svc.refreshDueObjectsForActiveCompanies(50, new Date(Date.now() + 1_000)); - - expect(result).toEqual({ companies: 1, checked: 1, refreshed: 1 }); + await runRefreshWork(svc); expect(resolve).toHaveBeenCalledTimes(1); const rows = await db.select().from(externalObjects); const activeObject = rows.find((row) => row.companyId === active.companyId); @@ -1139,9 +1145,8 @@ describeEmbeddedPostgres("externalObjectService", () => { .set({ nextRefreshAt: new Date(0) }) .where(eq(externalObjects.id, object.id)); - const refreshed = await svc.refreshDueObjects(companyId); + await runRefreshWork(svc); - expect(refreshed).toEqual([]); expect(resolve).toHaveBeenCalledTimes(1); }); @@ -1184,7 +1189,7 @@ describeEmbeddedPostgres("externalObjectService", () => { }); it("re-checks previously frozen GitHub not-found objects so they can self-heal", async () => { - const { companyId } = await createIssue(); + const { companyId, issueId } = await createIssue(); const canonical = canonicalizeExternalObjectUrl("https://github.com/acme/app/pull/42"); if (!canonical) throw new Error("expected canonical url"); const inserted = await db.insert(externalObjects).values({ @@ -1225,9 +1230,10 @@ describeEmbeddedPostgres("externalObjectService", () => { const svc = externalObjectService(db, { resolvers: [resolver], github: false }); expect(object.isTerminal).toBe(true); - const refreshed = await svc.refreshDueObjects(companyId); - expect(refreshed).toHaveLength(1); - expect(refreshed[0]?.refreshed).toBe(true); + const [linked] = await db.select().from(externalObjects).where(eq(externalObjects.companyId, companyId)); + await db.insert(externalObjectMentions).values({ companyId, sourceIssueId: issueId, + sourceKind: "description", objectId: linked!.id }); + await runRefreshWork(svc); const updated = await db.select().from(externalObjects).then((rows) => rows[0]!); expect(updated.statusLabel).toBe("Open"); @@ -1236,7 +1242,7 @@ describeEmbeddedPostgres("externalObjectService", () => { }); it("does not re-check probe-verified GitHub not-found objects (deleted references are terminal)", async () => { - const { companyId } = await createIssue(); + const { companyId, issueId } = await createIssue(); const canonical = canonicalizeExternalObjectUrl("https://github.com/acme/app/pull/42"); if (!canonical) throw new Error("expected canonical url"); await db.insert(externalObjects).values({ @@ -1273,14 +1279,16 @@ describeEmbeddedPostgres("externalObjectService", () => { }; const svc = externalObjectService(db, { resolvers: [resolver], github: false }); - const refreshed = await svc.refreshDueObjects(companyId); + const [linked] = await db.select().from(externalObjects).where(eq(externalObjects.companyId, companyId)); + await db.insert(externalObjectMentions).values({ companyId, sourceIssueId: issueId, + sourceKind: "description", objectId: linked!.id }); + await runRefreshWork(svc); - expect(refreshed).toEqual([]); expect(resolver.resolve).not.toHaveBeenCalled(); }); it("does not re-check terminal objects that are not GitHub not-found verdicts", async () => { - const { companyId } = await createIssue(); + const { companyId, issueId } = await createIssue(); const canonical = canonicalizeExternalObjectUrl("https://github.com/acme/app/pull/42"); if (!canonical) throw new Error("expected canonical url"); await db.insert(externalObjects).values({ @@ -1316,9 +1324,11 @@ describeEmbeddedPostgres("externalObjectService", () => { }; const svc = externalObjectService(db, { resolvers: [resolver], github: false }); - const refreshed = await svc.refreshDueObjects(companyId); + const [linked] = await db.select().from(externalObjects).where(eq(externalObjects.companyId, companyId)); + await db.insert(externalObjectMentions).values({ companyId, sourceIssueId: issueId, + sourceKind: "description", objectId: linked!.id }); + await runRefreshWork(svc); - expect(refreshed).toEqual([]); expect(resolver.resolve).not.toHaveBeenCalled(); }); diff --git a/server/src/__tests__/server-startup-feedback-export.test.ts b/server/src/__tests__/server-startup-feedback-export.test.ts index 5002dc44a50..f380ef478dd 100644 --- a/server/src/__tests__/server-startup-feedback-export.test.ts +++ b/server/src/__tests__/server-startup-feedback-export.test.ts @@ -209,7 +209,6 @@ const { const externalObjectsServiceMock = { registerRefreshWork: vi.fn((coordinator: { register: typeof deliveryWorkRegisterMock }, _canRun: () => boolean) => coordinator.register("external-object-refresh", { run: async () => undefined, hasPending: async () => false })), - refreshDueObjectsForActiveCompanies: vi.fn(async () => ({ companies: 0, checked: 0, refreshed: 0 })), }; const externalObjectsServiceFactoryMock = vi.fn(() => externalObjectsServiceMock); const routineServiceMock = { @@ -1404,7 +1403,6 @@ describe("startServer feedback export wiring", () => { expect(externalObjectsServiceMock.registerRefreshWork).toHaveBeenCalledTimes(1); expect(externalObjectsServiceMock.registerRefreshWork).toHaveBeenCalledWith(expect.any(Object), expect.any(Function)); expect(deliveryWorkRegisterMock.mock.calls.some(([queue]) => queue === DELIVERY_QUEUES.externalObjectRefresh)).toBe(true); - expect(externalObjectsServiceMock.refreshDueObjectsForActiveCompanies).not.toHaveBeenCalled(); expect(issueThreadInteractionServiceMock.sweepMergedPullRequestConfirmations).toHaveBeenCalledTimes(1); expect(executionWorkspaceServiceMock.sweepTerminalWorkspaces).toHaveBeenCalledTimes(1); expect(routineServiceMock.tickScheduledTriggers).toHaveBeenCalledTimes(1); @@ -1434,7 +1432,6 @@ describe("startServer feedback export wiring", () => { }); await startServer(); expect(externalObjectsServiceMock.registerRefreshWork).toHaveBeenCalledOnce(); - expect(externalObjectsServiceMock.refreshDueObjectsForActiveCompanies).not.toHaveBeenCalled(); }); it("keeps external object refresh active when heartbeat scheduling is disabled", async () => { @@ -1467,7 +1464,6 @@ describe("startServer feedback export wiring", () => { expect(externalObjectsServiceMock.registerRefreshWork).toHaveBeenCalledTimes(1); expect(externalObjectsServiceMock.registerRefreshWork).toHaveBeenCalledWith(expect.any(Object), expect.any(Function)); expect(deliveryWorkRegisterMock.mock.calls.some(([queue]) => queue === DELIVERY_QUEUES.externalObjectRefresh)).toBe(true); - expect(externalObjectsServiceMock.refreshDueObjectsForActiveCompanies).not.toHaveBeenCalled(); expect(routineServiceMock.tickScheduledTriggers).not.toHaveBeenCalled(); expect(heartbeatServiceMock.reconcileCostAccounting).toHaveBeenCalledTimes(1); expect(environmentCustomImagesServiceMock.cleanupExpiredSetupSessions).not.toHaveBeenCalled(); diff --git a/server/src/services/external-objects.ts b/server/src/services/external-objects.ts index 0208cdab7b6..a4d26014d26 100644 --- a/server/src/services/external-objects.ts +++ b/server/src/services/external-objects.ts @@ -1109,56 +1109,6 @@ export function externalObjectService( return results; } - async function refreshDueObjectsUnchecked(companyId: string, limit = 50, now = new Date()) { - const staleRefreshStartedBefore = new Date(now.getTime() - DEFAULT_REFRESH_LEASE_SECONDS * 1000); - const refreshable = refreshableObjectWhere(); - const due = await db - .select({ id: externalObjects.id }) - .from(externalObjects) - .where( - and( - eq(externalObjects.companyId, companyId), - refreshable, - lte(externalObjects.nextRefreshAt, now), - or( - isNull(externalObjects.refreshStartedAt), - lte(externalObjects.refreshStartedAt, staleRefreshStartedBefore), - ), - ), - ) - .limit(limit); - const results = []; - for (const row of due) { - results.push(await refreshObject(row.id, { - companyId, - actor: { actorType: "system", actorId: "external-object-resolver", agentId: null, runId: null }, - now, - })); - } - return results; - } - - async function refreshDueObjects(companyId: string, limit = 50, now = new Date()) { - if (!(await isEnabled())) return []; - return refreshDueObjectsUnchecked(companyId, limit, now); - } - - async function refreshDueObjectsForActiveCompanies(limitPerCompany = 50, now = new Date()) { - if (!(await isEnabled())) return { companies: 0, checked: 0, refreshed: 0 }; - const activeCompanies = await db - .select({ id: companies.id }) - .from(companies) - .where(eq(companies.status, "active")); - let checked = 0; - let refreshed = 0; - for (const company of activeCompanies) { - const results = await refreshDueObjectsUnchecked(company.id, limitPerCompany, now); - checked += results.length; - refreshed += results.filter((result) => result.refreshed).length; - } - return { companies: activeCompanies.length, checked, refreshed }; - } - function registerRefreshWork( coordinator: ReturnType, canRun: () => boolean, @@ -1234,8 +1184,6 @@ export function externalObjectService( getProjectSummary, refreshObject, refreshIssueObjects, - refreshDueObjects, - refreshDueObjectsForActiveCompanies, registerRefreshWork, }; } From 4b8d9a64ca3873100e0814ea7d89e3e8a11472cf Mon Sep 17 00:00:00 2001 From: Dotta Date: Fri, 30 Oct 2026 18:16:05 -0500 Subject: [PATCH 45/74] fix(server): notify routine status after subtree restore Co-Authored-By: Paperclip --- doc/plans/2026-10-30-routine-status-sleep.md | 4 +++- .../issue-tree-control-service.test.ts | 2 +- .../routine-execution-status-work.test.ts | 22 +++++++++++++++++++ server/src/services/issue-tree-control.ts | 2 ++ 4 files changed, 28 insertions(+), 2 deletions(-) diff --git a/doc/plans/2026-10-30-routine-status-sleep.md b/doc/plans/2026-10-30-routine-status-sleep.md index e852bf94861..0afe809bb8a 100644 --- a/doc/plans/2026-10-30-routine-status-sleep.md +++ b/doc/plans/2026-10-30-routine-status-sleep.md @@ -9,7 +9,9 @@ scanned every five minutes, even with no routines or only completed history. `routine-run-status.ts` owns the projection, custody checks and repair transaction. The worker loads actual mismatches in keyset pages of 50 at startup. It accepts issue hints from issue creation, status/origin changes, checkout/release, routine issue association, -execution control writes and review state writes. It uses the existing +execution control writes, review state writes and subtree status restoration. +Bulk restoration registers each issue hint in the same transaction before its +grouped write. Restored history does not depend on an optional agent wake. It uses the existing transaction-aware work notification protocol. An outer transaction owns the hint; a savepoint release does not wake the repair worker before commit. diff --git a/server/src/__tests__/issue-tree-control-service.test.ts b/server/src/__tests__/issue-tree-control-service.test.ts index 923f399d3bd..e98efb18309 100644 --- a/server/src/__tests__/issue-tree-control-service.test.ts +++ b/server/src/__tests__/issue-tree-control-service.test.ts @@ -41,7 +41,7 @@ describeEmbeddedPostgres("issueTreeControlService", () => { beforeAll(async () => { if (externalUrl) { await applyPendingMigrations(externalUrl); - db = createDb(externalUrl); + db = createDb(externalUrl, { maxConnections: 6 }); return; } tempDb = await startEmbeddedPostgresTestDatabase("paperclip-issue-tree-control-"); diff --git a/server/src/__tests__/routine-execution-status-work.test.ts b/server/src/__tests__/routine-execution-status-work.test.ts index e6bd818852c..0a6ff9339cb 100644 --- a/server/src/__tests__/routine-execution-status-work.test.ts +++ b/server/src/__tests__/routine-execution-status-work.test.ts @@ -1,3 +1,4 @@ +import { issueTreeControlService } from "../services/issue-tree-control.js"; import { writeExecutionControl } from "../services/execution-control-notifications.js"; import { DELIVERY_QUEUES } from "../services/delivery-work-notifications.js"; import { randomUUID } from "node:crypto"; @@ -174,6 +175,27 @@ const support = external ? { supported: true } : await getEmbeddedPostgresTestSu await db.transaction(async tx => { await notifyRoutineExecutionStatus(tx, otherRun.linkedIssueId!); }); await projected(otherRun.id, "completed"); await quiet(); expect(errors).toEqual([]); }); + it("projects subtree cancel and restore without waking agents, then becomes quiet", async () => { + const f = await fixture(), root = await dispatch(f), child = await dispatch(f); + await issueService(db).update(child.linkedIssueId!, { parentId: root.linkedIssueId }); + const tree = issueTreeControlService(db), actor = { actorType: "system" as const, actorId: "routine-status-test" }; + await start(); await quiet(); + const cancel = await tree.createHold(f.companyId, root.linkedIssueId!, { mode: "cancel", actor }); + expect((await tree.cancelIssueStatusesForHold(f.companyId, root.linkedIssueId!, cancel.hold.id)).updatedIssueIds.sort()) + .toEqual([root.linkedIssueId, child.linkedIssueId].sort()); + await projected(root.id, "failed"); await projected(child.id, "failed"); await quiet(); + const restore = await tree.createHold(f.companyId, root.linkedIssueId!, { mode: "restore", actor }); + const restored = await tree.restoreIssueStatusesForHold(f.companyId, root.linkedIssueId!, restore.hold.id, { actor }); + expect(restored.updatedIssues).toEqual(expect.arrayContaining([ + expect.objectContaining({ id: root.linkedIssueId, status: "todo" }), + expect.objectContaining({ id: child.linkedIssueId, status: "todo" }), + ])); + await projected(root.id, "issue_created"); await projected(child.id, "issue_created"); + expect(await saved(root.id)).toMatchObject({ completedAt: null, failureReason: null }); + expect(await saved(child.id)).toMatchObject({ completedAt: null, failureReason: null }); + expect(await db.select().from(heartbeatRuns)).toEqual([]); + await quiet(); expect(errors).toEqual([]); + }); it("serializes a direct status hook with later task changes before projecting the routine", async () => { const f = await fixture(), run = await dispatch(f); await start(); await quiet(); await db.update(issues).set({ status: "done" }).where(eq(issues.id, run.linkedIssueId!)); diff --git a/server/src/services/issue-tree-control.ts b/server/src/services/issue-tree-control.ts index 0d7633f846a..68ba89f527e 100644 --- a/server/src/services/issue-tree-control.ts +++ b/server/src/services/issue-tree-control.ts @@ -1,3 +1,4 @@ +import { notifyRoutineExecutionStatus } from "./routine-execution-status-notifications.js"; import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js"; import { notifyNativeStatusReassessment } from "./native-status-reassessment-notifications.js"; import { notifyUnsafeWorkspaceExportRecovery } from "./unsafe-workspace-export-notifications.js"; @@ -1006,6 +1007,7 @@ export function issueTreeControlService(db: Db) { for (const [status, issueIdsForStatus] of issueIdsByStatus) { if (issueIdsForStatus.length === 0) continue; for (const issueId of issueIdsForStatus) { + await notifyRoutineExecutionStatus(tx, issueId, companyId); await notifyNativeStatusReassessment(tx, { companyId, issueId }); await notifyTerminalWorkspaceWork(tx, `issue:${issueId}`); await notifyChatResponseAvailability(tx, { companyId, issueId }); From a836109fb0ba5387fead3f7426287e4b3c8ba767 Mon Sep 17 00:00:00 2001 From: Dotta Date: Fri, 30 Oct 2026 18:17:28 -0500 Subject: [PATCH 46/74] Restore complete parent sources in local integration resolution Reconstruct conflicted files from immutable parents and apply only line-bounded additive resolutions. Keep this unpublished integration correction visible in history. Co-Authored-By: Paperclip --- .../modules/wake-queue/adapters/postgres.ts | 1329 ++ server/src/services/budgets.ts | 1157 ++ server/src/services/companies.ts | 561 + .../services/delivery-work-notifications.ts | 132 + .../execution-control-notifications.ts | 31 + server/src/services/heartbeat/queue.ts | 5083 ++++++ server/src/services/heartbeat/retries.ts | 1933 +++ server/src/services/instance-settings.ts | 670 + .../issue-review-state-notifications.ts | 14 + server/src/services/issues.ts | 14119 ++++++++++++++++ 10 files changed, 25029 insertions(+) diff --git a/server/src/modules/wake-queue/adapters/postgres.ts b/server/src/modules/wake-queue/adapters/postgres.ts index 21b8da13658..125b0c86984 100644 --- a/server/src/modules/wake-queue/adapters/postgres.ts +++ b/server/src/modules/wake-queue/adapters/postgres.ts @@ -1 +1,1330 @@ +import { notifyRunDispatch } from "../../../services/run-dispatch-notifications.js"; import { notifyStatusCardIssueChange } from "../../../services/status-card-notifications.js"; +import { notifyExternalConversationSettlement } from "../../../services/external-conversation-settlement-notifications.js"; +import { notifyNativeStatusReassessment } from "../../../services/native-status-reassessment-notifications.js"; +import { notifyUnsafeWorkspaceExportRecovery } from "../../../services/unsafe-workspace-export-notifications.js"; +import { notifyChatInboundWakeNotices } from "../../../services/chat-inbound-wake-notice-notifications.js"; +import { notifyChatRunMilestones } from "../../../services/chat-run-milestone-notifications.js"; +import { DELIVERY_QUEUES, notifyDeliveryWork } from "../../../services/delivery-work-notifications.js"; +import { isAcknowledgedNativeReassignmentStop, isAcknowledgedNativeStop } from "../../../services/acknowledged-native-stop.js"; +import { readRunCancellation } from "../../../services/run-cancellation.js"; +import { isCompletedOnboardingHandoffWake } from "../../../services/chat-completion-delivery.js"; +import { instanceSettingsService } from "../../../services/instance-settings.js"; +import { currentConversationCommentCondition } from "../../../services/agent-conversations.js"; +import { getExecutionBlocker, hasPendingRunCleanup } from "../../../services/execution-blocker.js"; +import { and, asc, eq, inArray, isNull, notInArray, or, sql } from "drizzle-orm"; +import { readReleaseRun, type ReleaseRun } from "./release-run-read.js"; +import type { Db } from "@paperclipai/db"; +import { extractIssueReferenceIdentifiers, isUuidLike } from "@paperclipai/shared"; +import { + activityLog, + agentWakeupRequests, + agents, + chatActions, + heartbeatRuns, + issueComments, + issueRecoveryActions, + issueRelations, + issues, + nativeRunFinalizations, +} from "@paperclipai/db"; +import { hasConversationContinuationPolicy } from "../../../services/conversation-continuation.js"; +import { legacyExecutionNeedsReconciliationWithEvidence } from "../../../services/legacy-execution-recovery.js"; +import { + authorizeFailedChatRunRetryWake, + FailedChatRunRetryAuthorizationError, +} from "../../../services/durable-chat-wakeup.js"; +import { isRetiredExternalChatQuestionSource } from "../../../services/question-response-delivery.js"; +import { HttpError } from "../../../errors.js"; +import { evaluateAgentInvokabilityFromDb } from "../../../services/agent-invokability.js"; +import { issueTreeControlService, isVerifiedIssueTreeControlInteractionWake } from "../../../services/issue-tree-control.js"; +import { isAutomaticRecoverySuppressedByPauseHold } from "../../../services/recovery/pause-hold-guard.js"; +import { classifyContinuationFailure } from "../../../services/recovery/service.js"; +import { issueService } from "../../../services/issues.js"; +import { issueRecoveryActionService } from "../../../services/issue-recovery-actions.js"; +import { readContinuationAttempt } from "../../../services/recovery/run-liveness-continuations.js"; +import { withRecoveryContext } from "../../../services/recovery/status-only-context.js"; +import { parseIssueExecutionState } from "../../../services/issue-execution-policy.js"; +import { + queuedCommentIdsFromWakePayload, + withQueuedCommentIdsInWakePayload, +} from "../../../services/issue-queued-comment-queue.js"; +import { CHAT_CONVERSATION_ID_KEY, extractWakeCommentIds } from "../../run-dispatch/index.js"; +import { hasInteractionContinuationWakeContext } from "../domain/context.js"; +import { decidePreDrain, type PreDrainFacts } from "../domain/policy.js"; +import { + EXECUTION_REVIEW_PARTICIPANT_RECOVERY_RETRY_REASON, + isConfigurationIncompleteFailedRun, + isWorkspaceValidationFailedRun, + parseObject, + readNonEmptyString, +} from "../domain/values.js"; +import { requireTransactionScopeTx, TransactionScope } from "../application/ports.js"; +import type { + DeferredWakeCandidate, + InvokableAgentSnapshot, + IssueLockWriter, + IssueSnapshot, + LockedIssueExecution, + ReleaseTransactionResult, + RunSnapshot, + WakeAdmissionReader, + WakeAdmissionWriter, + WakeQueueHost, + WakeQueueTransaction, +} from "../application/ports.js"; +import type { RunSummary } from "../application/types.js"; + +const DEFERRED_WAKE_STATUS = "deferred_issue_execution"; +const DEFERRED_WAKE_CONTEXT_KEY = "_paperclipWakeContext"; +const EXECUTION_PATH_HEARTBEAT_RUN_STATUSES = ["queued", "running", "scheduled_retry"] as const; + +type HeartbeatRunRow = typeof heartbeatRuns.$inferSelect; +type IssueRow = typeof issues.$inferSelect; + +function normalizeAgentNameKey(value: string | null | undefined): string | null { + if (typeof value !== "string") return null; + const normalized = value.trim().toLowerCase(); + return normalized.length > 0 ? normalized : null; +} + +function toRequestedByActorType(value: string | null): "user" | "agent" | "system" | null { + // The database column is free text; map any value outside the union to + // null instead of widening the type back to string. + return value === "user" || value === "agent" || value === "system" ? value : null; +} + +function toRunSnapshot(row: ReleaseRun): RunSnapshot { + const configurationIncompletePayload = parseObject(parseObject(row.resultJson).configurationIncomplete); + return { + id: row.id, + companyId: row.companyId, + agentId: row.agentId, + status: row.status, + runtimeMode: row.runtimeMode, + conversationContinuation: row.runtimeMode === "legacy" && hasConversationContinuationPolicy(row.resultJson), + errorCode: row.errorCode, + responsibleUserId: row.responsibleUserId, + contextSnapshot: parseObject(row.contextSnapshot), + configurationIncompletePayload: Object.keys(configurationIncompletePayload).length > 0 ? configurationIncompletePayload : null, + operatorStopped: isOperatorStoppedRunRow(row), + }; +} + +const OPERATOR_STOP_ERROR_CODES = new Set(["operator_interrupted", "agent_paused"]); + +/** Mirrors recovery's `isOperatorCancelledRun`, plus agent pause, for the release-time recovery tail. */ +export function isOperatorStoppedRunRow(row: Pick): boolean { + if (row.status !== "cancelled") return false; + const cancellation = readRunCancellation(row.resultJson); + if (cancellation?.source === "queued_message") return false; + if (cancellation?.source === "operator" && cancellation.expected) return true; + if (row.errorCode && OPERATOR_STOP_ERROR_CODES.has(row.errorCode)) return true; + const actorType = parseObject(row.resultJson).cancelledByActorType; + return actorType === "user" || actorType === "board"; +} + +function toIssueSnapshot(row: IssueRow): IssueSnapshot { + return { + conversationAgentId: row.conversationAgentId, + conversationUserId: row.conversationUserId, + conversationState: row.conversationState, + id: row.id, + companyId: row.companyId, + identifier: row.identifier ?? "", + status: row.status, + assigneeAgentId: row.assigneeAgentId, + assigneeUserId: row.assigneeUserId, + hiddenAt: row.hiddenAt, + originKind: row.originKind, + monitorNextCheckAt: row.monitorNextCheckAt, + executionState: (row.executionState as Record | null) ?? null, + responsibleUserId: row.responsibleUserId, + parentId: row.parentId, + originId: row.originId, + originRunId: row.originRunId, + }; +} + +function toRunSummary(row: HeartbeatRunRow): RunSummary { + return { + id: row.id, + companyId: row.companyId, + agentId: row.agentId, + invocationSource: row.invocationSource, + triggerDetail: row.triggerDetail, + wakeupRequestId: row.wakeupRequestId, + }; +} + +function toDeferredWakeCandidate(row: typeof agentWakeupRequests.$inferSelect): DeferredWakeCandidate { + const payload = parseObject(row.payload); + const queuedCommentIds = queuedCommentIdsFromWakePayload(payload); + const deferredContextSeed = parseObject(payload[DEFERRED_WAKE_CONTEXT_KEY]); + const deferredCommentIds = extractWakeCommentIds(deferredContextSeed); + const wakeReason = readNonEmptyString(deferredContextSeed.wakeReason); + const queuedReason = wakeReason ?? readNonEmptyString(row.reason); + const queuedWakeIsCommentOnly = + !queuedReason || + queuedReason === "issue_commented" || + queuedReason === "issue_reopened_via_comment" || + queuedReason === "issue_comment_mentioned"; + const preservesIndependentContinuation = + hasInteractionContinuationWakeContext(deferredContextSeed) || + deferredContextSeed.resumeIntent === true || + !queuedWakeIsCommentOnly; + + return { + id: row.id, + companyId: row.companyId, + agentId: row.agentId, + reason: row.reason, + source: row.source, + triggerDetail: row.triggerDetail, + requestedByActorType: toRequestedByActorType(row.requestedByActorType), + requestedByActorId: row.requestedByActorId, + payload, + queuedCommentIds, + preservesIndependentContinuation, + deferredContextSeed, + deferredCommentIds, + wakeReason, + }; +} + +export type WakeQueuePostgresAdapterDeps = { + resolveResponsibleUserId: WakeQueueHost["resolveResponsibleUserId"]; + getRoutineEnv: WakeQueueHost["getRoutineEnv"]; + resolveSessionBeforeForWakeup: WakeQueueHost["resolveSessionBeforeForWakeup"]; +}; + +function buildHost(_tx: Db, deps: WakeQueuePostgresAdapterDeps): WakeQueueHost { + return { + resolveResponsibleUserId: deps.resolveResponsibleUserId, + getRoutineEnv: deps.getRoutineEnv, + resolveSessionBeforeForWakeup: deps.resolveSessionBeforeForWakeup, + }; +} + +function buildTransaction(tx: Db, deps: WakeQueuePostgresAdapterDeps, db: Db, run: ReleaseRun): WakeQueueTransaction { + const treeControlSvc = issueTreeControlService(tx); + const issuesSvc = issueService(tx); + const interruptQueueId = run.runtimeMode !== "native" && run.status === "cancelled" + ? readNonEmptyString(run.resultJson?.queuedCommentInterruptQueueId) + : null; + + return { + async findInvokableAgent({ companyId, agentId }): Promise { + const agent = await tx + .select() + .from(agents) + .where(and(eq(agents.id, agentId), eq(agents.companyId, companyId))) + .then((rows) => rows[0] ?? null); + if (!agent) return null; + const invokability = await evaluateAgentInvokabilityFromDb(tx, agent); + return { id: agent.id, companyId: agent.companyId, name: agent.name, invokable: invokability.invokable, + paused: !invokability.invokable && invokability.reason === "paused" }; + }, + + async findNextDeferredWake({ companyId, issueId, excludedWakeIds }) { + while (true) { + const row = await tx + .select() + .from(agentWakeupRequests) + .where( + and( + eq(agentWakeupRequests.companyId, companyId), + eq(agentWakeupRequests.status, DEFERRED_WAKE_STATUS), + excludedWakeIds?.length ? notInArray(agentWakeupRequests.id, excludedWakeIds) : undefined, + sql`${agentWakeupRequests.payload} ->> 'issueId' = ${issueId}`, + interruptQueueId ? eq(agentWakeupRequests.id, interruptQueueId) : undefined, + interruptQueueId ? eq(agentWakeupRequests.agentId, run.agentId) : undefined, + ), + ) + .orderBy(asc(agentWakeupRequests.requestedAt)) + .limit(1) + .then((rows) => rows[0] ?? null); + if (!row) return null; + const candidate = toDeferredWakeCandidate(row); + try { + const authorizedFailedChatRetry = await authorizeFailedChatRunRetryWake( + db, + tx, + { + phase: "promotion", + wakeupRequestId: row.id, + companyId, + agentId: row.agentId, + issueId, + contextSnapshot: candidate.deferredContextSeed, + }, + ); + return { ...candidate, authorizedFailedChatRetry }; + } catch (error) { + // Retire only a proven denial. Transient failures roll back this + // transaction instead of discarding otherwise authorized work. + if ( + !(error instanceof FailedChatRunRetryAuthorizationError) && + !( + error instanceof HttpError && + error.status >= 400 && + error.status < 500 + ) + ) { + throw error; + } + const now = new Date(); + await notifyChatInboundWakeNotices(tx, { companyId, issueId }); + await tx + .update(agentWakeupRequests) + .set({ + status: "cancelled", + finishedAt: now, + error: + "The exact failed chat request is no longer authorized. Send a new request in the current connected conversation.", + updatedAt: now, + }) + .where( + and( + eq(agentWakeupRequests.companyId, companyId), + eq(agentWakeupRequests.id, row.id), + eq(agentWakeupRequests.status, DEFERRED_WAKE_STATUS), + ), + ); + } + } + }, + + async getQueuedCommentLiveness({ companyId, issueId, wakeAgentId, finishingRunId, finishingRunAgentId, queuedCommentIds }) { + // Malformed ids (for example an 8-char short form written into a wake + // payload) can never match a comment row, but Postgres rejects the whole + // `inArray` statement on the first invalid uuid and blocks every queued + // wake for the issue. Skip them up front so one bad id cannot take down + // the batch; the liveness filters below already treat them as not live. + const queryableCommentIds = queuedCommentIds + .map((commentId) => commentId.trim()) + .filter((commentId) => isUuidLike(commentId)); + const rows = queryableCommentIds.length === 0 ? [] : await tx + .select({ id: issueComments.id, deletedAt: issueComments.deletedAt, createdByRunId: issueComments.createdByRunId }) + .from(issueComments) + .where(and(eq(issueComments.companyId, companyId), eq(issueComments.issueId, issueId), inArray(issueComments.id, queryableCommentIds), currentConversationCommentCondition())); + const targetsFinishingRunAgent = wakeAgentId === finishingRunAgentId; + const liveNonSelfCommentIds = queryableCommentIds.filter((commentId) => { + const row = rows.find((candidate) => candidate.id === commentId); + return Boolean(row && !row.deletedAt && (!targetsFinishingRunAgent || row.createdByRunId !== finishingRunId)); + }); + const containedSelfAuthoredComment = rows.some( + (row) => targetsFinishingRunAgent && !row.deletedAt && row.createdByRunId === finishingRunId, + ); + return { liveNonSelfCommentIds, containedSelfAuthoredComment }; + }, + + async cancelDeferredWake({ companyId, wakeId, reason, now }) { + await notifyChatInboundWakeNotices(tx, { companyId }); + const rows = await tx + .update(agentWakeupRequests) + .set({ status: "cancelled", finishedAt: now, error: reason, updatedAt: now }) + .where( + and( + eq(agentWakeupRequests.id, wakeId), + eq(agentWakeupRequests.companyId, companyId), + eq(agentWakeupRequests.status, DEFERRED_WAKE_STATUS), + ), + ) + .returning({ id: agentWakeupRequests.id }); + return rows.length > 0; + }, + + async normalizeDeferredWakeCommentIds({ companyId, wakeId, payload, liveCommentIds, now }) { + const rows = await tx + .update(agentWakeupRequests) + .set({ payload: withQueuedCommentIdsInWakePayload(payload, liveCommentIds), updatedAt: now }) + .where( + and( + eq(agentWakeupRequests.id, wakeId), + eq(agentWakeupRequests.companyId, companyId), + eq(agentWakeupRequests.status, DEFERRED_WAKE_STATUS), + ), + ) + .returning(); + const row = rows[0]; + return row ? toDeferredWakeCandidate(row) : null; + }, + + async failDeferredWake({ companyId, wakeId, now }) { + await notifyChatInboundWakeNotices(tx, { companyId }); + const rows = await tx + .update(agentWakeupRequests) + .set({ + status: "failed", + finishedAt: now, + error: "Deferred wake could not be promoted: agent is not invokable", + updatedAt: now, + }) + .where( + and( + eq(agentWakeupRequests.id, wakeId), + eq(agentWakeupRequests.companyId, companyId), + eq(agentWakeupRequests.status, DEFERRED_WAKE_STATUS), + ), + ) + .returning({ id: agentWakeupRequests.id }); + return rows.length > 0; + }, + + async getPauseHoldFacts({ companyId, issueId, wakeAgentId, deferredContextSeed, requestedByActorType, requestedByActorId }) { + const activePauseHold = await treeControlSvc.getActivePauseHoldGate(companyId, issueId); + if (!activePauseHold) { + return { + activePauseHold: false, + treeHoldInteractionWake: false, + holdId: null, + rootIssueId: null, + mode: null, + reason: null, + releasePolicy: null, + }; + } + const treeHoldInteractionWake = await isVerifiedIssueTreeControlInteractionWake(tx, { + companyId, + issueId, + agentId: wakeAgentId, + contextSnapshot: deferredContextSeed, + requestedByActorType, + requestedByActorId, + }); + return { + activePauseHold: true, + treeHoldInteractionWake, + holdId: activePauseHold.holdId, + rootIssueId: activePauseHold.rootIssueId, + mode: activePauseHold.mode, + reason: activePauseHold.reason, + releasePolicy: activePauseHold.releasePolicy, + }; + }, + + async getCommentSelfAuthorship({ companyId, issueId, finishingRunId, commentIds }) { + const rows = await tx + .select({ createdByRunId: issueComments.createdByRunId }) + .from(issueComments) + .where(and(eq(issueComments.companyId, companyId), eq(issueComments.issueId, issueId), inArray(issueComments.id, commentIds))); + return { allSelfAuthored: rows.length > 0 && rows.every((row) => row.createdByRunId === finishingRunId) }; + }, + + async isCompletedDelegationMention({ companyId, issueId, finishingRunId, wakeAgentId, commentIds }) { + if ( + run.companyId !== companyId || run.id !== finishingRunId || + readNonEmptyString(parseObject(run.contextSnapshot).issueId) !== issueId + ) return false; + const uniqueCommentIds = [...new Set(commentIds)]; + if (uniqueCommentIds.length === 0) return false; + const parent = await tx.select({ status: issues.status, assigneeAgentId: issues.assigneeAgentId, identifier: issues.identifier }) + .from(issues).where(and(eq(issues.companyId, companyId), eq(issues.id, issueId))) + .then((rows) => rows[0]); + if (parent?.status !== "done" || parent.assigneeAgentId !== run.agentId) return false; + + const comments = await tx.select({ body: issueComments.body, createdByRunId: issueComments.createdByRunId, + authorAgentId: issueComments.authorAgentId }) + .from(issueComments).where(and( + eq(issueComments.companyId, companyId), eq(issueComments.issueId, issueId), + inArray(issueComments.id, uniqueCommentIds), isNull(issueComments.deletedAt), currentConversationCommentCondition(), + )); + if (comments.length !== uniqueCommentIds.length || comments.some((comment) => + comment.createdByRunId !== run.id || (comment.authorAgentId !== null && comment.authorAgentId !== run.agentId) + )) return false; + const referencesByComment = comments.map((comment) => extractIssueReferenceIdentifiers(comment.body) + .filter((identifier) => identifier !== parent.identifier)); + // A parent link is context; every other reference must identify the + // single completed child. Never discard unrelated follow-up work. + if (referencesByComment.some((references) => references.length !== 1)) return false; + const identifiers = [...new Set(referencesByComment.flat())]; + const children = await tx.select({ identifier: issues.identifier, status: issues.status }) + .from(issues).where(and( + eq(issues.companyId, companyId), eq(issues.parentId, issueId), + eq(issues.assigneeAgentId, wakeAgentId), inArray(issues.identifier, identifiers), + )); + return referencesByComment.every((references) => { + const referencedChildren = children.filter((child) => child.identifier !== null && references.includes(child.identifier)); + return referencedChildren.length === 1 && referencedChildren[0].status === "done"; + }); + }, + + async isCompletedOnboardingHandoffWake(input) { + return isCompletedOnboardingHandoffWake(tx, input); + }, + + async reopenIssue({ companyId, issueId }) { + const updated = await issuesSvc.updateForCompany(issueId, companyId, { status: "todo", executionState: null }, tx); + return updated ? toIssueSnapshot(updated as unknown as IssueRow) : null; + }, + + async claimDeferredWakeForPromotion({ companyId, wakeId, now }) { + const claimed = await tx + .update(agentWakeupRequests) + .set({ + status: "queued", + reason: "issue_execution_promoted", + claimedAt: null, + finishedAt: null, + error: null, + updatedAt: now, + }) + .where( + and( + eq(agentWakeupRequests.id, wakeId), + eq(agentWakeupRequests.companyId, companyId), + eq(agentWakeupRequests.status, DEFERRED_WAKE_STATUS), + ), + ) + .returning({ id: agentWakeupRequests.id }); + return claimed.length > 0; + }, + + async finalizePromotedWake(input) { + await notifyChatRunMilestones(tx, { companyId: input.companyId, issueId: input.issue.id }); + const newRun = await tx + .insert(heartbeatRuns) + .values({ + companyId: input.deferredAgent.companyId, + agentId: input.deferredAgent.id, + invocationSource: input.source, + triggerDetail: input.triggerDetail, + status: "queued", + wakeupRequestId: input.wakeId, + retryOfRunId: input.authorizedFailedChatRetry + ? readNonEmptyString(input.contextSnapshot.retryOfRunId) + : null, + contextSnapshot: input.contextSnapshot, + responsibleUserId: input.responsibleUserId, + sessionIdBefore: input.sessionBefore, + continuationAttempt: readContinuationAttempt(input.contextSnapshot.livenessContinuationAttempt), + }) + .returning() + .then((rows) => rows[0]); + + // `claimDeferredWakeForPromotion` already moved this row off + // `deferred_issue_execution` inside this same transaction, so no + // concurrent claimer can still match that guard; this extra `runId is + // null` guard only protects against writing the link twice. + await tx + .update(agentWakeupRequests) + .set({ runId: newRun.id, updatedAt: input.now }) + .where( + and( + eq(agentWakeupRequests.id, input.wakeId), + eq(agentWakeupRequests.companyId, input.companyId), + isNull(agentWakeupRequests.runId), + ), + ); + + // Promoted mention wakes are issue-scoped, not issue ownership + // transfers. The lock-clearing step earlier in this transaction + // already set `executionRunId` to null for this issue, so the `is + // null` guard only protects against taking the lock twice. + await tx + .update(issues) + .set({ + executionRunId: newRun.id, + executionAgentNameKey: normalizeAgentNameKey(input.deferredAgent.name), + executionLockedAt: input.now, + updatedAt: input.now, + }) + .where( + and( + eq(issues.id, input.issue.id), + eq(issues.companyId, input.companyId), + eq(issues.assigneeAgentId, input.deferredAgent.id), + isNull(issues.executionRunId), + ), + ); + + return toRunSummary(newRun); + }, + + async hasExistingExecutionPath({ companyId, issueId, excludeRunId, agentId }) { + const row = await tx + .select({ id: heartbeatRuns.id }) + .from(heartbeatRuns) + .where( + and( + eq(heartbeatRuns.companyId, companyId), + inArray(heartbeatRuns.status, [...EXECUTION_PATH_HEARTBEAT_RUN_STATUSES]), + sql`${heartbeatRuns.contextSnapshot} ->> 'issueId' = ${issueId}`, + sql`${heartbeatRuns.id} <> ${excludeRunId}`, + agentId ? eq(heartbeatRuns.agentId, agentId) : sql`true`, + ), + ) + .limit(1) + .then((rows) => rows[0] ?? null); + return row !== null; + }, + + async hasExplicitBlockerPath({ companyId, issueId }) { + const row = await tx + .select({ issueId: issueRelations.issueId }) + .from(issueRelations) + .innerJoin(issues, eq(issueRelations.issueId, issues.id)) + .where( + and( + eq(issueRelations.companyId, companyId), + eq(issueRelations.relatedIssueId, issueId), + eq(issueRelations.type, "blocks"), + eq(issues.companyId, companyId), + notInArray(issues.status, ["done", "cancelled"]), + isNull(issues.hiddenAt), + ), + ) + .limit(1) + .then((rows) => rows[0] ?? null); + return row !== null; + }, + + async isAutomaticRecoverySuppressedByPauseHold({ companyId, issueId }) { + return isAutomaticRecoverySuppressedByPauseHold(tx, companyId, issueId, treeControlSvc); + }, + + async isImmediateRecoverySourceBlocked({ companyId, runId }) { + if (companyId !== run.companyId || runId !== run.id) { + throw new Error( + "wake-queue: recovery source does not match the locked execution", + ); + } + const failedChatRequestOwner = run.wakeupRequestId + ? await tx + .select({ id: chatActions.id }) + .from(chatActions) + .where( + and( + eq(chatActions.id, run.wakeupRequestId), + eq(chatActions.companyId, run.companyId), + inArray(chatActions.kind, ["inbound_wakeup", "failed_run_retry"]), + ), + ) + .limit(1) + .then((rows) => rows[0] ?? null) + : null; + return ( + Boolean(failedChatRequestOwner) || + run.errorCode === "chat_failed_run_retry_not_authorized" || + classifyContinuationFailure(run).kind === "non_retryable" + ); + }, + + async queueReviewParticipantRecoveryRun({ companyId, issue, finishingRun, recoveryAgent, sessionBefore, now }) { + await notifyStatusCardIssueChange(tx, companyId); + const executionState = parseIssueExecutionState(issue.executionState); + const wakeupRequest = await tx + .insert(agentWakeupRequests) + .values({ + companyId, + agentId: recoveryAgent.id, + source: "automation", + triggerDetail: "system", + reason: EXECUTION_REVIEW_PARTICIPANT_RECOVERY_RETRY_REASON, + payload: withRecoveryContext( + { + issueId: issue.id, + retryOfRunId: finishingRun.id, + retryReason: EXECUTION_REVIEW_PARTICIPANT_RECOVERY_RETRY_REASON, + currentStageId: executionState?.currentStageId ?? null, + currentStageType: executionState?.currentStageType ?? null, + }, + "normal_model", + ), + status: "queued", + requestedByActorType: "system", + requestedByActorId: null, + updatedAt: now, + }) + .returning() + .then((rows) => rows[0]); + + await notifyRunDispatch(tx, { companyId }); + const queuedRun = await tx + .insert(heartbeatRuns) + .values({ + companyId, + agentId: recoveryAgent.id, + invocationSource: "automation", + triggerDetail: "system", + status: "queued", + wakeupRequestId: wakeupRequest.id, + contextSnapshot: withRecoveryContext( + { + issueId: issue.id, + taskId: issue.id, + wakeReason: EXECUTION_REVIEW_PARTICIPANT_RECOVERY_RETRY_REASON, + retryReason: EXECUTION_REVIEW_PARTICIPANT_RECOVERY_RETRY_REASON, + source: "issue.execution_review_recovery", + retryOfRunId: finishingRun.id, + currentStageId: executionState?.currentStageId ?? null, + currentStageType: executionState?.currentStageType ?? null, + reviewRecoveryInstruction: + "The previous reviewer run ended while this execution-review stage was still pending. Submit the review decision now, or mark the issue blocked with the exact unblock action.", + }, + "normal_model", + ), + sessionIdBefore: sessionBefore, + retryOfRunId: finishingRun.id, + updatedAt: now, + }) + .returning() + .then((rows) => rows[0]); + + await tx + .update(agentWakeupRequests) + .set({ runId: queuedRun.id, updatedAt: now }) + .where(and(eq(agentWakeupRequests.id, wakeupRequest.id), eq(agentWakeupRequests.companyId, companyId))); + + await tx + .update(issues) + .set({ + executionRunId: queuedRun.id, + executionAgentNameKey: normalizeAgentNameKey(recoveryAgent.name), + executionLockedAt: now, + updatedAt: now, + }) + .where(and(eq(issues.id, issue.id), eq(issues.companyId, companyId))); + + return toRunSummary(queuedRun); + }, + + async queueImmediateRecoveryRun({ + companyId, + issue, + finishingRun, + recoveryAgent, + reason, + contextSnapshot, + responsibleUserId, + sessionBefore, + now, + }) { + await notifyChatRunMilestones(tx, { companyId, issueId: issue.id }); + const wakeupRequest = await tx + .insert(agentWakeupRequests) + .values({ + companyId, + agentId: recoveryAgent.id, + source: "automation", + triggerDetail: "system", + reason, + payload: withRecoveryContext({ issueId: issue.id, retryOfRunId: finishingRun.id }, "normal_model"), + status: "queued", + requestedByActorType: "system", + requestedByActorId: null, + updatedAt: now, + }) + .returning() + .then((rows) => rows[0]); + + await notifyRunDispatch(tx, { companyId }); + const queuedRun = await tx + .insert(heartbeatRuns) + .values({ + companyId, + agentId: recoveryAgent.id, + invocationSource: "automation", + triggerDetail: "system", + status: "queued", + wakeupRequestId: wakeupRequest.id, + contextSnapshot, + responsibleUserId, + sessionIdBefore: sessionBefore, + retryOfRunId: finishingRun.id, + updatedAt: now, + }) + .returning() + .then((rows) => rows[0]); + + await tx + .update(agentWakeupRequests) + .set({ runId: queuedRun.id, updatedAt: now }) + .where(and(eq(agentWakeupRequests.id, wakeupRequest.id), eq(agentWakeupRequests.companyId, companyId))); + + await tx + .update(issues) + .set({ + executionRunId: queuedRun.id, + executionAgentNameKey: normalizeAgentNameKey(recoveryAgent.name), + executionLockedAt: now, + updatedAt: now, + }) + .where(and(eq(issues.id, issue.id), eq(issues.companyId, companyId))); + + return toRunSummary(queuedRun); + }, + }; +} + +async function recordNativeTerminalRecoveryIfNeeded(tx: Db, run: ReleaseRun, issue: IssueRow, now: Date): Promise { + const applies = + run.runtimeMode === "native" && + ["failed", "timed_out", "interrupted", "cancelled"].includes(run.status) && + issue.assigneeAgentId === run.agentId && + !["done", "cancelled"].includes(issue.status); + if (!applies || isAcknowledgedNativeStop(run) || isAcknowledgedNativeReassignmentStop(run)) return false; + + const existing = await tx + .select({ id: issueRecoveryActions.id, evidence: issueRecoveryActions.evidence }) + .from(issueRecoveryActions) + .where( + and( + eq(issueRecoveryActions.companyId, issue.companyId), + eq(issueRecoveryActions.sourceIssueId, issue.id), + or( + inArray(issueRecoveryActions.status, ["active", "escalated"]), + sql`${issueRecoveryActions.evidence}->'automaticRecovery'->>'runId' = ${run.id}`, + ), + ), + ) + .limit(1); + let nativeFailureBlock: { runId: string; statusVersion: number } | undefined; + if (issue.status !== "blocked") { + const projected = await issueService(tx).update(issue.id, { status: "blocked" }, tx); + if (projected) { + nativeFailureBlock = { runId: run.id, statusVersion: projected.statusVersion }; + await notifyDeliveryWork(tx, DELIVERY_QUEUES.mcpEvents, `task:${issue.id}`); + await tx.insert(activityLog).values({ + companyId: issue.companyId, actorType: "system", actorId: "execution-recovery", + action: "issue.updated", entityType: "issue", entityId: issue.id, runId: run.id, + details: { status: "blocked", previousStatus: issue.status, reason: "native_continuation_requires_reconciliation" }, + }); + } + } + // Status projection is required even when restart/finalization created the + // incident first. Preserve its owner, cause, retry budget, and prior evidence. + if (nativeFailureBlock) { + for (const action of existing) { + await tx.update(issueRecoveryActions).set({ + evidence: { ...action.evidence, nativeFailureBlock }, updatedAt: now, + }).where(and(eq(issueRecoveryActions.id, action.id), eq(issueRecoveryActions.companyId, issue.companyId))); + } + } + if (!existing.length) { + await tx + .update(nativeRunFinalizations) + .set({ + phase: "terminal_failure", + leaseOwner: null, + leaseExpiresAt: null, + nextAttemptAt: null, + recoveryState: "blocked", + failureCode: "native_continuation_requires_reconciliation", + updatedAt: now, + }) + .where( + and( + eq(nativeRunFinalizations.companyId, issue.companyId), + eq(nativeRunFinalizations.runId, run.id), + isNull(nativeRunFinalizations.resultId), + ), + ); + await issueRecoveryActionService(tx).upsertSourceScoped({ + companyId: issue.companyId, + sourceIssueId: issue.id, + kind: "active_run_watchdog", + ownerType: "board", + returnOwnerAgentId: run.agentId, + cause: "native_continuation_requires_reconciliation", + fingerprint: `native-continuation:${run.id}`, + evidence: { runId: run.id, originalFailureCode: run.errorCode, ...(nativeFailureBlock ? { nativeFailureBlock } : {}) }, + nextAction: + "Inspect the original failure and reconcile the previous execution before continuing. Automatic recovery cannot start another incident.", + maxAttempts: 3, + wakePolicy: null, + supersedeOnIdentityChange: true, + }); + } + return true; +} + +/** + * Builds the temporary transaction-scope handle the admission port needs. + * `heartbeat.ts` calls this through `createWakeQueue`'s own wrapper; it + * never builds a `TransactionScope` itself. + */ +export function createAdmissionTransactionScope(companyId: string, tx: Db): TransactionScope { + return TransactionScope.create(companyId, tx); +} + +function requireAdmissionTx(scope: TransactionScope | null | undefined, companyId: string): Db { + return requireTransactionScopeTx(scope, companyId) as Db; +} + +export function createWakeAdmissionReader(): WakeAdmissionReader { + return { + async matchesActiveWakeActor(scope, input) { + const tx = requireAdmissionTx(scope, input.companyId); + if (!input.wakeupRequestId) return false; + const actor = await tx + .select({ + type: agentWakeupRequests.requestedByActorType, + id: agentWakeupRequests.requestedByActorId, + }) + .from(agentWakeupRequests) + .where( + and( + eq(agentWakeupRequests.companyId, input.companyId), + eq(agentWakeupRequests.id, input.wakeupRequestId), + ), + ) + .limit(1) + .then((rows) => rows[0] ?? null); + return ( + actor !== null && + actor.type === input.requestedByActorType && + actor.id === input.requestedByActorId + ); + }, + async isSameExecutionAgent( + scope, + { + companyId, + activeExecutionRunAgentId, + issueExecutionAgentNameKey, + agentNameKey, + }, + ) { + const tx = requireAdmissionTx(scope, companyId); + const executionAgent = await tx + .select({ name: agents.name }) + .from(agents) + .where( + and( + eq(agents.id, activeExecutionRunAgentId), + eq(agents.companyId, companyId), + ), + ) + .then((rows) => rows[0] ?? null); + const executionAgentNameKey = + normalizeAgentNameKey(issueExecutionAgentNameKey) ?? + normalizeAgentNameKey(executionAgent?.name); + return ( + Boolean(executionAgentNameKey) && executionAgentNameKey === agentNameKey + ); + }, + + async findExistingDeferredWake( + scope, + { companyId, agentId, issueId, chatConversationId, durableActor }, + ) { + const tx = requireAdmissionTx(scope, companyId); + const row = await tx + .select() + .from(agentWakeupRequests) + .where( + and( + eq(agentWakeupRequests.companyId, companyId), + eq(agentWakeupRequests.agentId, agentId), + eq(agentWakeupRequests.status, DEFERRED_WAKE_STATUS), + sql`${agentWakeupRequests.payload} ->> 'issueId' = ${issueId}`, + ...(chatConversationId + ? [ + sql`coalesce(${agentWakeupRequests.payload} -> ${DEFERRED_WAKE_CONTEXT_KEY}::text ->> ${CHAT_CONVERSATION_ID_KEY}::text, ${chatConversationId}) = ${chatConversationId}`, + ] + : []), + ...(durableActor + ? [ + durableActor.type === null + ? isNull(agentWakeupRequests.requestedByActorType) + : eq( + agentWakeupRequests.requestedByActorType, + durableActor.type, + ), + durableActor.id === null + ? isNull(agentWakeupRequests.requestedByActorId) + : eq( + agentWakeupRequests.requestedByActorId, + durableActor.id, + ), + ] + : []), + ), + ) + .orderBy(asc(agentWakeupRequests.requestedAt)) + .limit(1) + .then((rows) => rows[0] ?? null); + if (!row) return null; + const payload = parseObject(row.payload); + return { + id: row.id, + runId: row.runId, + payload, + deferredContext: parseObject(payload[DEFERRED_WAKE_CONTEXT_KEY]), + coalescedCount: row.coalescedCount, + }; + }, + }; +} + +export function createWakeAdmissionWriter(): WakeAdmissionWriter { + return { + async coalesceIntoActiveExecutionRun(scope, input) { + const tx = requireAdmissionTx(scope, input.companyId); + const now = new Date(); + await notifyChatRunMilestones(tx, { companyId: input.companyId, runId: input.activeExecutionRunId }); + const mergedRun = await tx + .update(heartbeatRuns) + .set({ contextSnapshot: input.mergedContextSnapshot, updatedAt: now }) + .where( + and( + eq(heartbeatRuns.id, input.activeExecutionRunId), + eq(heartbeatRuns.companyId, input.companyId), + ), + ) + .returning() + .then((rows) => rows[0] ?? null); + if (!mergedRun) { + // The compare-and-set write affected no row. Throw to roll the + // transaction back instead of recording a coalesced wake against a + // run this write never touched. + throw new Error( + "wake-queue: the coalesce target run was not found for this company", + ); + } + await tx.insert(agentWakeupRequests).values({ + ...input.durableReceipt, + companyId: input.companyId, + agentId: input.agentId, + source: input.source, + triggerDetail: input.triggerDetail, + reason: "issue_execution_same_name", + payload: input.payload, + status: "coalesced", + coalescedCount: 1, + requestedByActorType: input.requestedByActorType, + requestedByActorId: input.requestedByActorId, + idempotencyKey: input.idempotencyKey, + runId: mergedRun.id, + finishedAt: now, + }); + return mergedRun as unknown as Record; + }, + + async mergeIntoExistingDeferredWake(scope, input) { + const tx = requireAdmissionTx(scope, input.companyId); + if (input.manualUserWakeActorId) await notifyChatInboundWakeNotices(tx, { companyId: input.companyId }); + const [prior] = await tx.select().from(agentWakeupRequests).where(and( + eq(agentWakeupRequests.companyId, input.companyId), eq(agentWakeupRequests.id, input.existingDeferredWakeId), + eq(agentWakeupRequests.status, DEFERRED_WAKE_STATUS), + )).for("update"); + if (!prior) throw new Error("wake-queue: the deferred wake to merge into was not found for this company"); + const priorPayload = parseObject(prior.payload), issueId = readNonEmptyString(priorPayload.issueId); + const priorCommentId = readNonEmptyString(priorPayload.commentId), priorActor = readNonEmptyString(prior.requestedByActorId); + const [priorComment] = prior.requestedByActorType === "user" && priorActor && issueId && isUuidLike(issueId) && + priorCommentId && isUuidLike(priorCommentId) ? await tx.select({ id: issueComments.id, body: issueComments.body }).from(issueComments).where(and( + eq(issueComments.companyId, input.companyId), eq(issueComments.issueId, issueId), eq(issueComments.id, priorCommentId), + eq(issueComments.authorType, "user"), eq(issueComments.authorUserId, priorActor), + isNull(issueComments.deletedAt), isNull(issueComments.createdByRunId), + )) : []; + if ((input.coalescedReceipt || priorComment?.body.trim()) && + (readNonEmptyString(input.mergedPayload.issueId) !== issueId || (input.coalescedReceipt && + (input.coalescedReceipt.agentId !== prior.agentId || readNonEmptyString(input.coalescedReceipt.payload.issueId) !== issueId)))) + throw new Error("wake-queue: merged receipt does not match the saved queue scope"); + if (priorComment?.body.trim()) { + const [recorded] = await tx.select({ id: agentWakeupRequests.id }).from(agentWakeupRequests).where(and( + eq(agentWakeupRequests.companyId, input.companyId), eq(agentWakeupRequests.agentId, prior.agentId), + eq(agentWakeupRequests.status, "coalesced"), eq(agentWakeupRequests.requestedByActorType, "user"), + eq(agentWakeupRequests.requestedByActorId, priorActor!), + prior.runId ? eq(agentWakeupRequests.runId, prior.runId) : isNull(agentWakeupRequests.runId), + sql`${agentWakeupRequests.payload}->>'issueId' = ${issueId}`, + sql`${agentWakeupRequests.payload}->>'commentId' = ${priorComment.id}`, + sql`${agentWakeupRequests.payload}->>'coalescedIntoWakeupRequestId' = ${prior.id}`, + )).limit(1); + if (!recorded) await tx.insert(agentWakeupRequests).values({ + companyId: input.companyId, agentId: prior.agentId, source: prior.source, triggerDetail: prior.triggerDetail, + reason: prior.reason, payload: { issueId, commentId: priorComment.id, coalescedIntoWakeupRequestId: prior.id }, + status: "coalesced", coalescedCount: 1, requestedByActorType: "user", requestedByActorId: priorActor, + requestedAt: prior.requestedAt, finishedAt: new Date(), runId: prior.runId, + }); + } + const rows = await tx + .update(agentWakeupRequests) + .set({ + payload: input.mergedPayload, + coalescedCount: input.nextCoalescedCount, + ...(input.manualUserWakeActorId ? { + requestedByActorType: "user", + requestedByActorId: input.manualUserWakeActorId, + } : {}), + updatedAt: new Date(), + }) + .where( + and( + eq(agentWakeupRequests.id, input.existingDeferredWakeId), + eq(agentWakeupRequests.companyId, input.companyId), + eq(agentWakeupRequests.status, DEFERRED_WAKE_STATUS), + ), + ) + .returning({ id: agentWakeupRequests.id }); + if (rows.length === 0) { + // The compare-and-set write affected no row: a concurrent writer + // already moved this wake off `deferred_issue_execution`. Roll the + // transaction back instead of leaving the merge half-applied. + throw new Error( + "wake-queue: the deferred wake to merge into was not found for this company", + ); + } + if (input.coalescedReceipt) { + await tx.insert(agentWakeupRequests).values({ + ...input.coalescedReceipt, + companyId: input.companyId, + agentId: prior.agentId, + runId: prior.runId, + payload: { ...input.coalescedReceipt.payload, coalescedIntoWakeupRequestId: prior.id }, + status: "coalesced", + coalescedCount: 1, + finishedAt: new Date(), + }); + } + }, + + async insertNewDeferredWake(scope, input) { + const tx = requireAdmissionTx(scope, input.companyId); + await tx.insert(agentWakeupRequests).values({ + ...input.durableReceipt, + companyId: input.companyId, + agentId: input.agentId, + source: input.source, + triggerDetail: input.triggerDetail, + reason: "issue_execution_deferred", + payload: input.payload, + status: DEFERRED_WAKE_STATUS, + requestedByActorType: input.requestedByActorType, + requestedByActorId: input.requestedByActorId, + idempotencyKey: input.idempotencyKey, + }); + }, + }; +} + +export function createPostgresWakeQueueAdapter(db: Db, deps: WakeQueuePostgresAdapterDeps): IssueLockWriter { + return { + async withIssueExecutionLock(input, fn): Promise { + return db.transaction(async (rawTx) => { + const tx = rawTx as unknown as Db; + const evidence = await readReleaseRun(tx, input); + const run = evidence.run; + if (!run) { + throw new Error(`wake-queue: run ${input.runId} was not found while releasing issue execution`); + } + const runSnapshot = toRunSnapshot(run); + if (input.resumeStrandedQueue && (!run.finishedAt || !["succeeded", "failed", "cancelled", "timed_out", "interrupted"].includes(run.status))) { + return { outcome: { kind: "released" }, postCommitEffects: [], run: runSnapshot }; + } + const contextIssueId = readNonEmptyString(parseObject(run.contextSnapshot).issueId); + + // Lock the context issue (if any) and every issue that still references this + // run, in id order, so two concurrent finalizations can never deadlock on + // each other's row-lock acquisition order. + await tx.execute( + contextIssueId + ? sql` + select id from issues + where company_id = ${input.companyId} + and ( + id = ${contextIssueId} + or execution_run_id = ${run.id} + or checkout_run_id = ${run.id} + ) + order by id + for update + ` + : sql` + select id from issues + where company_id = ${input.companyId} + and (execution_run_id = ${run.id} or checkout_run_id = ${run.id}) + order by id + for update + `, + ); + + if (input.resumeStrandedQueue && run.runtimeMode === "native") { + const [finalized] = contextIssueId && run.nativeIssueId === contextIssueId + ? await tx.select({ runId: nativeRunFinalizations.runId }).from(nativeRunFinalizations).where(and( + eq(nativeRunFinalizations.companyId, input.companyId), eq(nativeRunFinalizations.runId, run.id), + eq(nativeRunFinalizations.issueId, run.nativeIssueId), eq(nativeRunFinalizations.phase, "committed"), + )).limit(1) + : []; + if (!finalized) return { outcome: { kind: "released" }, postCommitEffects: [], run: runSnapshot }; + } + if (input.resumeStrandedQueue && await hasPendingRunCleanup(tx, input.companyId, run.id)) { + return { outcome: { kind: "released" }, postCommitEffects: [], run: runSnapshot }; + } + + const candidateIssues = await tx + .select() + .from(issues) + .where( + and( + eq(issues.companyId, input.companyId), + contextIssueId + ? or(eq(issues.id, contextIssueId), eq(issues.executionRunId, run.id), eq(issues.checkoutRunId, run.id)) + : or(eq(issues.executionRunId, run.id), eq(issues.checkoutRunId, run.id)), + ), + ) + .orderBy(asc(issues.id)); + + for (const issue of candidateIssues) { + if (issue.executionRunId === run.id || issue.checkoutRunId === run.id) { + await notifyExternalConversationSettlement(tx, { companyId: input.companyId, issueId: issue.id }); + await notifyUnsafeWorkspaceExportRecovery(tx, { companyId: input.companyId, issueId: issue.id }); + await notifyNativeStatusReassessment(tx, { companyId: input.companyId, issueId: issue.id }); + } + } + // Two separate updates: a retry can move `executionRunId` to a new run + // while `checkoutRunId` still points at this one finishing. + await tx + .update(issues) + .set({ executionRunId: null, executionAgentNameKey: null, executionLockedAt: null, updatedAt: input.now }) + .where(and(eq(issues.companyId, input.companyId), eq(issues.executionRunId, run.id))); + await tx + .update(issues) + .set({ checkoutRunId: null, updatedAt: input.now }) + .where(and(eq(issues.companyId, input.companyId), eq(issues.checkoutRunId, run.id))); + + const issueRow = + (contextIssueId ? candidateIssues.find((candidate) => candidate.id === contextIssueId) : candidateIssues[0]) ?? null; + + // A queue interrupt authorizes only its original pending queue. Replays + // after dispatch or deleting the final message cannot launch other work. + const interruptQueueId = run.runtimeMode !== "native" + ? readNonEmptyString(run.resultJson?.queuedCommentInterruptQueueId) + : null; + const [interruptedQueue] = interruptQueueId && issueRow + ? await tx.select({ id: agentWakeupRequests.id }).from(agentWakeupRequests).where(and( + eq(agentWakeupRequests.id, interruptQueueId), + eq(agentWakeupRequests.companyId, run.companyId), + eq(agentWakeupRequests.agentId, run.agentId), + eq(agentWakeupRequests.status, "deferred_issue_execution"), + sql`${agentWakeupRequests.payload}->>'issueId' = ${issueRow.id}`, + )).limit(1) + : []; + const cancellation = readRunCancellation(run.resultJson); + const queuedOperatorStop = run.runtimeMode === "legacy" && run.status === "cancelled" && + run.runtimeModeResolvedAt === null && run.startedAt === null && run.processStartedAt === null && + run.processPid === null && run.processGroupId === null && + cancellation?.source === "operator" && cancellation.expected; + const preDrainFacts: PreDrainFacts = { + issueRowPresent: issueRow !== null, + executionRunIdMatchesRun: !issueRow || !issueRow.executionRunId || issueRow.executionRunId === run.id, + isWorkspaceValidationFailedRun: isWorkspaceValidationFailedRun(run), + isConfigurationIncompleteFailedRun: isConfigurationIncompleteFailedRun(run), + issueStatus: issueRow?.status ?? "", + hasAssigneeUser: Boolean(issueRow?.assigneeUserId), + assigneeAgentMatchesRunAgent: issueRow?.assigneeAgentId === run.agentId, + legacyExecutionNeedsReconciliation: evidence.kind === "complete" && + await legacyExecutionNeedsReconciliationWithEvidence(tx as unknown as Db, evidence.run), + // An operator stop never promotes old queued work by itself. The + // next explicit wake adopts those messages atomically when it + // queues a run. + executionCancellationAcknowledged: + isAcknowledgedNativeReassignmentStop(run) || + (run.status === "cancelled" && + (queuedOperatorStop || parseObject(run.resultJson?.executionCancellation).state === "acknowledged" || isAcknowledgedNativeStop(run)) && + !interruptedQueue), + }; + const preDrain = decidePreDrain(preDrainFacts); + + if (preDrain.kind === "released") { + return { outcome: { kind: "released" }, postCommitEffects: [], run: runSnapshot }; + } + + // decidePreDrain only returns "blocked" or "proceed" when the issue row is present. + if (!issueRow) { + throw new Error(`wake-queue: pre-drain decision ${preDrain.kind} reached without an issue row`); + } + + // Enqueue does not stamp executionRunId until dispatch. A concurrent + // queued successor still owns the next turn, including during a late + // finalization/stranded-queue retry under this issue lock. Another + // agent's review participation retains its separate recovery path. + const [successor] = await tx.select({ id: heartbeatRuns.id }).from(heartbeatRuns).where(and( + eq(heartbeatRuns.companyId, input.companyId), + input.resumeStrandedQueue ? undefined : eq(heartbeatRuns.agentId, run.agentId), + input.resumeStrandedQueue ? undefined : sql`${heartbeatRuns.id} <> ${run.id}`, + or(eq(heartbeatRuns.nativeIssueId, issueRow.id), + sql`${heartbeatRuns.contextSnapshot}->>'issueId' = ${issueRow.id}`), + or(inArray(heartbeatRuns.status, ["queued", "running", "scheduled_retry"]), + input.resumeStrandedQueue ? sql`(${heartbeatRuns.createdAt}, ${heartbeatRuns.id}) > ( + SELECT source.created_at, source.id FROM heartbeat_runs source + WHERE source.company_id = ${input.companyId} AND source.id = ${run.id})` : undefined), + )).limit(1); + if (successor) return { outcome: { kind: "released" }, postCommitEffects: [], run: runSnapshot }; + + if (preDrain.kind === "blocked") { + return { + outcome: { + kind: "blocked", + issue: toIssueSnapshot(issueRow), + previousStatus: issueRow.status as "todo" | "in_progress", + noticeKind: preDrain.noticeKind, + }, + postCommitEffects: [], + run: runSnapshot, + }; + } + + // The durable external answer owns the sole successor of this retired + // question source; release locks without creating another incident. + if ( + run.runtimeMode === "native" && + run.status === "cancelled" && + run.errorCode === "external_chat_continuation" && + (await isRetiredExternalChatQuestionSource(tx, { + companyId: run.companyId, + issueId: issueRow.id, + agentId: run.agentId, + runId: run.id, + })) + ) { + return { + outcome: { kind: "released" }, + postCommitEffects: [], + run: runSnapshot, + }; + } + + if (await recordNativeTerminalRecoveryIfNeeded(tx, run, issueRow, input.now)) { + return { outcome: { kind: "released" }, postCommitEffects: [], run: runSnapshot }; + } + + // A release must leave deferred messages intact while execution is held. + // The finishing conversation may still own its lease until finally cleanup. + // Allow only bounded retry planning in that case; admission stays gated. + const executionBlocker = await getExecutionBlocker(tx, issueRow.companyId, issueRow.id); + const recoveryOnly = Boolean(executionBlocker && + executionBlocker.cause === "execution_owner_active" && executionBlocker.runId === run.id && + runSnapshot.conversationContinuation && ["failed", "timed_out", "interrupted"].includes(run.status)); + if (executionBlocker && !recoveryOnly) { + return { outcome: { kind: "released" }, postCommitEffects: [], run: runSnapshot }; + } + + // Releases still settle while Agent Chat is disabled, but no deferred + // turn or recovery successor may be created. Check here in the shared + // transaction so cleanup retries and restart sweeps use the same gate. + if (issueRow.conversationAgentId && !(await instanceSettingsService(tx).getExperimental()).enableAgentChat) { + return { outcome: { kind: "released" }, postCommitEffects: [], run: runSnapshot }; + } + + const locked: LockedIssueExecution = { primaryIssue: toIssueSnapshot(issueRow), run: runSnapshot, recoveryOnly }; + const result = await fn(locked, { host: buildHost(tx, deps), transaction: buildTransaction(tx, deps, db, run) }); + // Explicit retry admission requires the exact source's task claim. + // Preserve it only when the existing queue-first recovery policy + // requests that retry. The clear/restore stays inside this task lock. + if (run.runtimeMode === "legacy" && ["failed", "timed_out"].includes(run.status) && + run.contextSnapshot?.explicitUserContinuation && issueRow.executionRunId === run.id && + result.postCommitEffects.some(effect => effect.kind === "conversation_retry_requested" && effect.runId === run.id)) { + await tx.update(issues).set({ executionRunId: run.id, + executionAgentNameKey: issueRow.executionAgentNameKey, + executionLockedAt: issueRow.executionLockedAt, + }).where(and(eq(issues.companyId, input.companyId), eq(issues.id, issueRow.id), + sql`${issues.executionRunId} is null`)); + } + return { ...result, run: runSnapshot }; + }); + }, + }; +} diff --git a/server/src/services/budgets.ts b/server/src/services/budgets.ts index 0eff1c83613..cc4cf161464 100644 --- a/server/src/services/budgets.ts +++ b/server/src/services/budgets.ts @@ -1 +1,1158 @@ +import { notifyRunDispatch } from "./run-dispatch-notifications.js"; import { notifyStatusCardIssueChange } from "./status-card-notifications.js"; +import { notifyApprovalWork } from "./approval-work-notifications.js"; +import { notifyCompanyRetentionWork } from "./retention-work-notifications.js"; +import { addCents, compareCents, normalizeCents } from "@paperclipai/shared"; +import { and, desc, eq, inArray, sql, or, isNull } from "drizzle-orm"; +import type { Db } from "@paperclipai/db"; + +import { + agents, + approvals, + budgetIncidents, + budgetPolicies, + budgetReservations, + companies, + costEvents, + heartbeatRuns, + projects, +} from "@paperclipai/db"; +import type { + BudgetIncident, + BudgetIncidentResolutionInput, + BudgetMetric, + BudgetOverview, + PauseReason, + BudgetPolicy, + BudgetPolicySummary, + BudgetPolicyUpsertInput, + BudgetScopeType, + BudgetThresholdType, + BudgetWindowKind, +} from "@paperclipai/shared"; +import { HttpError, notFound, unprocessable } from "../errors.js"; +import { logger } from "../middleware/logger.js"; +import { withAccountingTransaction } from "./accounting-transaction.js"; +import { logActivity, type LogActivityInput, type ActivityPublication } from "./activity-log.js"; +import { computeObservedSpend, computeObservedSpends, type ObservedSpend } from "./budget-exposure.js"; +import { notifyDeliveryWork, DELIVERY_QUEUES } from "./delivery-work-notifications.js"; +import { listPendingBudgetCancellationScopes } from "./heartbeat/budget-cancellation.js"; +import { withBudgetSpan } from "./budget-tracing.js"; +export { computeObservedSpend, computeObservedSpends } from "./budget-exposure.js"; +export type { BudgetExposure } from "./budget-exposure.js"; + +type ScopeRecord = { + companyId: string; + name: string; + paused: boolean; + pauseReason: PauseReason | null; + budgetHeld: boolean; +}; + +type PolicyRow = typeof budgetPolicies.$inferSelect; +type IncidentRow = typeof budgetIncidents.$inferSelect; + +export type BudgetEnforcementScope = { + companyId: string; + scopeType: BudgetScopeType; + scopeId: string; + /** Snapshot taken under the company lock, before a later budget grant can admit new work. */ + createdBefore?: Date; + /** Revalidated at the durable cancellation write, under the admission lock. */ + enforcement?: { policyId: string; version: number }; +}; + +export type BudgetServiceHooks = { + cancelWorkForScope?: (scope: BudgetEnforcementScope) => Promise; +}; + +function currentUtcMonthWindow(now = new Date()) { + const year = now.getUTCFullYear(); + const month = now.getUTCMonth(); + const start = new Date(Date.UTC(year, month, 1, 0, 0, 0, 0)); + const end = new Date(Date.UTC(year, month + 1, 1, 0, 0, 0, 0)); + return { start, end }; +} + +function resolveWindow(windowKind: BudgetWindowKind, now = new Date()) { + if (windowKind === "lifetime") { + return { + start: new Date(Date.UTC(1970, 0, 1, 0, 0, 0, 0)), + end: new Date(Date.UTC(9999, 0, 1, 0, 0, 0, 0)), + }; + } + return currentUtcMonthWindow(now); +} + +function budgetStatusFromObserved( + observedAmount: string, + amount: number, + warnPercent: number, +): BudgetPolicySummary["status"] { + if (amount <= 0) return "ok"; + if (compareCents(observedAmount, amount) >= 0) return "hard_stop"; + if (compareCents(observedAmount, Math.ceil((amount * warnPercent) / 100)) >= 0) return "warning"; + return "ok"; +} + +function normalizeScopeName(scopeType: BudgetScopeType, name: string) { + if (scopeType === "company") return name; + return name.trim().length > 0 ? name : scopeType; +} + +async function resolveScopeRecord(db: Db, scopeType: BudgetScopeType, scopeId: string): Promise { + if (scopeType === "company") { + const row = await db + .select({ + companyId: companies.id, + name: companies.name, + status: companies.status, + pauseReason: companies.pauseReason, + pausedAt: companies.pausedAt, + }) + .from(companies) + .where(eq(companies.id, scopeId)) + .then((rows) => rows[0] ?? null); + if (!row) throw notFound("Company not found"); + return { + companyId: row.companyId, + name: row.name, + paused: row.status === "paused" || Boolean(row.pausedAt), + pauseReason: (row.pauseReason as ScopeRecord["pauseReason"]) ?? null, + budgetHeld: row.pauseReason === "budget", + }; + } + + if (scopeType === "agent") { + const row = await db + .select({ + companyId: agents.companyId, + name: agents.name, + status: agents.status, + pauseReason: agents.pauseReason, + lifecycleHolds: agents.lifecycleHolds, + }) + .from(agents) + .where(eq(agents.id, scopeId)) + .then((rows) => rows[0] ?? null); + if (!row) throw notFound("Agent not found"); + return { + companyId: row.companyId, + name: row.name, + paused: row.status === "paused", + pauseReason: (row.pauseReason as ScopeRecord["pauseReason"]) ?? null, + budgetHeld: row.lifecycleHolds.includes("budget"), + }; + } + + const row = await db + .select({ + companyId: projects.companyId, + name: projects.name, + pauseReason: projects.pauseReason, + pausedAt: projects.pausedAt, + }) + .from(projects) + .where(eq(projects.id, scopeId)) + .then((rows) => rows[0] ?? null); + if (!row) throw notFound("Project not found"); + return { + companyId: row.companyId, + name: row.name, + paused: Boolean(row.pausedAt), + pauseReason: (row.pauseReason as ScopeRecord["pauseReason"]) ?? null, + budgetHeld: row.pauseReason === "budget", + }; +} + +// Keep a large dashboard from filling the database pool with queued reads. +async function mapBudgetReads(rows: T[], read: (row: T) => Promise): Promise { + const result: R[] = []; + for (let start = 0; start < rows.length; start += 4) { + result.push(...await Promise.all(rows.slice(start, start + 4).map(read))); + } + return result; +} + +/** Filter in PostgreSQL so one run never loads unrelated agents' policies. */ +export function budgetPoliciesForRun(companyId: string, agentId: string | null, projectId: string | null) { + return or( + and(eq(budgetPolicies.scopeType, "company"), eq(budgetPolicies.scopeId, companyId)), + agentId ? and(eq(budgetPolicies.scopeType, "agent"), eq(budgetPolicies.scopeId, agentId)) : undefined, + projectId ? and(eq(budgetPolicies.scopeType, "project"), eq(budgetPolicies.scopeId, projectId)) : undefined, + ); +} + +function unpricedUsageBlocks(policy: PolicyRow, observed: ObservedSpend) { + return policy.unpricedUsagePolicy !== "allow" + && (observed.unpricedEventCount > 0 || observed.unknownBrowserRunCount > 0); +} + +function observedBlocks(policy: PolicyRow, observed: Awaited>) { + if (!policy.isActive || !policy.hardStopEnabled || policy.amount <= 0) return false; + return compareCents(observed.totalExact, policy.amount) >= 0 || (observed.pendingRunCount - observed.recoveringRunCount) > 0 || unpricedUsageBlocks(policy, observed); +} +export async function policyBlocks(db: Db, policy: PolicyRow) { + if (!policy.isActive || !policy.hardStopEnabled || policy.amount <= 0) return false; + return observedBlocks(policy, await computeObservedSpend(db, policy)); +} + +/** Supplemental handoff check after authoritative accounting admission commits. + * Read only: the caller can hold issue/run ownership, so this must never take + * the company lock, reconcile holds, or deliver effects. A later concurrent + * charge targets already-admitted work through the normal durable Stop path. */ +export async function recheckAdmittedRunBudgetReadOnly(db: Db, input: { + companyId: string; agentId: string; runId: string; projectId: string | null; reservationId: string; +}): Promise { + return withBudgetSpan("budget.admission.evaluate", async () => { + const [reservation] = await db.select({ amountCents: budgetReservations.amountCents }).from(budgetReservations).where(and( + eq(budgetReservations.id, input.reservationId), eq(budgetReservations.companyId, input.companyId), + eq(budgetReservations.runId, input.runId), eq(budgetReservations.agentId, input.agentId), + sql`${budgetReservations.projectId} is not distinct from ${input.projectId}::uuid`, + eq(budgetReservations.state, "held"), + )); + if (!reservation) return "Provider dispatch no longer owns a held budget reservation"; + const policies = await db.select().from(budgetPolicies).where(and(eq(budgetPolicies.companyId, input.companyId), + eq(budgetPolicies.isActive, true), eq(budgetPolicies.hardStopEnabled, true), eq(budgetPolicies.metric, "billed_cents"), + sql`${budgetPolicies.amount} > 0`, budgetPoliciesForRun(input.companyId, input.agentId, input.projectId))); + const observations = await computeObservedSpends(db, policies); + for (const [index, policy] of policies.entries()) { + const observed = observations[index]; + if (observedBlocks(policy, observed)) return "Applicable budget no longer permits provider work"; + const capacity = compareCents(addCents(observed.ledgerCents, observed.effectiveHeldCents), policy.amount); + if (capacity > 0 || (compareCents(reservation.amountCents, 0) === 0 && capacity >= 0)) + return "Available budget is reserved by unfinished runs"; + } + return null; + }); +} + +function buildApprovalPayload(input: { + policy: PolicyRow; + scopeName: string; + thresholdType: BudgetThresholdType; + amountObserved: number; + amountObservedExact: string; + windowStart: Date; + windowEnd: Date; +}) { + return { + scopeType: input.policy.scopeType, + scopeId: input.policy.scopeId, + scopeName: input.scopeName, + metric: input.policy.metric, + windowKind: input.policy.windowKind, + thresholdType: input.thresholdType, + budgetAmount: input.policy.amount, + observedAmount: input.amountObserved, + observedAmountExact: input.amountObservedExact, + warnPercent: input.policy.warnPercent, + windowStart: input.windowStart.toISOString(), + windowEnd: input.windowEnd.toISOString(), + policyId: input.policy.id, + guidance: "Resolve pending or unpriced accounting, raise the budget, or keep the scope paused.", + }; +} + +async function markApprovalStatus( + db: Db, + approvalId: string | null, + status: "approved" | "rejected", + decisionNote: string | null | undefined, + decidedByUserId: string, +) { + if (!approvalId) return; + const [approval] = await db.select({ companyId: approvals.companyId }).from(approvals).where(eq(approvals.id, approvalId)); + if (!approval) return; + await notifyApprovalWork(db, approval.companyId, { approvalId }); + await db + .update(approvals) + .set({ + status, + decisionNote: decisionNote ?? null, + decidedByUserId, + decidedAt: new Date(), + updatedAt: new Date(), + }) + .where(eq(approvals.id, approvalId)); +} + +/** Internal: caller must hold the company accounting transaction lock. */ +export function budgetServiceInTransaction(db: Db, publications: ActivityPublication[] = []) { + const recordActivity = (input: LogActivityInput) => logActivity(db, input, publications); + async function pauseScopeForBudget(policy: PolicyRow) { + const now = new Date(); + if (policy.scopeType === "agent") { + const { applyAgentBudgetHoldInTransaction } = await import("./agent-lifecycle.js"); + await applyAgentBudgetHoldInTransaction(db, policy.companyId, policy.scopeId, true); + return; + } + + if (policy.scopeType === "project") { + await notifyRunDispatch(db, { companyId: policy.companyId }); + await db + .update(projects) + .set({ + pauseReason: "budget", + pausedAt: now, + updatedAt: now, + }) + .where(and(eq(projects.id, policy.scopeId), isNull(projects.pausedAt))); + return; + } + + const scope = await resolveScopeRecord(db, "company", policy.scopeId); + if (scope.paused) return; + await notifyRunDispatch(db, { companyId: policy.companyId }); + await notifyCompanyRetentionWork(db, policy.companyId); + await notifyStatusCardIssueChange(db, policy.companyId); + await notifyDeliveryWork(db, DELIVERY_QUEUES.agentTimers); + await notifyDeliveryWork(db, DELIVERY_QUEUES.routineSchedules); + await notifyDeliveryWork(db, DELIVERY_QUEUES.issueMonitors); + await notifyDeliveryWork(db, DELIVERY_QUEUES.agentLifecycle, policy.companyId); + await notifyDeliveryWork(db, DELIVERY_QUEUES.skillSourceSync); + await db + .update(companies) + .set({ + status: "paused", + pauseReason: "budget", + pausedAt: now, + updatedAt: now, + }) + .where(and(eq(companies.id, policy.scopeId), eq(companies.status, "active"))); + } + + async function pauseAndCancelScopeForBudget(policy: PolicyRow, requestCancellation = true) { + await pauseScopeForBudget(policy); + if (!requestCancellation) return; + // A pending generation already covers all admitted work in this scope. + // Rechecks must not invalidate a worker that is delivering that intent. + await notifyDeliveryWork(db, DELIVERY_QUEUES.budgetEnforcement, policy.companyId); + await db.update(budgetPolicies).set({ + enforcementVersion: sql`case when ${budgetPolicies.enforcementVersion} > ${budgetPolicies.enforcementDeliveredVersion} + then ${budgetPolicies.enforcementVersion} else ${budgetPolicies.enforcementVersion} + 1 end`, + updatedAt: sql`case when ${budgetPolicies.enforcementVersion} > ${budgetPolicies.enforcementDeliveredVersion} + then ${budgetPolicies.updatedAt} else now() end`, + }).where(eq(budgetPolicies.id, policy.id)); + } + + async function resumeScopeFromBudget(policy: PolicyRow) { + const policies = await db.select().from(budgetPolicies).where(and( + eq(budgetPolicies.companyId, policy.companyId), eq(budgetPolicies.scopeType, policy.scopeType), + eq(budgetPolicies.scopeId, policy.scopeId), eq(budgetPolicies.isActive, true), eq(budgetPolicies.hardStopEnabled, true), + )); + const observations = await computeObservedSpends(db, policies); + if (policies.some((candidate, index) => observedBlocks(candidate, observations[index]))) return; + const now = new Date(); + if (policy.scopeType === "agent") { + const { applyAgentBudgetHoldInTransaction } = await import("./agent-lifecycle.js"); + await applyAgentBudgetHoldInTransaction(db, policy.companyId, policy.scopeId, false); + return; + } + + if (policy.scopeType === "project") { + await notifyRunDispatch(db, { companyId: policy.companyId }); + await db + .update(projects) + .set({ + pauseReason: null, + pausedAt: null, + updatedAt: now, + }) + .where(and(eq(projects.id, policy.scopeId), eq(projects.pauseReason, "budget"))); + return; + } + + await notifyDeliveryWork(db, DELIVERY_QUEUES.skillSourceSync); + const scope = await resolveScopeRecord(db, "company", policy.scopeId); + if (scope.pauseReason !== "budget") return; + await notifyRunDispatch(db, { companyId: policy.companyId }); + await notifyCompanyRetentionWork(db, policy.companyId); + await notifyStatusCardIssueChange(db, policy.companyId); + await notifyDeliveryWork(db, DELIVERY_QUEUES.agentTimers); + await notifyDeliveryWork(db, DELIVERY_QUEUES.routineSchedules); + await notifyDeliveryWork(db, DELIVERY_QUEUES.issueMonitors); + await notifyDeliveryWork(db, DELIVERY_QUEUES.agentLifecycle, policy.companyId); + await db + .update(companies) + .set({ + status: "active", + pauseReason: null, + pausedAt: null, + updatedAt: now, + }) + .where(and(eq(companies.id, policy.scopeId), eq(companies.status, "paused"), eq(companies.pauseReason, "budget"))); + } + + async function getPolicyRow(policyId: string) { + const policy = await db + .select() + .from(budgetPolicies) + .where(eq(budgetPolicies.id, policyId)) + .then((rows) => rows[0] ?? null); + if (!policy) throw notFound("Budget policy not found"); + return policy; + } + + async function listPolicyRows(companyId: string) { + return db + .select() + .from(budgetPolicies) + .where(eq(budgetPolicies.companyId, companyId)) + .orderBy(desc(budgetPolicies.updatedAt)); + } + + async function buildPolicySummary(policy: PolicyRow): Promise { + const scope = await resolveScopeRecord(db, policy.scopeType as BudgetScopeType, policy.scopeId); + const observed = await computeObservedSpend(db, policy); + const { total: observedAmount, totalExact: observedAmountExact, unpricedEventCount, pendingRunCount } = observed; + const { windowStart: start, windowEnd: end } = observed; + const amount = policy.isActive ? policy.amount : 0; + const utilizationPercent = + amount > 0 ? Number(((observedAmount / amount) * 100).toFixed(2)) : 0; + return { + policyId: policy.id, + companyId: policy.companyId, + scopeType: policy.scopeType as BudgetScopeType, + scopeId: policy.scopeId, + scopeName: normalizeScopeName(policy.scopeType as BudgetScopeType, scope.name), + metric: policy.metric as BudgetMetric, + windowKind: policy.windowKind as BudgetWindowKind, + amount, + observedAmount, + observedAmountExact, + reservationCents: policy.reservationCents, + unpricedEventCount, + pendingRunCount, + unpricedUsagePolicy: policy.unpricedUsagePolicy as "block" | "allow", + remainingAmount: amount > 0 ? Math.max(0, amount - observedAmount) : 0, + utilizationPercent, + warnPercent: policy.warnPercent, + hardStopEnabled: policy.hardStopEnabled, + notifyEnabled: policy.notifyEnabled, + isActive: policy.isActive, + status: observedBlocks(policy, observed) ? "hard_stop" : policy.isActive + ? budgetStatusFromObserved(observedAmountExact, amount, policy.warnPercent) + : "ok", + paused: scope.paused, + pauseReason: scope.pauseReason, + windowStart: start, + windowEnd: end, + }; + } + + async function createIncidentIfNeeded( + policy: PolicyRow, + thresholdType: BudgetThresholdType, + observed: Awaited>, + ) { + const { windowStart: start, windowEnd: end } = observed; + const existing = await db + .select() + .from(budgetIncidents) + .where( + and( + eq(budgetIncidents.policyId, policy.id), + eq(budgetIncidents.windowStart, start), + eq(budgetIncidents.thresholdType, thresholdType), + or(eq(budgetIncidents.status, "open"), and(eq(budgetIncidents.status, "dismissed"), eq(budgetIncidents.amountLimit, policy.amount))), + ), + ) + .orderBy(desc(budgetIncidents.createdAt)) + .then((rows) => rows[0] ?? null); + if (existing) return { incident: existing, created: false }; + + const { total: amountObserved, totalExact: amountObservedExact } = observed; + const scope = await resolveScopeRecord(db, policy.scopeType as BudgetScopeType, policy.scopeId); + const payload = buildApprovalPayload({ + policy, + scopeName: normalizeScopeName(policy.scopeType as BudgetScopeType, scope.name), + thresholdType, + amountObserved, + amountObservedExact, + windowStart: start, + windowEnd: end, + }); + + if (thresholdType === "hard") await notifyApprovalWork(db, policy.companyId); + const approval = thresholdType === "hard" + ? await db + .insert(approvals) + .values({ + companyId: policy.companyId, + type: "budget_override_required", + requestedByUserId: null, + requestedByAgentId: null, + status: "pending", + payload, + }) + .returning() + .then((rows) => rows[0] ?? null) + : null; + + const incident = await db + .insert(budgetIncidents) + .values({ + companyId: policy.companyId, + policyId: policy.id, + scopeType: policy.scopeType, + scopeId: policy.scopeId, + metric: policy.metric, + windowKind: policy.windowKind, + windowStart: start, + windowEnd: end, + thresholdType, + amountLimit: policy.amount, + amountObserved: sql`${amountObservedExact}::numeric`, + status: "open", + approvalId: approval?.id ?? null, + }) + .returning() + .then((rows) => rows[0] ?? null); + return incident ? { incident, created: true } : null; + } + + async function resolveOpenSoftIncidents(policyId: string) { + await db + .update(budgetIncidents) + .set({ + status: "resolved", + resolvedAt: new Date(), + updatedAt: new Date(), + }) + .where( + and( + eq(budgetIncidents.policyId, policyId), + eq(budgetIncidents.thresholdType, "soft"), + eq(budgetIncidents.status, "open"), + ), + ); + } + + async function resolveOpenIncidentsForPolicy( + policyId: string, + approvalStatus: "approved" | "rejected" | null, + decidedByUserId: string | null, + thresholdType?: BudgetThresholdType, + ) { + const condition = and(eq(budgetIncidents.policyId, policyId), eq(budgetIncidents.status, "open"), + thresholdType ? eq(budgetIncidents.thresholdType, thresholdType) : undefined); + const openRows = await db + .select() + .from(budgetIncidents) + .where(condition); + + await db + .update(budgetIncidents) + .set({ + status: "resolved", + resolvedAt: new Date(), + updatedAt: new Date(), + }) + .where(condition); + + if (!approvalStatus || !decidedByUserId) return; + for (const row of openRows) { + await markApprovalStatus(db, row.approvalId ?? null, approvalStatus, "Resolved via budget update", decidedByUserId); + } + } + + async function hydrateIncidentRows(rows: IncidentRow[]): Promise { + const approvalIds = rows.map((row) => row.approvalId).filter((value): value is string => Boolean(value)); + const approvalRows = approvalIds.length > 0 + ? await db + .select({ id: approvals.id, status: approvals.status }) + .from(approvals) + .where(inArray(approvals.id, approvalIds)) + : []; + const approvalStatusById = new Map(approvalRows.map((row) => [row.id, row.status])); + + return mapBudgetReads(rows, async (row) => { + const scope = await resolveScopeRecord(db, row.scopeType as BudgetScopeType, row.scopeId); + return { + id: row.id, + companyId: row.companyId, + policyId: row.policyId, + scopeType: row.scopeType as BudgetScopeType, + scopeId: row.scopeId, + scopeName: normalizeScopeName(row.scopeType as BudgetScopeType, scope.name), + metric: row.metric as BudgetMetric, + windowKind: row.windowKind as BudgetWindowKind, + windowStart: row.windowStart, + windowEnd: row.windowEnd, + thresholdType: row.thresholdType as BudgetThresholdType, + amountLimit: row.amountLimit, + amountObserved: row.amountObserved, + status: row.status as BudgetIncident["status"], + approvalId: row.approvalId ?? null, + approvalStatus: row.approvalId ? approvalStatusById.get(row.approvalId) ?? null : null, + resolvedAt: row.resolvedAt ?? null, + createdAt: row.createdAt, + updatedAt: row.updatedAt, + }; + }); + } + + async function reconcileScope(companyId: string, scopeType: BudgetScopeType, scopeId: string, + prepared?: { scope: ScopeRecord; policies: PolicyRow[]; observations: ObservedSpend[] }) { + const scope = prepared?.scope ?? await resolveScopeRecord(db, scopeType, scopeId); + if (scope.companyId !== companyId) throw notFound("Budget scope not found"); + const policies = prepared?.policies ?? await db.select().from(budgetPolicies).where(and( + eq(budgetPolicies.companyId, companyId), eq(budgetPolicies.scopeType, scopeType), eq(budgetPolicies.scopeId, scopeId), + )); + let recoveringPolicy: PolicyRow | undefined; + const observations = prepared?.observations ?? await computeObservedSpends(db, policies); + for (const [index, policy] of policies.entries()) { + const observed = observations[index]; + if (observedBlocks(policy, observed)) { + await createIncidentIfNeeded(policy, "hard", observed); + await pauseAndCancelScopeForBudget(policy, !scope.budgetHeld); + const title = scopeType === "company" ? "Company" : scopeType === "agent" ? "Agent" : "Project"; + if (observed.pendingRunCount > 0) { + return { scopeType, scopeId, scopeName: scope.name, reason: `${title} cannot start work while completed runs await accounting.` }; + } + if (unpricedUsageBlocks(policy, observed)) { + return { scopeType, scopeId, scopeName: scope.name, reason: `${title} cannot start work because recorded usage has no reliable price.` }; + } + return { scopeType, scopeId, scopeName: scope.name, reason: scopeType === "project" + ? "Project cannot start work because its budget hard-stop is still exceeded." + : `${title} is paused because its budget hard-stop was reached.` }; + } + // An open native coordinator must finish the original run's accounting. + // Deny fresh work without pausing/cancelling that recovery or its hold. + if (policy.isActive && policy.hardStopEnabled && policy.amount > 0 && observed.recoveringRunCount > 0) recoveringPolicy ??= policy; + // A closed UTC window, inactive policy, or budget raise releases only + // budget-owned pauses. Manual pauses remain operator decisions. + await resolveOpenIncidentsForPolicy(policy.id, "approved", "budget_service", "hard"); + if (!policy.isActive || !policy.notifyEnabled || compareCents(observed.totalExact, Math.ceil(policy.amount * policy.warnPercent / 100)) < 0) { + await resolveOpenIncidentsForPolicy(policy.id, "approved", "budget_service", "soft"); + } + } + // Agent status has its own admission gate and error code. Preserve a + // manual pause without misclassifying it as a budget failure. + if (scopeType !== "agent" && scope.paused && scope.pauseReason !== "budget") { + return { scopeType, scopeId, scopeName: scope.name, reason: `${scopeType === "company" ? "Company" : "Project"} is paused and cannot start new work.` }; + } + if (scope.paused && scope.pauseReason === "budget" && !policies.length) { + return { scopeType, scopeId, scopeName: scope.name, reason: "Budget pause requires a policy or an explicit operator resume." }; + } + if (scope.budgetHeld && policies[0]) await resumeScopeFromBudget(policies[0]); + if (recoveringPolicy) return { scopeType, scopeId, scopeName: scope.name, + reason: "New work must wait while a native run recovers its unfinished accounting." }; + return null; + } + + async function evaluateAffectedScopes(companyId: string, agentId: string | null, projectId: string | null) { + const candidatePolicies = await db + .select() + .from(budgetPolicies) + .where( + and( + eq(budgetPolicies.companyId, companyId), + eq(budgetPolicies.isActive, true), + budgetPoliciesForRun(companyId, agentId, projectId), + ), + ); + + const policies = candidatePolicies.filter(policy => policy.metric === "billed_cents" && policy.amount > 0); + const observations = await computeObservedSpends(db, policies); + for (const [index, policy] of policies.entries()) { + const observed = observations[index]; + const observedAmount = observed.total; + const softThreshold = Math.ceil((policy.amount * policy.warnPercent) / 100); + + if (policy.notifyEnabled && compareCents(observed.totalExact, softThreshold) >= 0 && (!policy.hardStopEnabled || compareCents(observed.totalExact, policy.amount) < 0)) { + const softIncident = await createIncidentIfNeeded(policy, "soft", observed); + if (softIncident?.created) { + await recordActivity({ + companyId: policy.companyId, + actorType: "system", + actorId: "budget_service", + action: "budget.soft_threshold_crossed", + entityType: "budget_incident", + entityId: softIncident.incident.id, + details: { + scopeType: policy.scopeType, + scopeId: policy.scopeId, + amountObserved: observedAmount, + amountLimit: policy.amount, + }, + }); + } + } + + if (observedBlocks(policy, observed)) { + await resolveOpenSoftIncidents(policy.id); + const hardIncident = await createIncidentIfNeeded(policy, "hard", observed); + await pauseAndCancelScopeForBudget(policy); + if (hardIncident?.created) { + await recordActivity({ + companyId: policy.companyId, + actorType: "system", + actorId: "budget_service", + action: "budget.hard_threshold_crossed", + entityType: "budget_incident", + entityId: hardIncident.incident.id, + details: { + scopeType: policy.scopeType, + scopeId: policy.scopeId, + amountObserved: observedAmount, + amountLimit: policy.amount, + approvalId: hardIncident.incident.approvalId ?? null, + }, + }); + } + } + } + // A final receipt or price correction can reduce a partial observation. + // Release only this scope's budget-owned hold after all its policies agree. + const healthyScopes = new Map(); + for (const [index, policy] of policies.entries()) { + if (!observedBlocks(policy, observations[index])) healthyScopes.set(`${policy.scopeType}:${policy.scopeId}`, policy); + } + for (const policy of healthyScopes.values()) { + const scope = await resolveScopeRecord(db, policy.scopeType as BudgetScopeType, policy.scopeId); + if (scope.budgetHeld) await resumeScopeFromBudget(policy); + } + } + + async function evaluateAdmissionInTransaction(companyId: string, agentId: string | null, + context?: { issueId?: string | null; projectId?: string | null }) { + const scopes: Array<{ scopeType: BudgetScopeType; scopeId: string }> = [ + { scopeType: "company", scopeId: companyId }, ...(agentId ? [{ scopeType: "agent" as const, scopeId: agentId }] : []), + ...(context?.projectId ? [{ scopeType: "project" as const, scopeId: context.projectId }] : []), + ]; + const records = await Promise.all(scopes.map(scope => resolveScopeRecord(db, scope.scopeType, scope.scopeId))); + if (records.some(scope => scope.companyId !== companyId)) throw notFound("Budget scope not found"); + const policies = await db.select().from(budgetPolicies).where(and(eq(budgetPolicies.companyId, companyId), + budgetPoliciesForRun(companyId, agentId, context?.projectId ?? null))); + const observations = await computeObservedSpends(db, policies); + const reservationIndexes = policies.map((policy, i) => policy.isActive && policy.hardStopEnabled && policy.metric === "billed_cents" && policy.amount > 0 ? i : -1).filter(i => i >= 0); + const admission = (block: Awaited>) => ({ block, + policies: reservationIndexes.map(i => policies[i]), observations: reservationIndexes.map(i => observations[i]) }); + for (const [index, { scopeType, scopeId }] of scopes.entries()) { + const indexes = policies.map((policy, i) => policy.scopeType === scopeType && policy.scopeId === scopeId ? i : -1).filter(i => i >= 0); + const block = await reconcileScope(companyId, scopeType, scopeId, { scope: records[index], + policies: indexes.map(i => policies[i]), observations: indexes.map(i => observations[i]) }); + if (block) return admission(block); + } + return admission(null); + } + + const evaluateAdmission = (...args: Parameters) => + withBudgetSpan("budget.admission.evaluate", () => evaluateAdmissionInTransaction(...args)); + + return { + reconcileScope, + listPolicies: async (companyId: string): Promise => { + const rows = await listPolicyRows(companyId); + return rows.map((row) => ({ + ...row, + scopeType: row.scopeType as BudgetScopeType, + unpricedUsagePolicy: row.unpricedUsagePolicy as "block" | "allow", + metric: row.metric as BudgetMetric, + windowKind: row.windowKind as BudgetWindowKind, + })); + }, + + upsertPolicy: async ( + companyId: string, + input: BudgetPolicyUpsertInput, + actorUserId: string | null, + ): Promise => { + const scope = await resolveScopeRecord(db, input.scopeType, input.scopeId); + if (scope.companyId !== companyId) { + throw unprocessable("Budget scope does not belong to company"); + } + + if (input.reservationCents !== undefined && compareCents(input.reservationCents, 0) < 0) throw unprocessable("Reservation cannot be negative"); + const metric = input.metric ?? "billed_cents"; + const windowKind = input.windowKind ?? (input.scopeType === "project" ? "lifetime" : "calendar_month_utc"); + const existing = await db + .select() + .from(budgetPolicies) + .where( + and( + eq(budgetPolicies.companyId, companyId), + eq(budgetPolicies.scopeType, input.scopeType), + eq(budgetPolicies.scopeId, input.scopeId), + eq(budgetPolicies.metric, metric), + eq(budgetPolicies.windowKind, windowKind), + ), + ) + .then((rows) => rows[0] ?? null); + + if (!existing && input.amount === undefined) throw unprocessable("Amount is required for a new budget policy"); + const amount = Math.max(0, Math.floor(input.amount ?? existing!.amount)); + // A saved zero disables enforcement. Raising it is an explicit limit + // setting; editing a disabled positive policy must preserve that choice. + const nextIsActive = amount > 0 && (input.isActive ?? (existing?.amount === 0 ? true : existing?.isActive) ?? true); + const now = new Date(); + await notifyDeliveryWork(db, DELIVERY_QUEUES.budgetEnforcement, companyId); + const row = existing + ? await db + .update(budgetPolicies) + .set({ + amount, + reservationCents: normalizeCents(input.reservationCents ?? existing.reservationCents ?? 0), + warnPercent: input.warnPercent ?? existing.warnPercent, + hardStopEnabled: input.hardStopEnabled ?? existing.hardStopEnabled, + notifyEnabled: input.notifyEnabled ?? existing.notifyEnabled, + unpricedUsagePolicy: input.unpricedUsagePolicy ?? existing.unpricedUsagePolicy, + isActive: nextIsActive, + updatedByUserId: actorUserId, + updatedAt: now, + enforcementVersion: sql`${budgetPolicies.enforcementVersion} + 1`, + }) + .where(eq(budgetPolicies.id, existing.id)) + .returning() + .then((rows) => rows[0]) + : await db + .insert(budgetPolicies) + .values({ + companyId, + scopeType: input.scopeType, + scopeId: input.scopeId, + metric, + windowKind, + amount, + reservationCents: normalizeCents(input.reservationCents ?? 0), + warnPercent: input.warnPercent ?? 80, + hardStopEnabled: input.hardStopEnabled ?? true, + notifyEnabled: input.notifyEnabled ?? true, + unpricedUsagePolicy: input.unpricedUsagePolicy ?? "block", + isActive: nextIsActive, + createdByUserId: actorUserId, + updatedByUserId: actorUserId, + }) + .returning() + .then((rows) => rows[0]); + + if (input.scopeType === "company" && windowKind === "calendar_month_utc") { + await db + .update(companies) + .set({ + budgetMonthlyCents: amount, + updatedAt: now, + }) + .where(eq(companies.id, input.scopeId)); + } + + if (input.scopeType === "agent" && windowKind === "calendar_month_utc") { + await db + .update(agents) + .set({ + budgetMonthlyCents: amount, + updatedAt: now, + }) + .where(eq(agents.id, input.scopeId)); + } + + if (row.isActive && amount > 0) { + const observed = await computeObservedSpend(db, row); + if (!observedBlocks(row, observed)) { + await resumeScopeFromBudget(row); + await resolveOpenIncidentsForPolicy(row.id, "approved", actorUserId ?? "budget_service", "hard"); + if (row.notifyEnabled && compareCents(observed.totalExact, Math.ceil(row.amount * row.warnPercent / 100)) >= 0) { + await createIncidentIfNeeded(row, "soft", observed); + } else { + await resolveOpenIncidentsForPolicy(row.id, null, null, "soft"); + } + } else { + const softThreshold = Math.ceil((row.amount * row.warnPercent) / 100); + if (row.notifyEnabled && compareCents(observed.totalExact, softThreshold) >= 0 && (!row.hardStopEnabled || compareCents(observed.totalExact, row.amount) < 0)) { + await createIncidentIfNeeded(row, "soft", observed); + } + if (observedBlocks(row, observed)) { + await resolveOpenSoftIncidents(row.id); + await createIncidentIfNeeded(row, "hard", observed); + await pauseAndCancelScopeForBudget(row); + } + } + } else { + await resumeScopeFromBudget(row); + await resolveOpenIncidentsForPolicy(row.id, "approved", actorUserId ?? "budget_service"); + } + + await recordActivity({ + companyId, + actorType: "user", + actorId: actorUserId ?? "board", + action: "budget.policy_upserted", + entityType: "budget_policy", + entityId: row.id, + details: { + scopeType: row.scopeType, + scopeId: row.scopeId, + amount: row.amount, + windowKind: row.windowKind, + }, + }); + + return buildPolicySummary(row); + }, + + overview: async (companyId: string): Promise => { + const rows = await listPolicyRows(companyId); + const policies = await mapBudgetReads(rows, buildPolicySummary); + const activeIncidentRows = await db + .select() + .from(budgetIncidents) + .where(and(eq(budgetIncidents.companyId, companyId), eq(budgetIncidents.status, "open"))) + .orderBy(desc(budgetIncidents.createdAt)); + const activeIncidents = await hydrateIncidentRows(activeIncidentRows); + return { + companyId, + policies, + activeIncidents, + pausedAgentCount: new Set(policies.filter((policy) => policy.scopeType === "agent" && policy.paused).map((policy) => policy.scopeId)).size, + pausedProjectCount: new Set(policies.filter((policy) => policy.scopeType === "project" && policy.paused).map((policy) => policy.scopeId)).size, + pendingApprovalCount: activeIncidents.filter((incident) => incident.approvalStatus === "pending").length, + }; + }, + + evaluateAffectedScopes, + evaluateCostEvent: (event: typeof costEvents.$inferSelect) => evaluateAffectedScopes(event.companyId, event.agentId, event.projectId), + + evaluateAdmission, + getInvocationBlock: async (companyId: string, agentId: string | null, context?: { issueId?: string | null; projectId?: string | null }) => + (await evaluateAdmission(companyId, agentId, context)).block, + + resolveIncident: async ( + companyId: string, + incidentId: string, + input: BudgetIncidentResolutionInput, + actorUserId: string, + ): Promise => { + const incident = await db + .select() + .from(budgetIncidents) + .where(eq(budgetIncidents.id, incidentId)) + .then((rows) => rows[0] ?? null); + if (!incident) throw notFound("Budget incident not found"); + if (incident.companyId !== companyId) throw notFound("Budget incident not found"); + + const policy = await getPolicyRow(incident.policyId); + if (input.action === "raise_budget_and_resume") { + const nextAmount = Math.max(0, Math.floor(input.amount ?? 0)); + const observed = await computeObservedSpend(db, policy); + if (unpricedUsageBlocks(policy, observed)) { + throw unprocessable("Resolve unpriced usage or explicitly allow it in the budget policy before resuming"); + } + if (observed.pendingRunCount > 0) throw unprocessable("Completed runs must finish accounting before resuming"); + if (compareCents(observed.totalExact, nextAmount) >= 0) { + throw unprocessable("New budget must exceed current observed spend"); + } + + const now = new Date(); + await notifyDeliveryWork(db, DELIVERY_QUEUES.budgetEnforcement, companyId); + await db + .update(budgetPolicies) + .set({ + amount: nextAmount, + isActive: true, + updatedByUserId: actorUserId, + updatedAt: now, + enforcementVersion: sql`${budgetPolicies.enforcementVersion} + 1`, + }) + .where(eq(budgetPolicies.id, policy.id)); + + if (policy.scopeType === "company" && policy.windowKind === "calendar_month_utc") { + await db + .update(companies) + .set({ budgetMonthlyCents: nextAmount, updatedAt: now }) + .where(eq(companies.id, policy.scopeId)); + } + + if (policy.scopeType === "agent" && policy.windowKind === "calendar_month_utc") { + await db + .update(agents) + .set({ budgetMonthlyCents: nextAmount, updatedAt: now }) + .where(eq(agents.id, policy.scopeId)); + } + + await resumeScopeFromBudget(policy); + await db + .update(budgetIncidents) + .set({ + status: "resolved", + resolvedAt: now, + updatedAt: now, + }) + .where(and(eq(budgetIncidents.policyId, policy.id), eq(budgetIncidents.status, "open"))); + + await markApprovalStatus(db, incident.approvalId ?? null, "approved", input.decisionNote, actorUserId); + } else { + await db + .update(budgetIncidents) + .set({ + status: "dismissed", + resolvedAt: new Date(), + updatedAt: new Date(), + }) + .where(eq(budgetIncidents.id, incident.id)); + await markApprovalStatus(db, incident.approvalId ?? null, "rejected", input.decisionNote, actorUserId); + } + + await recordActivity({ + companyId: incident.companyId, + actorType: "user", + actorId: actorUserId, + action: "budget.incident_resolved", + entityType: "budget_incident", + entityId: incident.id, + details: { + action: input.action, + amount: input.amount ?? null, + scopeType: incident.scopeType, + scopeId: incident.scopeId, + }, + }); + + const [updated] = await hydrateIncidentRows([{ + ...incident, + status: input.action === "raise_budget_and_resume" ? "resolved" : "dismissed", + resolvedAt: new Date(), + updatedAt: new Date(), + }]); + return updated!; + }, + }; +} + +/** Fence the durable stop intent against policy edits and admission. External + * process shutdown must happen after this transaction, without holding locks + * that the executor needs to persist its final usage receipt. */ +export async function withCurrentBudgetEnforcement(db: Db, scope: BudgetEnforcementScope, work: (tx: Db) => Promise): Promise { + return withAccountingTransaction(db, scope.companyId, async (tx) => { + if (scope.enforcement) { + const [policy] = await tx.select().from(budgetPolicies).where(and( + eq(budgetPolicies.id, scope.enforcement.policyId), eq(budgetPolicies.companyId, scope.companyId), + )); + if (!policy || policy.scopeType !== scope.scopeType || policy.scopeId !== scope.scopeId + || policy.enforcementVersion !== scope.enforcement.version || !(await policyBlocks(tx, policy))) return null; + } + return work(tx); + }); +} + +async function deliverScopes(items: T[], deliver: (item: T) => Promise) { + let next = 0; + await Promise.all(Array.from({ length: Math.min(4, items.length) }, async () => { + while (next < items.length) await deliver(items[next++]); + })); +} + +/** Cancellation is an at-least-once external effect. Only the delivery worker + * invokes this: request paths commit intent and return without awaiting it. */ +export async function deliverBudgetEnforcement(db: Db, hooks: BudgetServiceHooks, companyId?: string, _agentId?: string | null) { + if (!hooks.cancelWorkForScope) return; + const attempted = new Set(); + const scopeKey = (scope: BudgetEnforcementScope) => `${scope.companyId}:${scope.scopeType}:${scope.scopeId}`; + const pending = await db.select().from(budgetPolicies).where(and( + sql`${budgetPolicies.enforcementVersion} > ${budgetPolicies.enforcementDeliveredVersion}`, + companyId ? eq(budgetPolicies.companyId, companyId) : undefined, + )).orderBy(budgetPolicies.updatedAt, budgetPolicies.id).limit(100); + await deliverScopes(pending, policy => withBudgetSpan("budget.enforcement.scope", async () => { + try { + const cancellation = await withAccountingTransaction(db, policy.companyId, async tx => { + const [current] = await tx.select().from(budgetPolicies).where(eq(budgetPolicies.id, policy.id)); + if (!current || current.enforcementVersion !== policy.enforcementVersion || !(await policyBlocks(tx, current))) return null; + return { companyId: policy.companyId, scopeType: policy.scopeType as BudgetScopeType, scopeId: policy.scopeId, + createdBefore: new Date(), enforcement: { policyId: policy.id, version: policy.enforcementVersion } }; + }); + if (cancellation) { + attempted.add(scopeKey(cancellation)); + await hooks.cancelWorkForScope!(cancellation); + } + await db.update(budgetPolicies).set({ enforcementDeliveredVersion: policy.enforcementVersion }) + .where(and(eq(budgetPolicies.id, policy.id), eq(budgetPolicies.enforcementVersion, policy.enforcementVersion))); + } catch (error) { + logger.warn({ err: error, policyId: policy.id }, "Budget cancellation delivery pending; accounting is committed"); + // Rotate a failed scope behind other pending work. A poisoned delivery + // must not monopolize the bounded batch on every retry. + await db.update(budgetPolicies).set({ updatedAt: new Date() }).where(and(eq(budgetPolicies.id, policy.id), + eq(budgetPolicies.enforcementVersion, policy.enforcementVersion))).catch(() => undefined); + } + }, { "budget.enforcement.queue_delay_ms": Math.max(0, Date.now() - policy.updatedAt.getTime()) })); + // A committed run Stop outlives a policy edit or deletion. It remains owed + // even if no policy generation now authorizes a fresh cancellation. + const committed = (await listPendingBudgetCancellationScopes(db)) + .filter(scope => (!companyId || scope.companyId === companyId) && !attempted.has(scopeKey(scope))); + await deliverScopes(committed, async scope => { + try { await hooks.cancelWorkForScope!(scope); } + catch (error) { logger.warn({ err: error, companyId: scope.companyId, scopeType: scope.scopeType, scopeId: scope.scopeId }, "Committed budget Stop remains pending"); } + }); +} + +export async function hasPendingBudgetEnforcement(db: Db): Promise { + const [pending] = await db.select({ id: budgetPolicies.id }).from(budgetPolicies) + .where(sql`${budgetPolicies.enforcementVersion} > ${budgetPolicies.enforcementDeliveredVersion}`).limit(1); + return Boolean(pending) || (await listPendingBudgetCancellationScopes(db, 1)).length > 0; +} + +const budgetOwnedScope = or( + and(eq(budgetPolicies.scopeType, "company"), sql`exists (select 1 from ${companies} where ${companies.id} = ${budgetPolicies.scopeId} and ${companies.pauseReason} = 'budget')`), + and(eq(budgetPolicies.scopeType, "agent"), sql`exists (select 1 from ${agents} where ${agents.id} = ${budgetPolicies.scopeId} and ${agents.lifecycleHolds} @> '["budget"]'::jsonb)`), + and(eq(budgetPolicies.scopeType, "project"), sql`exists (select 1 from ${projects} where ${projects.id} = ${budgetPolicies.scopeId} and ${projects.pauseReason} = 'budget')`), +)!; + +const openIncidentScope = sql`exists (select 1 from ${budgetIncidents} + where ${budgetIncidents.policyId} = ${budgetPolicies.id} and ${budgetIncidents.status} = 'open')`; +const unfinishedRunScope = sql`exists (select 1 from ${heartbeatRuns} + where ${heartbeatRuns.companyId} = ${budgetPolicies.companyId} and ${heartbeatRuns.costAccountingPending} = true + and ${heartbeatRuns.costAccountedAt} is null and (${budgetPolicies.scopeType} = 'company' + or (${budgetPolicies.scopeType} = 'agent' and ${heartbeatRuns.agentId} = ${budgetPolicies.scopeId}) + or (${budgetPolicies.scopeType} = 'project' and ${heartbeatRuns.usageJson}->'ledgerScope'->>'projectId' = ${budgetPolicies.scopeId}::text)))`; + +export async function nextBudgetReconciliationAt(db: Db): Promise { + const [held] = await db.select({ id: budgetPolicies.id }).from(budgetPolicies) + .where(and(eq(budgetPolicies.windowKind, "calendar_month_utc"), or(budgetOwnedScope, openIncidentScope))).limit(1); + return held ? currentUtcMonthWindow().end : null; +} + +/** Startup repairs configured policy scopes, not every agent. Afterwards only + * held monthly scopes owe a timed reevaluation; writes reconcile their own scopes. */ +export async function reconcileDueBudgetPolicies(db: Db, options: { startup?: boolean; cursor?: string | null } = {}): Promise<{ nextCursor: string | null }> { + const actionable = options.startup ? or(budgetOwnedScope, openIncidentScope, + sql`${budgetPolicies.enforcementVersion} > ${budgetPolicies.enforcementDeliveredVersion}`, + and(eq(budgetPolicies.isActive, true), eq(budgetPolicies.hardStopEnabled, true), sql`${budgetPolicies.amount} > 0`, unfinishedRunScope)) + : and(eq(budgetPolicies.windowKind, "calendar_month_utc"), or(budgetOwnedScope, openIncidentScope)); + const rows = await db.select({ id: budgetPolicies.id, companyId: budgetPolicies.companyId, scopeType: budgetPolicies.scopeType, scopeId: budgetPolicies.scopeId }) + .from(budgetPolicies).where(and(actionable, options.cursor ? sql`${budgetPolicies.id} > ${options.cursor}::uuid` : undefined)) + .orderBy(budgetPolicies.id).limit(100); + const scopes = new Map(rows.map(scope => [`${scope.companyId}:${scope.scopeType}:${scope.scopeId}`, scope])); + for (const scope of scopes.values()) { + try { + await withAccountingTransaction(db, scope.companyId, (tx, publications) => budgetServiceInTransaction(tx, publications) + .reconcileScope(scope.companyId, scope.scopeType as BudgetScopeType, scope.scopeId)); + } catch (error) { + if (error instanceof HttpError && error.status === 404) continue; + throw error; + } + } + return { nextCursor: rows.length === 100 ? rows[rows.length - 1].id : null }; +} + +export function budgetService(db: Db, _hooks: BudgetServiceHooks = {}) { + const reads = budgetServiceInTransaction(db); + const mutate = (companyId: string, work: (service: ReturnType) => Promise) => + withAccountingTransaction(db, companyId, (tx, publications) => work(budgetServiceInTransaction(tx, publications))); + return { + ...reads, + evaluateAdmission: (companyId: string, agentId: string | null, context?: { issueId?: string | null; projectId?: string | null }) => + mutate(companyId, service => service.evaluateAdmission(companyId, agentId, context)), + evaluateAffectedScopes: (companyId: string, agentId: string | null, projectId: string | null) => + mutate(companyId, service => service.evaluateAffectedScopes(companyId, agentId, projectId)), + reconcileScope: (companyId: string, scopeType: BudgetScopeType, scopeId: string) => + mutate(companyId, service => service.reconcileScope(companyId, scopeType, scopeId)), + reconcilePolicies: () => reconcileDueBudgetPolicies(db, { startup: true }), + upsertPolicy: (companyId: string, input: BudgetPolicyUpsertInput, actorUserId: string | null) => + mutate(companyId, service => service.upsertPolicy(companyId, input, actorUserId)), + evaluateCostEvent: (event: typeof costEvents.$inferSelect) => mutate(event.companyId, service => service.evaluateCostEvent(event)), + getInvocationBlock: (companyId: string, agentId: string | null, context?: { issueId?: string | null; projectId?: string | null }) => + mutate(companyId, service => service.getInvocationBlock(companyId, agentId, context)), + resolveIncident: (companyId: string, incidentId: string, input: BudgetIncidentResolutionInput, actorUserId: string) => + mutate(companyId, service => service.resolveIncident(companyId, incidentId, input, actorUserId)), + }; +} diff --git a/server/src/services/companies.ts b/server/src/services/companies.ts index 0eff1c83613..0292dfa9378 100644 --- a/server/src/services/companies.ts +++ b/server/src/services/companies.ts @@ -1 +1,562 @@ +import { notifyRunDispatch } from "./run-dispatch-notifications.js"; import { notifyStatusCardIssueChange } from "./status-card-notifications.js"; +import { notifyChatInboundWakeNotices } from "./chat-inbound-wake-notice-notifications.js"; +import { notifyCompanyRetentionWork } from "./retention-work-notifications.js"; +import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js"; +import { isRunnerApiResponseAsset } from "./asset-content-access.js"; +import { deleteCompany } from "./company-deletion.js"; +import { fenceActiveCompanyAgentStopsInTransaction } from "./agent-lifecycle.js"; +import { publishAccountingActivities } from "./accounting-transaction.js"; +import { budgetServiceInTransaction, type BudgetServiceHooks } from "./budgets.js"; +import { and, count, eq, gte, inArray, isNull, lt, notInArray, sql } from "drizzle-orm"; +import type { Db } from "@paperclipai/db"; +import { + companies, + companyLogos, + assets, + agents, + agentWakeupRequests, + issues, + heartbeatRuns, + costEvents, +} from "@paperclipai/db"; +import { notFound, unprocessable } from "../errors.js"; +import { isCloudManagedInstance } from "./cloud-instance.js"; +import { notifyCloudOfPrimaryCompanyLifecycleChange } from "./cloud-lifecycle-sync.js"; +import { + MAX_ISSUE_PREFIX_ATTEMPTS, + deriveIssuePrefixBase, + isIssuePrefixConflict, + issuePrefixSuffixForAttempt, + pickAvailableIssuePrefix, + rekeyCompanyIssueIdentifiers, +} from "./issue-prefix.js"; +import { environmentService } from "./environments.js"; +import { logActivity, type ActivityPublication } from "./activity-log.js"; +import { builtInAgentService } from "./built-in-agents.js"; + + +const UUID_RE = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i; +export interface CompanyActivityActor { + actorType: "user" | "agent" | "system" | "plugin"; + actorId: string; + agentId?: string | null; + runId?: string | null; +} + +const SYSTEM_COMPANY_ACTOR: CompanyActivityActor = { + actorType: "system", + actorId: "system", + agentId: null, + runId: null, +}; + +export function companyService(db: Db, _budgetHooks: BudgetServiceHooks = {}) { + const environmentsSvc = environmentService(db); + const builtInAgents = builtInAgentService(db); + + type CompanyTx = Parameters[0]>[0]; + + async function applyArchiveCascadeInTx(tx: CompanyTx, id: string) { + const [agentCount] = await tx.select({ value: count() }).from(agents).where(and(eq(agents.companyId, id), notInArray(agents.status, ["paused", "terminated", "pending_approval"]))); + + const [runCount] = await tx + .select({ value: count() }) + .from(heartbeatRuns) + .where(and( + eq(heartbeatRuns.companyId, id), + inArray(heartbeatRuns.status, ["queued", "running", "scheduled_retry"]), + )); + + await notifyChatInboundWakeNotices(tx, { companyId: id }); + await tx + .update(agentWakeupRequests) + .set({ + status: "cancelled", + error: "Cancelled because the company was archived", + finishedAt: new Date(), + updatedAt: new Date(), + }) + .where(and( + eq(agentWakeupRequests.companyId, id), + inArray(agentWakeupRequests.status, ["queued", "deferred_issue_execution", "claimed"]), + isNull(agentWakeupRequests.runId), + )); + + return { agentsPauseRequested: agentCount.value, runsCancellationRequested: runCount.value }; + } + + async function finalizeArchive( + id: string, + actor: CompanyActivityActor, + cascade: { agentsPauseRequested: number; runsCancellationRequested: number }, + ) { + // The committed company status blocks dispatch immediately. Lifecycle + // projection owns provider termination, including paused agents with work. + await logActivity(db, { + companyId: id, + actorType: actor.actorType, + actorId: actor.actorId, + agentId: actor.agentId ?? null, + runId: actor.runId ?? null, + action: "company.archived", + entityType: "company", + entityId: id, + details: { + agentsPauseRequested: cascade.agentsPauseRequested, + runsCancellationRequested: cascade.runsCancellationRequested, + }, + }); + } + + const companySelection = { + id: companies.id, + name: companies.name, + description: companies.description, + status: companies.status, + issuePrefix: companies.issuePrefix, + issueCounter: companies.issueCounter, + budgetMonthlyCents: companies.budgetMonthlyCents, + spentMonthlyCents: companies.spentMonthlyCents, + defaultResponsibleUserId: companies.defaultResponsibleUserId, + requireBoardApprovalForNewAgents: companies.requireBoardApprovalForNewAgents, + interactionResolverGovernance: companies.interactionResolverGovernance, + feedbackDataSharingEnabled: companies.feedbackDataSharingEnabled, + feedbackDataSharingConsentAt: companies.feedbackDataSharingConsentAt, + feedbackDataSharingConsentByUserId: companies.feedbackDataSharingConsentByUserId, + feedbackDataSharingTermsVersion: companies.feedbackDataSharingTermsVersion, + logoAssetId: companyLogos.assetId, + createdAt: companies.createdAt, + updatedAt: companies.updatedAt, + }; + + function enrichCompany(company: T) { + return { + ...company, + logoUrl: company.logoAssetId ? `/api/assets/${company.logoAssetId}/content` : null, + }; + } + + function currentUtcMonthWindow(now = new Date()) { + const year = now.getUTCFullYear(); + const month = now.getUTCMonth(); + return { + start: new Date(Date.UTC(year, month, 1, 0, 0, 0, 0)), + end: new Date(Date.UTC(year, month + 1, 1, 0, 0, 0, 0)), + }; + } + + async function getMonthlySpendByCompanyIds( + companyIds: string[], + database: Pick = db, + ) { + if (companyIds.length === 0) return new Map(); + const { start, end } = currentUtcMonthWindow(); + const rows = await database + .select({ + companyId: costEvents.companyId, + spentMonthlyCents: sql`coalesce(sum(${costEvents.costCents}), 0)::double precision`, + }) + .from(costEvents) + .where( + and( + inArray(costEvents.companyId, companyIds), + gte(costEvents.occurredAt, start), + lt(costEvents.occurredAt, end), + ), + ) + .groupBy(costEvents.companyId); + return new Map(rows.map((row) => [row.companyId, Number(row.spentMonthlyCents ?? 0)])); + } + + async function hydrateCompanySpend( + rows: T[], + database: Pick = db, + ) { + const spendByCompanyId = await getMonthlySpendByCompanyIds(rows.map((row) => row.id), database); + return rows.map((row) => ({ + ...row, + spentMonthlyCents: spendByCompanyId.get(row.id) ?? 0, + })); + } + + function getCompanyQuery(database: Pick) { + return database + .select(companySelection) + .from(companies) + .leftJoin(companyLogos, eq(companyLogos.companyId, companies.id)); + } + + /** + * Decides whether a rename must move the company onto a new issue prefix, and + * returns the exact prefix pair to re-key. + * + * Self-hosted companies pick their prefix from the name at creation and keep + * it, so a rename leaves the prefix alone. On a hosted/managed instance the + * company is provisioned for the operator, so the name is the only prefix + * source the operator ever chose — a rename re-derives it. Returns null when + * the current prefix is already correct or when the suffix space is + * exhausted. + */ + async function resolveRenamedIssuePrefix( + tx: CompanyTx, + companyId: string, + companyPatch: Partial, + ): Promise<{ fromPrefix: string; toPrefix: string } | null> { + // Only patch and environment facts gate the lock. Every comparison against + // the company's own name or prefix happens below, under the lock. + // An explicit prefix in the patch is the caller's decision; never override it. + if (companyPatch.issuePrefix !== undefined) return null; + const nextName = companyPatch.name; + if (typeof nextName !== "string" || nextName.trim().length === 0) return null; + if (!isCloudManagedInstance()) return null; + + // Lock the company row before comparing anything against it. Two concurrent + // updates would otherwise each decide from the row they read before either + // committed, and both ways of getting that wrong end with a company whose + // prefix disagrees with its own identifiers: + // + // - Two renames: the second re-keys from the prefix it read, finds the + // identifiers the first already moved, and leaves them on the first + // rename's prefix while the row carries the second one's. + // - A rename plus a stale form that resubmits the original name: the + // second sees a name equal to the one it read, skips re-derivation, and + // restores the old name on top of the first rename's prefix. + // + // Reading the row under the lock makes the second transaction decide from + // what the first actually committed. Only a managed instance takes this + // lock, and only for an update that carries a name. + const locked = await tx + .select({ name: companies.name, issuePrefix: companies.issuePrefix }) + .from(companies) + .where(eq(companies.id, companyId)) + .for("update") + .then((rows) => rows[0] ?? null); + if (!locked || nextName === locked.name) return null; + + const nextBase = deriveIssuePrefixBase(nextName); + // A rename that keeps the same base keeps the current prefix, including + // any disambiguating suffix it was allocated. + if (nextBase === deriveIssuePrefixBase(locked.name)) return null; + if (nextBase === locked.issuePrefix) return null; + + const candidate = await pickAvailableIssuePrefix(tx, nextBase); + if (!candidate || candidate === locked.issuePrefix) return null; + return { fromPrefix: locked.issuePrefix, toPrefix: candidate }; + } + + async function createCompanyWithUniquePrefix(data: typeof companies.$inferInsert) { + const base = deriveIssuePrefixBase(data.name); + let suffix = 1; + while (suffix <= MAX_ISSUE_PREFIX_ATTEMPTS) { + const candidate = `${base}${issuePrefixSuffixForAttempt(suffix)}`; + try { + const rows = await db + .insert(companies) + .values({ ...data, issuePrefix: candidate }) + .returning(); + return rows[0]; + } catch (error) { + if (!isIssuePrefixConflict(error)) throw error; + } + suffix += 1; + } + throw new Error("Unable to allocate unique issue prefix"); + } + + return { + list: async () => { + const rows = await getCompanyQuery(db); + const hydrated = await hydrateCompanySpend(rows); + return hydrated.map((row) => enrichCompany(row)); + }, + + getById: async (id: string) => { + // Non-UUID refs previously reached the uuid-typed query and threw a + // DrizzleQueryError ("invalid input syntax for type uuid"), surfacing + // as HTTP 500 from GET /api/companies/:companyId. Treat them as + // not-found so the route returns 404. + if (!UUID_RE.test(id)) return null; + const row = await getCompanyQuery(db) + .where(eq(companies.id, id)) + .then((rows) => rows[0] ?? null); + if (!row) return null; + const [hydrated] = await hydrateCompanySpend([row], db); + return enrichCompany(hydrated); + }, + + create: async (data: typeof companies.$inferInsert) => { + const created = await createCompanyWithUniquePrefix(data); + await environmentsSvc.ensureLocalEnvironment(created.id); + await builtInAgents.autoProvisionBundledAgents(created.id); + const row = await getCompanyQuery(db) + .where(eq(companies.id, created.id)) + .then((rows) => rows[0] ?? null); + if (!row) throw notFound("Company not found after creation"); + const [hydrated] = await hydrateCompanySpend([row], db); + return enrichCompany(hydrated); + }, + + update: async ( + id: string, + data: Partial & { logoAssetId?: string | null }, + actor: CompanyActivityActor = SYSTEM_COMPANY_ACTOR, + ) => { + const budgetPublications: ActivityPublication[] = []; + const result = await db.transaction(async (tx) => { + if (data.status !== undefined || data.budgetMonthlyCents !== undefined) { + await tx.select({ id: companies.id }).from(companies).where(eq(companies.id, id)).for("no key update"); + } + const existing = await getCompanyQuery(tx) + .where(eq(companies.id, id)) + .then((rows) => rows[0] ?? null); + if (!existing) return null; + + const { logoAssetId, ...companyPatch } = data; + const willReactivate = existing.status !== "active" && companyPatch.status === "active"; + const willArchive = existing.status !== "archived" && companyPatch.status === "archived"; + + if (logoAssetId !== undefined && logoAssetId !== null) { + const nextLogoAsset = await tx + .select() + .from(assets) + .where(eq(assets.id, logoAssetId)) + .then((rows) => rows[0] ?? null); + if (!nextLogoAsset) throw notFound("Logo asset not found"); + if (await isRunnerApiResponseAsset(tx, nextLogoAsset)) { + throw unprocessable("API response snapshots cannot be company logos; upload a logo asset"); + } + if (nextLogoAsset.companyId !== existing.id) { + throw unprocessable("Logo asset must belong to the same company"); + } + } + + const renamedPrefix = await resolveRenamedIssuePrefix(tx, id, companyPatch); + + if (companyPatch.status !== undefined && companyPatch.status !== existing.status) { + await notifyRunDispatch(tx, { companyId: id }); + await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers); + await notifyDeliveryWork(tx, DELIVERY_QUEUES.skillSourceSync); + await notifyDeliveryWork(tx, DELIVERY_QUEUES.routineSchedules); + await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors); + await notifyStatusCardIssueChange(tx, id); + await notifyCompanyRetentionWork(tx, id); + await notifyChatInboundWakeNotices(tx, { companyId: id }); + await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentLifecycle, id); + } + const updated = await tx + .update(companies) + .set({ + ...companyPatch, + ...(renamedPrefix ? { issuePrefix: renamedPrefix.toPrefix } : {}), + updatedAt: new Date(), + }) + .where(eq(companies.id, id)) + .returning() + .then((rows) => rows[0] ?? null); + if (!updated) return null; + + let issuePrefixRederived: { + previousIssuePrefix: string; + issuePrefix: string; + issuesRekeyed: number; + casesRekeyed: number; + } | null = null; + if (renamedPrefix) { + const rekeyed = await rekeyCompanyIssueIdentifiers(tx, { + companyId: id, + fromPrefix: renamedPrefix.fromPrefix, + toPrefix: renamedPrefix.toPrefix, + }); + issuePrefixRederived = { + previousIssuePrefix: renamedPrefix.fromPrefix, + issuePrefix: renamedPrefix.toPrefix, + issuesRekeyed: rekeyed.issues, + casesRekeyed: rekeyed.cases, + }; + } + + let agentsRestored = 0; + if (willReactivate) { + const restoredRows = await tx.select({ id: agents.id }).from(agents).where(and(eq(agents.companyId, id), eq(agents.pauseReason, "company_archived"))); + agentsRestored = restoredRows.length; + } + + const archiveCascade = willArchive ? await applyArchiveCascadeInTx(tx, id) : null; + if (companyPatch.status !== existing.status && (companyPatch.status === "paused" || companyPatch.status === "archived")) { + await fenceActiveCompanyAgentStopsInTransaction(tx as unknown as Db, id, companyPatch.status); + } + + if (logoAssetId === null) { + await tx.delete(companyLogos).where(eq(companyLogos.companyId, id)); + } else if (logoAssetId !== undefined) { + await tx + .insert(companyLogos) + .values({ + companyId: id, + assetId: logoAssetId, + }) + .onConflictDoUpdate({ + target: companyLogos.companyId, + set: { + assetId: logoAssetId, + updatedAt: new Date(), + }, + }); + } + + if (logoAssetId !== undefined && existing.logoAssetId && existing.logoAssetId !== logoAssetId) { + await tx.delete(assets).where(eq(assets.id, existing.logoAssetId)); + } + + if (data.budgetMonthlyCents !== undefined) { + await budgetServiceInTransaction(tx as unknown as Db, budgetPublications).upsertPolicy(id, { + scopeType: "company", scopeId: id, amount: data.budgetMonthlyCents, isActive: data.budgetMonthlyCents > 0, windowKind: "calendar_month_utc", + }, actor.actorType === "user" ? actor.actorId : null); + const [budgetUpdated] = await tx.select().from(companies).where(eq(companies.id, id)); + Object.assign(updated, budgetUpdated); + } + const [hydrated] = await hydrateCompanySpend([{ + ...updated, + logoAssetId: logoAssetId === undefined ? existing.logoAssetId : logoAssetId, + }], tx); + + const shouldLogReactivation = willReactivate && + (existing.status === "archived" || agentsRestored > 0); + + return { + company: enrichCompany(hydrated), + reactivated: shouldLogReactivation ? { agentsRestored } : null, + archiveCascade, + unarchived: willReactivate && existing.status === "archived", + issuePrefixRederived, + }; + }); + if (!result) return null; + publishAccountingActivities(id, budgetPublications); + // Post-commit, fire-and-forget, and BEFORE any finalization that + // could throw: a Cloud-pinned primary company that crossed the + // archived boundary (either direction) rings the harness so the + // stack itself can converge. The status transaction has already + // committed, so a later cascade or activity-log failure must not + // leave Cloud unaware of a company that is in fact archived. + if (result.archiveCascade || result.unarchived) { + void notifyCloudOfPrimaryCompanyLifecycleChange(id); + } + if (result.issuePrefixRederived) { + await logActivity(db, { + companyId: id, + actorType: actor.actorType, + actorId: actor.actorId, + agentId: actor.agentId ?? null, + runId: actor.runId ?? null, + action: "company.updated", + entityType: "company", + entityId: id, + details: { + source: "company_rename", + reason: "issue_prefix_rederived", + ...result.issuePrefixRederived, + }, + }); + } + if (result.reactivated) { + await logActivity(db, { + companyId: id, + actorType: actor.actorType, + actorId: actor.actorId, + agentId: actor.agentId ?? null, + runId: actor.runId ?? null, + action: "company.reactivated", + entityType: "company", + entityId: id, + details: { agentsRestored: result.reactivated.agentsRestored }, + }); + } + if (result.archiveCascade) { + await finalizeArchive(id, actor, result.archiveCascade); + } + return result.company; + }, + + archive: async (id: string, actor: CompanyActivityActor = SYSTEM_COMPANY_ACTOR) => { + const result = await db.transaction(async (tx) => { + const existing = await tx + .select({ status: companies.status }) + .from(companies) + .where(eq(companies.id, id)) + .for("no key update") + .then((rows) => rows[0] ?? null); + if (!existing) return null; + + const wasAlreadyArchived = existing.status === "archived"; + + if (!wasAlreadyArchived) { + await notifyRunDispatch(tx, { companyId: id }); + await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers); + await notifyDeliveryWork(tx, DELIVERY_QUEUES.skillSourceSync); + await notifyDeliveryWork(tx, DELIVERY_QUEUES.routineSchedules); + await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors); + await notifyStatusCardIssueChange(tx, id); + await notifyCompanyRetentionWork(tx, id); + await notifyChatInboundWakeNotices(tx, { companyId: id }); + await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentLifecycle, id); + await tx + .update(companies) + .set({ status: "archived", updatedAt: new Date() }) + .where(eq(companies.id, id)); + } + + const cascade = wasAlreadyArchived ? null : await applyArchiveCascadeInTx(tx, id); + if (!wasAlreadyArchived) await fenceActiveCompanyAgentStopsInTransaction(tx as unknown as Db, id, "archived"); + + const row = await getCompanyQuery(tx) + .where(eq(companies.id, id)) + .then((rows) => rows[0] ?? null); + if (!row) return null; + const [hydrated] = await hydrateCompanySpend([row], tx); + return { + company: enrichCompany(hydrated), + cascade, + }; + }); + if (!result) return null; + + // Same doorbell rule as update(): the archive is committed, so ring + // before finalization, which can throw without undoing it. + if (result.cascade) { + void notifyCloudOfPrimaryCompanyLifecycleChange(id); + await finalizeArchive(id, actor, result.cascade); + } + + return result.company; + }, + + remove: (id: string) => deleteCompany(db, id), + + stats: () => + Promise.all([ + db + .select({ companyId: agents.companyId, count: count() }) + .from(agents) + .groupBy(agents.companyId), + db + .select({ companyId: issues.companyId, count: count() }) + .from(issues) + .groupBy(issues.companyId), + ]).then(([agentRows, issueRows]) => { + const result: Record = {}; + for (const row of agentRows) { + result[row.companyId] = { agentCount: row.count, issueCount: 0 }; + } + for (const row of issueRows) { + if (result[row.companyId]) { + result[row.companyId].issueCount = row.count; + } else { + result[row.companyId] = { agentCount: 0, issueCount: row.count }; + } + } + return result; + }), + }; +} diff --git a/server/src/services/delivery-work-notifications.ts b/server/src/services/delivery-work-notifications.ts index 6b39445d0ba..c1593839a35 100644 --- a/server/src/services/delivery-work-notifications.ts +++ b/server/src/services/delivery-work-notifications.ts @@ -2,4 +2,136 @@ import { signalDatabaseWork, subscribeDatabaseWork } from "@paperclipai/db"; /** Topics for existing durable queues; register intent before their writes. */ export const DELIVERY_QUEUES = { + scheduledRetryDispatch: "scheduled-retry-dispatch", + queuedRunDispatch: "queued-run-dispatch", statusCardEvaluation: "status-card-evaluation", + runEvidenceMigration: "run-evidence-migration", + providerTraceRetention: "provider-trace-retention", + nativeRunnerEvidenceRetention: "native-runner-evidence-retention", + skillVersionRetention: "skill-version-retention", + databaseBackups: "database-backups", + importSpoolCleanup: "import-spool-cleanup", + externalConversationSettlement: "external-conversation-settlement", + nativeRecoveryNotices: "native-recovery-notices", + nativeEnvironmentSettlement: "native-environment-settlement", + nativeRunProjection: "native-run-projection", + nativeResultCommit: "native-result-commit", + nativeWorkspaceFinalization: "native-workspace-finalization", + retainedNativeSessionCleanup: "retained-native-session-cleanup", + nativeStatusReassessment: "native-status-reassessment", + nativeReviewResponse: "native-review-response", + nativeResponsePresentation: "native-response-presentation", + nativePendingEffects: "native-pending-effects", + unsafeWorkspaceExportRecovery: "unsafe-workspace-export-recovery", + automaticCompletionReviews: "automatic-completion-reviews", + obsoletePolicyReviews: "obsolete-policy-reviews", + cancelledWorkspacePreservation: "cancelled-workspace-preservation", + instructionCollection: "instruction-collection", + directoryCleanup: "directory-cleanup", + instructionCommit: "instruction-commit", + scheduledRetryCancellation: "scheduled-retry-cancellation", + executionFinalization: "execution-finalization", + workspaceRecovery: "workspace-recovery", + executionReplacement: "execution-replacement", + executionContinuation: "execution-continuation", + executionStatus: "execution-status", + executionDisposition: "execution-disposition", + environmentCleanup: "environment-cleanup", + approvalQueueSeeding: "approval-queue-seeding", + approvalRetention: "approval-retention", + joinRequestRetention: "join-request-retention", + decisionTargets: "decision-targets", + decisionEffectRecovery: "decision-effect-recovery", + decisionExpiry: "decision-expiry", + decisionContinuation: "decision-continuation", + + decisionArchiveDelivery: "decision-archive-delivery", + toolConnectionHealth: "tool-connection-health", + toolReviewProjection: "tool-review-projection", + toolReviewInterruption: "tool-review-interruption", + toolReviewRecovery: "tool-review-recovery", + toolReviewExpiry: "tool-review-expiry", + gatewayCredentialExpiry: "gateway-credential-expiry", + secretProposalExpiry: "secret-proposal-expiry", + customImageExpiry: "custom-image-expiry", + historicalCleanup: "historical-cleanup", + adapterLoginCleanup: "adapter-login-cleanup", + setupTokenCleanup: "setup-token-cleanup", + localLoginCleanup: "local-login-cleanup", + budgetEnforcement: "budget-enforcement", + accounting: "cost-accounting", + taskDrainExpiry: "task-drain-expiry", + agentLifecycle: "agent-lifecycle", + chatBoardMessages: "chat-board-messages", + chatTaskStarts: "chat-task-starts", + chatReceiptReactions: "chat-receipt-reactions", + chatSessionStops: "chat-session-stops", + chatProviderEffects: "chat-provider-effects", + chatGitHubWebhookRecovery: "chat-github-webhook-recovery", + chatGitHubIngress: "chat-github-ingress", + chatGitHubPublications: "chat-github-publications", + chatGitHubReviewProjection: "chat-github-review-projection", + chatGitHubReviewChecks: "chat-github-review-checks", + chatGitHubTerminalResponses: "chat-github-terminal-responses", + chatGitHubAcknowledgements: "chat-github-acknowledgements", + chatFailedRunRetries: "chat-failed-run-retries", + chatTelegramMaintenance: "chat-telegram-maintenance", + chatReceivers: "chat-receivers", + voiceSessionRecovery: "voice-session-recovery", + voiceReports: "voice-call-reports", + voiceInboundRetention: "voice-inbound-retention", + voiceInboundCleanup: "voice-inbound-cleanup", + voiceReplies: "voice-reply-delivery", + chatTeamsConsentExpiry: "chat-teams-consent-expiry", + chatTeamsAttemptRecovery: "chat-teams-attempt-recovery", + chatCommandRegistration: "chat-command-registration", + chatTelegramSubscriptions: "chat-telegram-subscriptions", + chatSessionSyncs: "chat-session-syncs", + chatVerification: "chat-verification", + chatInbound: "chat-inbound", + chatPublications: "chat-publications", + chatNativeProgress: "chat-native-progress", + chatInboundWakeNotices: "chat-inbound-wake-notices", + chatInteractionContinuations: "chat-interaction-continuations", + chatTerminalConfirmations: "chat-terminal-confirmations", + chatFailedRetryNotices: "chat-failed-retry-notices", + chatRunMilestones: "chat-run-milestones", + chatReceipts: "chat-slack-receipts", + browser: "browser-use", + email: "email-channels", + museMaintenance: "muse-maintenance", + mcpEvents: "mcp-events", + dotEvents: "dot-events", + grokDispatch: "grok-dispatch", + skillSourceSync: "skill-source-sync", + skillPackages: "skill-packages", + runtimeAssetRetention: "runtime-asset-retention", + fastResponse: "fast-responses", + feedback: "feedback-exports", + chatCompletion: "chat-completions", + connection: "connection-continuations", + gitRestorePoints: "git-restore-points", + nativeWorkspaceSyncRetention: "native-workspace-sync-retention", + question: "question-responses", + pluginSchedules: "plugin-schedules", + pluginLifecycle: "plugin-resource-lifecycle", + toolAction: "tool-action-receipts", + terminalWorkspaces: "terminal-workspaces", + routineSchedules: "routine-schedules", + issueMonitors: "issue-monitors", + agentTimers: "agent-timers", + caseHandoffs: "case-handoffs", + nativeCheckpointExpiration: "native-checkpoint-expiration", +} as const; +export type DeliveryQueue = typeof DELIVERY_QUEUES[keyof typeof DELIVERY_QUEUES]; + +export async function notifyDeliveryWork(transaction: object, queue: DeliveryQueue, hint?: string): Promise { + await signalDatabaseWork(transaction, queue, hint); +} +// Public delivery notification observers only see settled work. The coordinator +// uses the lower-level lifecycle subscription to also fence in-flight writes. +export function subscribeDeliveryWork(owner: object, queue: DeliveryQueue, wake: () => void): () => void { + return subscribeDatabaseWork(owner, queue, event => { + if (event === "settled") wake(); + }); +} diff --git a/server/src/services/execution-control-notifications.ts b/server/src/services/execution-control-notifications.ts index 01ef4cf933e..84fc80d9ea7 100644 --- a/server/src/services/execution-control-notifications.ts +++ b/server/src/services/execution-control-notifications.ts @@ -1 +1,32 @@ +import { notifyRunDispatch } from "./run-dispatch-notifications.js"; +import { notifyStatusCardIssueChange } from "./status-card-notifications.js"; +import type { Db } from "@paperclipai/db"; +import { DELIVERY_QUEUES, notifyDeliveryWork, type DeliveryQueue } from "./delivery-work-notifications.js"; + +export type ExecutionWorkScope = { companyId: string; agentId?: string; runId?: string; issueId?: string }; + +export async function notifyExecutionRunWork(transaction: object, scope?: ExecutionWorkScope) { + if (scope) { + await notifyRunDispatch(transaction, scope); + await notifyStatusCardIssueChange(transaction, scope.companyId); + } + for (const queue of [DELIVERY_QUEUES.executionStatus, DELIVERY_QUEUES.executionFinalization, + DELIVERY_QUEUES.executionReplacement, DELIVERY_QUEUES.executionDisposition, DELIVERY_QUEUES.issueMonitors]) { + await notifyDeliveryWork(transaction, queue); + } +} + +export function writeExecutionControl( + db: Db | Parameters[0]>[0], + queue: DeliveryQueue | readonly DeliveryQueue[], + write: (tx: Db) => PromiseLike, + scope: ExecutionWorkScope | undefined, +): Promise { + return db.transaction(async transaction => { + await notifyRunDispatch(transaction, scope); if (scope) await notifyStatusCardIssueChange(transaction, scope.companyId); + await notifyDeliveryWork(transaction, DELIVERY_QUEUES.issueMonitors); + for (const topic of typeof queue === "string" ? [queue] : queue) await notifyDeliveryWork(transaction, topic); + return write(transaction as unknown as Db); + }); +} diff --git a/server/src/services/heartbeat/queue.ts b/server/src/services/heartbeat/queue.ts index 0e6a67a4dcb..16207b0fb28 100644 --- a/server/src/services/heartbeat/queue.ts +++ b/server/src/services/heartbeat/queue.ts @@ -1 +1,5084 @@ +import { notifyRunDispatch } from "../run-dispatch-notifications.js"; import { notifyStatusCardIssueChange } from "../status-card-notifications.js"; +import { DELIVERY_QUEUES, notifyDeliveryWork } from "../delivery-work-notifications.js"; +import { notifyExternalAgentSettlement } from "../../modules/external-agents/index.js"; +import { notifyNativeStatusReassessment } from "../native-status-reassessment-notifications.js"; +import { notifyUnsafeWorkspaceExportRecovery, notifyUnsafeWorkspaceExportRunChange } from "../unsafe-workspace-export-notifications.js"; +import { notifyCancelledWorkspacePreservation } from "../cancelled-workspace-preservation-notifications.js"; +import { notifyInstructionCollection } from "../instruction-collection-notifications.js"; +import { notifyScheduledRetryCancellation } from "../scheduled-retry-cancellation-notifications.js"; +import { notifyChatResponseAvailability } from "../chat-response-availability.js"; +import { notifyChatInboundWakeNotices } from "../chat-inbound-wake-notice-notifications.js"; +import { notifyChatRunMilestones } from "../chat-run-milestone-notifications.js"; +import { assertTaskRetryConversationAllowed, readTaskRetryIntent, readTaskRetryBoardMessageOrigin, retireSupersededTaskRetry, taskRetryMutationAllowed } from "../task-retry.js"; +import { insertIssueComment } from "../issue-comment-writes.js"; +import { withExternalAdmissionGuard, assertNoExternalOverlap } from "../../modules/external-agents/index.js"; +import { isAgentAwaitingSetup } from "../../modules/agent-lifecycle/index.js"; +import { normalizeAgentNameKey } from "./retries.js"; +import { publishActiveDotComment } from "../dot-assignment-follow-up.js"; +import { publishActiveMuseComment } from "../muse-assignment-follow-up.js"; +import { + type WakeupOptions, + mergeCoalescedContextSnapshot, + enrichWakeContextSnapshot, + hasInteractionContinuationWakeContext, + isInteractionResolutionWakePayload, +} from "./run-preparation.js"; +import { resolveExecutionWorkspaceReuseRequestForIssue } from "./workspaces.js"; +import { + CHAT_COMPLETION_WAKE_REASON, + isCompletedOnboardingHandoffWake, +} from "../chat-completion-delivery.js"; +import { + compareCents, + hasWorkspaceRestoreFailure, +} from "@paperclipai/shared"; +import { nonIdleSlackIssueCondition } from "../slack-conversation-state.js"; +import { readQueuedInteractionResponse } from "../queued-interaction-response.js"; +import { + isConversation, + isConversationReset, + isConversationExecutionWake, + isWaitingConversation, +} from "../agent-conversations.js"; +import { legacyControllerClaim } from "../legacy-controller-lease.js"; +import { + admitExplicitNativeContinuation, + undeliveredLegacyUserCommentIds, +} from "../explicit-native-continuation.js"; +import { canRetryStoppedRun } from "../cancelled-native-startup.js"; +import { getExecutionBlocker, hasPendingRunCleanup } from "../execution-blocker.js"; +import { validateNativeWorkspaceReconstruction } from "../native-runtime/native-workspace-reconstruction.js"; +import { isConversationAdapter } from "../conversation-continuation.js"; +import { recordExecutionWait } from "../execution-wait.js"; +import { + getNativeReviewAssignment, + readNativeReviewAssignmentContext, +} from "../native-runtime/native-review-participant.js"; +import { claimQueuedNativeReviewRun } from "../native-runtime/native-review-dispatch.js"; +import { adapterExecutionControls } from "../adapter-execution-control.js"; +import { executionFailureRetryCount } from "../execution-recovery-attempt.js"; +import { + slackMentionAllowsRunStart, + resumeDeferredChatWake, + assertDurableChatWakeupReceipt, + assertDurableChatWakeupRequest, + withDurableChatAdmission, + authorizeFailedChatRunRetryWake, + FailedChatRunRetryAuthorizationError, + unadmittedChatWakeupCondition, +} from "../durable-chat-wakeup.js"; +import { agentService } from "../agents.js"; +import { normalizeLegacyRunnerProvider } from "@paperclipai/adapter-utils"; +import { randomUUID } from "node:crypto"; +import { + and, + asc, + desc, + eq, + gt, + gte, + inArray, + isNull, + lt, + lte, + ne, + notInArray, + or, + sql, +} from "drizzle-orm"; +import type { Db } from "@paperclipai/db"; +import { AGENT_DEFAULT_MAX_CONCURRENT_RUNS } from "@paperclipai/shared"; +import { + agents, + caseHandoffs, + cases, + agentWakeupRequests, + activityLog, + chatConversations, + chatEndpoints, + companies, + costEvents, + executionWorkspaces, + heartbeatRunEvents, + heartbeatRuns, + issueComments, + issueRecoveryActions, + issueRelations, + issues, +} from "@paperclipai/db"; +import { + conflict, + HttpError, + notFound, +} from "../../errors.js"; +import { logger } from "../../middleware/logger.js"; +import { publishLiveEvent } from "../live-events.js"; +import { allocateHeartbeatRunEventSeq } from "../heartbeat-run-events.js"; +import { + queuedCommentIdsFromWakePayload, + queuedCommentIdsFromRunContext, + withQueuedCommentIdsInWakePayload, + withQueuedCommentIdsInRunContext, +} from "../issue-queued-comment-queue.js"; +import { + parseObject, + asBoolean, + asNumber, +} from "../../adapters/utils.js"; +import { isExternalChatWaitAuthorizationContention } from "../native-runtime/chat-attachment-reuse.js"; +import { emitAgentTaskRun } from "../agent-task-run-telemetry.js"; +import { + CHAT_CONTROL_RECOVERY_ADMISSION_KEY, + CHAT_CONTROL_RECOVERY_STOP_CODE, + CHAT_CONTROL_RECOVERY_UNRESOLVED_CODE, + chatControlRecoveryAdmission, + readChatControlRecoveryAdmission, + readChatControlNativeParent, + readChatControlRecoveryStop, +} from "../chat-control-recovery-stop.js"; +import { + ISSUE_NEW_INPUT_ACTIVITY_ACTIONS, + ISSUE_PROGRESS_ACTIVITY_ACTIONS, + ISSUE_REWAKE_LOOKBACK_MS, + ISSUE_REWAKE_RUN_SAMPLE_LIMIT, + evaluateIssueRewakeThrottle, + isThrottleCandidateIssueRewake, +} from "../issue-rewake-throttle.js"; +import { logActivity } from "../activity-log.js"; +import { ISSUE_BLOCKERS_RESOLVED_WAKE_REASON } from "../issue-dependency-wakeups.js"; +import { + ISSUE_TREE_CONTROL_INTERACTION_WAKE_REASONS, + isVerifiedIssueTreeControlInteractionWake, +} from "../issue-tree-control.js"; +import { + buildExecutionWorkspaceAdapterConfig, + isUnrunnableWorktreeCombo, + parseIssueExecutionWorkspaceSettings, + resolveEffectiveWorkspaceStrategyType, + resolveExecutionWorkspaceMode, + WORKSPACE_WORKTREE_REQUIRES_PROJECT_CODE, + WORKSPACE_WORKTREE_REQUIRES_PROJECT_MESSAGE, + WORKSPACE_WORKTREE_REQUIRES_PROJECT_REMEDIATION, +} from "../execution-workspace-policy.js"; +import { readContinuationAttempt } from "../recovery/index.js"; +import { allowsIssueInteractionWake, wakeChatConversationsConflict } from "../../modules/run-dispatch/index.js"; +import { withAgentStartLock } from "../agent-start-lock.js"; +import { + evaluateAgentInvokability, + shouldCancelRunsForNonInvokableAgent, + type AgentOrgRow, +} from "../agent-invokability.js"; +import { isHeartbeatWakeOnDemandEnabled, supportsUnscopedTimerHeartbeat } from "../heartbeat-policy.js"; +import { isUuidLike } from "@paperclipai/shared"; +import { retryChatControlAdmission } from "../chat-control-admission-retry.js"; + +import type { HeartbeatServiceOptions } from "../heartbeat.js"; +import type { createHeartbeatRunState } from "./run-state.js"; +import type { createHeartbeatRunPreparation } from "./run-preparation.js"; +import type { HeartbeatRetryDependencies } from "./retries.js"; +import type { HeartbeatRecoveryDependencies, createHeartbeatRecovery } from "./recovery.js"; +import type { issueService } from "../issues.js"; +import type { issueTreeControlService } from "../issue-tree-control.js"; +import type { budgetService } from "../budgets.js"; +import type { instanceSettingsService } from "../instance-settings.js"; +import type { createRunDispatch } from "../../modules/run-dispatch/index.js"; +import type { PostCommitEffect as WakeQueuePostCommitEffect, createWakeQueue } from "../../modules/wake-queue/index.js"; + +type HeartbeatRun = typeof heartbeatRuns.$inferSelect; +type HeartbeatRunState = ReturnType; +type RunPreparation = ReturnType; + +/** Queue policy uses the service's lifecycle effects and process-wide ownership sets. */ +export interface HeartbeatQueueDependencies extends Pick, + Pick, + Pick, + Pick { + sweepPendingCleanupLeases: ReturnType["sweepPendingCleanupLeases"]; + options: Pick; + issuesSvc: Pick, "getDependencyReadiness" | "listDependencyReadiness">; + treeControlSvc: Pick, "getActivePauseHoldGate">; + budgets: Pick, "getInvocationBlock">; + instanceSettings: Pick, "getExperimental">; + runDispatch: Pick, "cancelStaleQueuedRun">; + wakeQueue: Pick, "createAdmissionTransactionScope" | "admitWakeBehindIssueExecution">; + activeRunExecutions: Set; + activeRunExecutionPromises: Set>; + activeWakeupPromises: Set>; + liveRunExecutions: { has(id: string): boolean }; + isHeartbeatRunTerminalStatus: (status: string | null | undefined) => boolean; + isSameTaskScope: (left: string | null, right: string | null) => boolean; + runTaskKey: (run: HeartbeatRun) => string | null; + filterZombieCoalesceTarget: (run: T | null, live: { has(id: string): boolean }) => T | null; + getSchedulingSuppression: () => Promise<{ suppressed: boolean; reason: "worktree_instance" | "database_restore_in_progress" | "task_drain" | null }>; + canStartNewExternalBotWork: (agent: typeof agents.$inferSelect) => Promise; + executeRun: (runId: string) => Promise; + releaseIssueExecutionAndPromote: (run: Pick, options?: { suppressImmediateRecovery?: boolean; resumeStrandedQueue?: boolean; deferredPostCommitEffects?: WakeQueuePostCommitEffect[] }) => Promise; + applyWakeQueuePostCommitEffects: (effects: WakeQueuePostCommitEffect[]) => Promise; + publishRunLifecyclePluginEvent: (run: HeartbeatRun) => void; + cancelRunInternal: (runId: string, reason?: string, options?: { errorCode?: string }) => Promise; + cancelActiveForAgentInternal: (agentId: string, reason: string) => Promise; + resumeExecutionWaitComments: () => Promise; + resumeQueuedCommentInterrupt: (companyId: string, queueId: string) => Promise; + resumeSavedLegacyComments: (companyId: string, queueId: string) => Promise; + formatIssueIdentifierLink: (identifier: string | null, fallback: string) => string; +} + +const HEARTBEAT_MAX_CONCURRENT_RUNS_DEFAULT = AGENT_DEFAULT_MAX_CONCURRENT_RUNS; + +const HEARTBEAT_MAX_CONCURRENT_RUNS_MIN = 1; + +const HEARTBEAT_MAX_CONCURRENT_RUNS_MAX = 50; + +export const DEFERRED_WAKE_CONTEXT_KEY = "_paperclipWakeContext"; + +export const EXECUTION_PATH_HEARTBEAT_RUN_STATUSES = [ + "queued", + "running", + "scheduled_retry", +] as const; + +const TIMER_ACTIONABLE_ISSUE_STATUSES = ["todo", "in_progress"] as const; + +export class ChatControlRecoveryUnresolvedError extends Error { + constructor() { + super( + "Run admission could not acquire its database locks after bounded retries. No provider work started. Review database contention and send a fresh request; this attempt will not automatically retry.", + ); + } +} + +const RUNNING_ISSUE_WAKE_REASONS_REQUIRING_FOLLOWUP = new Set([ + CHAT_COMPLETION_WAKE_REASON, + "approval_approved", + ISSUE_BLOCKERS_RESOLVED_WAKE_REASON, + "issue_recovery_action_restored", +]); + +function normalizeMaxConcurrentRuns(value: unknown) { + const parsed = Math.floor( + asNumber(value, HEARTBEAT_MAX_CONCURRENT_RUNS_DEFAULT), + ); + if (!Number.isFinite(parsed)) return HEARTBEAT_MAX_CONCURRENT_RUNS_DEFAULT; + return Math.max( + HEARTBEAT_MAX_CONCURRENT_RUNS_MIN, + Math.min(HEARTBEAT_MAX_CONCURRENT_RUNS_MAX, parsed), + ); +} + +export function shouldRequireIssueCommentForWake( + contextSnapshot: Record | null | undefined, +) { + if (contextSnapshot?.skipIssueComment === true) return false; + + const wakeReason = readNonEmptyString(contextSnapshot?.wakeReason); + return ( + wakeReason === "issue_assigned" || + wakeReason === "execution_review_requested" || + wakeReason === "execution_approval_requested" || + wakeReason === "execution_changes_requested" + ); +} + +async function listUnresolvedBlockerSummaries( + dbOrTx: Pick, + companyId: string, + issueId: string, + unresolvedBlockerIssueIds: string[], +) { + const ids = [...new Set(unresolvedBlockerIssueIds.filter(Boolean))]; + if (ids.length === 0) return []; + return dbOrTx + .select({ + id: issues.id, + identifier: issues.identifier, + title: issues.title, + status: issues.status, + priority: issues.priority, + assigneeAgentId: issues.assigneeAgentId, + assigneeUserId: issues.assigneeUserId, + }) + .from(issueRelations) + .innerJoin(issues, eq(issueRelations.issueId, issues.id)) + .where( + and( + eq(issueRelations.companyId, companyId), + eq(issueRelations.type, "blocks"), + eq(issueRelations.relatedIssueId, issueId), + inArray(issues.id, ids), + ), + ) + .orderBy(asc(issues.title)); +} + +export function shouldDeferFollowupWakeForSameIssue(input: { + activeRunStatus: string | null | undefined; + isSameExecutionAgent: boolean; + wakeCommentId: string | null | undefined; + forceFreshSession: boolean; +}) { + // A comment follow-up or explicit fresh-session wake needs a new run boundary. + if (!input.isSameExecutionAgent) return false; + if (input.activeRunStatus !== "running") return false; + if (input.wakeCommentId) return true; + if (input.forceFreshSession) return true; + return false; +} + +export function shouldQueueFollowupForRunningIssueWake(input: { + contextSnapshot: Record | null | undefined; + wakeCommentId: string | null; +}) { + if (input.wakeCommentId) return true; + // A structured interaction response is new input just like a comment. It + // must run after the turn that created the interaction instead of being + // merged into that still-running turn. + if ( + readNonEmptyString(input.contextSnapshot?.interactionId) && + readNonEmptyString(input.contextSnapshot?.interactionStatus) + ) { + return true; + } + const wakeReason = readNonEmptyString(input.contextSnapshot?.wakeReason); + if (wakeReason === "issue_children_completed" && ( + input.contextSnapshot?.onboardingCompletion === true || + (input.contextSnapshot?.statusDecisionSource === "native_status_decision" && + readNonEmptyString(input.contextSnapshot?.nativeChildCompletionDecisionId)) + )) return true; + return Boolean( + wakeReason && RUNNING_ISSUE_WAKE_REASONS_REQUIRING_FOLLOWUP.has(wakeReason), + ); +} + +function readNonEmptyString(value: unknown): string | null { + return typeof value === "string" && value.trim().length > 0 ? value : null; +} + +/** Queue admission and dispatch share the service lifecycle callbacks and execution ownership. */ +export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDependencies) { + const { + setRunStatus, + setWakeupStatus, + appendRunEvent, + releaseIssueExecutionAndPromote, + applyWakeQueuePostCommitEffects, + issuesSvc, + options, + publishRunLifecyclePluginEvent, + finalizeAgentStatus, + getAgent, + cancelRunInternal, + toAgentOrgRow, + getAgentInvokability, + budgets, + activeRunExecutions, + treeControlSvc, + runDispatch, + applyRunDispatchPostCommitEffects, + resolveResponsibleUserIdForRun, + getIssueExecutionContext, + isHeartbeatRunTerminalStatus, + liveRunExecutions, + getSchedulingSuppression, + resumeExecutionWaitComments, + getWorktreeExecutionCutoff, + resumeQueuedCommentInterrupt, + resumeSavedLegacyComments, + cancelActiveForAgentInternal, + listCompanyAgentOrgRows, + executeRun, + canStartNewExternalBotWork, + activeRunExecutionPromises, + activeWakeupPromises, + instanceSettings, + getRun, + sweepPendingCleanupLeases, + resolveExplicitResumeSessionOverride, + resolveSessionBeforeForWakeup, + hasResolvablePriorSessionWorkspaceForWake, + getRoutineEnvForExecutionIssue, + resolveResponsibleUserIdForRunSeed, + formatIssueIdentifierLink, + wakeQueue, + isSameTaskScope, + runTaskKey, + filterZombieCoalesceTarget, + } = dependencies; + + function parseHeartbeatPolicy(agent: typeof agents.$inferSelect) { + const runtimeConfig = parseObject(agent.runtimeConfig); + const heartbeat = parseObject(runtimeConfig.heartbeat); + + return { + enabled: asBoolean(heartbeat.enabled, false), + intervalSec: Math.max(0, asNumber(heartbeat.intervalSec, 0)), + wakeOnDemand: isHeartbeatWakeOnDemandEnabled(agent), + // A Dot binding has one external turn. Competing assignments must retain + // their queue position instead of claiming a second run that cannot bind. + maxConcurrentRuns: agent.adapterType === "paperclip_runner" && + ["openai_dot", "grok_bot", "muse"].includes(String(parseObject(agent.adapterConfig).provider)) + ? 1 : normalizeMaxConcurrentRuns(heartbeat.maxConcurrentRuns), + skipTimerWhenNoActionableWork: asBoolean( + heartbeat.skipTimerWhenNoActionableWork ?? + heartbeat.requireActionableTimerWork ?? + heartbeat.issueOnlyTimer, + false, + ), + maxDailyRuns: normalizeOptionalNonNegativeInteger( + heartbeat.maxDailyRuns ?? + heartbeat.dailyRunLimit ?? + heartbeat.dailyRunCap ?? + heartbeat.maxRunsPerDay, + ), + maxDailyCostCents: normalizeOptionalNonNegativeInteger( + heartbeat.maxDailyCostCents ?? + heartbeat.dailyCostCentsLimit ?? + heartbeat.dailySpendCentsLimit ?? + heartbeat.dailyBudgetCents, + ), + }; + } + + function normalizeOptionalNonNegativeInteger(value: unknown) { + if (value === null || value === undefined || value === "") return null; + const normalized = Math.floor(asNumber(value, 0)); + return normalized >= 0 ? normalized : null; + } + + function currentUtcDayWindow(now = new Date()) { + const start = new Date( + Date.UTC( + now.getUTCFullYear(), + now.getUTCMonth(), + now.getUTCDate(), + 0, + 0, + 0, + 0, + ), + ); + const end = new Date( + Date.UTC( + now.getUTCFullYear(), + now.getUTCMonth(), + now.getUTCDate() + 1, + 0, + 0, + 0, + 0, + ), + ); + return { start, end }; + } + + async function getHeartbeatDailyCapBlock( + agent: typeof agents.$inferSelect, + policy: ReturnType, + options: { + checkRunCap?: boolean; + checkCostCap?: boolean; + excludeRunId?: string | null; + now?: Date; + } = {}, + client: Pick = db, + ) { + const checkRunCap = options.checkRunCap ?? true; + const checkCostCap = options.checkCostCap ?? true; + const { start, end } = currentUtcDayWindow(options.now); + if (checkRunCap && policy.maxDailyRuns !== null) { + const conditions = [ + eq(heartbeatRuns.companyId, agent.companyId), + eq(heartbeatRuns.agentId, agent.id), + gte(heartbeatRuns.startedAt, start), + lt(heartbeatRuns.startedAt, end), + notInArray(heartbeatRuns.status, ["queued", "scheduled_retry"]), + ]; + if (options.excludeRunId) { + conditions.push(sql`${heartbeatRuns.id} <> ${options.excludeRunId}`); + } + const [row] = await client + .select({ total: sql`count(*)::integer` }) + .from(heartbeatRuns) + .where(and(...conditions)); + const observed = Number(row?.total ?? 0); + if (observed >= policy.maxDailyRuns) { + return { + reason: "heartbeat.daily_run_limit", + observed, + limit: policy.maxDailyRuns, + }; + } + } + + if (checkCostCap && policy.maxDailyCostCents !== null) { + const [row] = await client + .select({ + total: sql`coalesce(sum(${costEvents.costCents}), 0)::text`, + }) + .from(costEvents) + .where( + and( + eq(costEvents.companyId, agent.companyId), + eq(costEvents.agentId, agent.id), + gte(costEvents.occurredAt, start), + lt(costEvents.occurredAt, end), + ), + ); + const observed = Number(row?.total ?? 0); + if (compareCents(String(row?.total ?? 0), policy.maxDailyCostCents) >= 0) { + return { + reason: "heartbeat.daily_cost_limit", + observed, + limit: policy.maxDailyCostCents, + }; + } + } + + return null; + } + + async function cancelQueuedRunForHeartbeatDailyCap( + run: typeof heartbeatRuns.$inferSelect, + dailyCapBlock: NonNullable< + Awaited> + >, + ) { + const now = new Date(); + const reason = + "Cancelled because the agent reached a per-day heartbeat budget cap before adapter invocation"; + const cancelled = await setRunStatus(run.id, "cancelled", { + finishedAt: now, + error: reason, + errorCode: dailyCapBlock.reason, + resultJson: { + ...parseObject(run.resultJson), + stopReason: dailyCapBlock.reason, + observed: dailyCapBlock.observed, + limit: dailyCapBlock.limit, + effectiveTimeoutSec: 0, + timeoutConfigured: false, + timeoutSource: "heartbeat_daily_cap_gate", + timeoutFired: false, + }, + }); + if (!cancelled) return null; + + await setWakeupStatus(run.wakeupRequestId, "skipped", { + finishedAt: now, + error: reason, + }); + + await appendRunEvent(cancelled, { + eventType: "lifecycle", + stream: "system", + level: "warn", + message: reason, + payload: { + reason: dailyCapBlock.reason, + observed: dailyCapBlock.observed, + limit: dailyCapBlock.limit, + }, + }); + + await releaseIssueExecutionAndPromote(cancelled, { + suppressImmediateRecovery: true, + }); + + return cancelled; + } + + async function hasActionableTimerWork(agent: typeof agents.$inferSelect) { + const row = await db + .select({ id: issues.id }) + .from(issues) + .where( + and( + eq(issues.companyId, agent.companyId), + eq(issues.assigneeAgentId, agent.id), + isNull(issues.assigneeUserId), + isNull(issues.hiddenAt), + inArray(issues.status, [...TIMER_ACTIONABLE_ISSUE_STATUSES]), + isNull(issues.conversationAgentId), + nonIdleSlackIssueCondition(), + ), + ) + .limit(1) + .then((rows) => rows[0] ?? null); + return Boolean(row); + } + + async function nextTimerHeartbeatAt(agent: typeof agents.$inferSelect, now = new Date()): Promise { + const policy = parseHeartbeatPolicy(agent); + if (!policy.enabled || policy.intervalSec <= 0) return null; + + if (policy.maxDailyRuns === 0 || policy.maxDailyCostCents === 0) return null; + if (policy.skipTimerWhenNoActionableWork && !(await hasActionableTimerWork(agent))) return null; + const dueAt = new Date(agent.lastHeartbeatAt ?? agent.createdAt).getTime() + policy.intervalSec * 1000; + const cap = await getHeartbeatDailyCapBlock(agent, policy, { now }); + if (!cap) return dueAt; + return Math.max(dueAt, currentUtcDayWindow(now).end.getTime()); + } + + async function markTimerHeartbeatChecked( + agentId: string, + source: WakeupOptions["source"], + ) { + if (source !== "timer") return; + await db.transaction(async tx => { + await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers); + const now = new Date(); + await tx.update(agents).set({ lastHeartbeatAt: now, updatedAt: now }) + .where(eq(agents.id, agentId)); + }); + } + + async function claimDueTimerHeartbeat( + agent: typeof agents.$inferSelect, + now: Date, + intervalSec: number, + ) { + const dueBefore = new Date(now.getTime() - intervalSec * 1000); + const claimed = await db.transaction(async tx => { + await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers); + return tx + .update(agents) + .set({ lastHeartbeatAt: now, updatedAt: now }) + .where(and( + eq(agents.id, agent.id), + eq(agents.companyId, agent.companyId), + or( + lte(agents.lastHeartbeatAt, dueBefore), + and(isNull(agents.lastHeartbeatAt), lte(agents.createdAt, dueBefore)), + ), + )) + .returning({ id: agents.id }) + .then((rows) => rows[0] ?? null); + }); + if (!claimed) return null; + return { wasFirstHeartbeat: !agent.lastHeartbeatAt }; + } + + function issueRunPriorityRank(priority: string | null | undefined) { + switch (priority) { + case "critical": + return 0; + case "high": + return 1; + case "medium": + return 2; + case "low": + return 3; + default: + return 4; + } + } + + async function listQueuedRunDependencyReadiness( + companyId: string, + queuedRuns: Array, + ) { + const issueIds = [ + ...new Set( + queuedRuns + .map((run) => + readNonEmptyString(parseObject(run.contextSnapshot).issueId), + ) + .filter((issueId): issueId is string => Boolean(issueId)), + ), + ]; + if (issueIds.length === 0) { + return new Map< + string, + Awaited> + >(); + } + return issuesSvc.listDependencyReadiness(companyId, issueIds); + } + + async function countRunningRunsForAgent(agentId: string) { + const [{ count }] = await db + .select({ count: sql`count(*)` }) + .from(heartbeatRuns) + .where( + and( + eq(heartbeatRuns.agentId, agentId), + eq(heartbeatRuns.status, "running"), + ), + ); + return Number(count ?? 0); + } + + async function withChatControlRecoveryGate( + run: typeof heartbeatRuns.$inferSelect, + stage: "claim" | "dispatch", + onClear: (tx: Db) => Promise, + nativeRecovery = false, + ) { + const issueId = readNonEmptyString( + parseObject(run.contextSnapshot).issueId, + ); + if (!issueId || run.invocationSource !== "automation") return onClear(db); + await options.beforeChatControlRecoveryCheck?.({ + runId: run.id, + issueId, + stage, + }); + if (stage === "dispatch" && nativeRecovery) { + // Protected historical/already-admitted owners did not previously take + // this fence's locks. A fresh exact read keeps unrelated Board contention + // from turning their recovery into a new admission failure. + const [current] = await db + .select() + .from(heartbeatRuns) + .where( + and( + eq(heartbeatRuns.id, run.id), + eq(heartbeatRuns.companyId, run.companyId), + eq(heartbeatRuns.agentId, run.agentId), + eq(heartbeatRuns.status, "running"), + ), + ) + .limit(1); + if ( + current && + current.wakeupRequestId === run.wakeupRequestId && + parseObject(current.contextSnapshot).issueId === issueId + ) { + const admission = readChatControlRecoveryAdmission(current); + const expected = readChatControlRecoveryAdmission(run); + if ( + expected !== "invalid" && + (admission === "admitted" || + (admission === "historical" && expected === "historical")) + ) + return onClear(db); + } + } + let terminal: typeof heartbeatRuns.$inferSelect | null = null; + try { + const attempt = () => db.transaction(async (tx) => { + terminal = null; + // Same queue-edit lock order, then the close committer's conversation + // row. NOWAIT releases partial locks on contention. Claim defers to the + // queue; dispatch retries this transaction before considering failure. + const [issue] = await tx + .select({ id: issues.id }) + .from(issues) + .where( + and(eq(issues.id, issueId), eq(issues.companyId, run.companyId)), + ) + .for("update", { noWait: true }) + .limit(1); + if (!issue) return null; + if (run.wakeupRequestId) + await tx + .select({ id: agentWakeupRequests.id }) + .from(agentWakeupRequests) + .where( + and( + eq(agentWakeupRequests.id, run.wakeupRequestId), + eq(agentWakeupRequests.companyId, run.companyId), + eq(agentWakeupRequests.agentId, run.agentId), + ), + ) + .for("update", { noWait: true }) + .limit(1); + const [current] = await tx + .select() + .from(heartbeatRuns) + .where( + and( + eq(heartbeatRuns.id, run.id), + eq(heartbeatRuns.companyId, run.companyId), + eq(heartbeatRuns.agentId, run.agentId), + ), + ) + .for("update", { noWait: true }) + .limit(1); + if ( + !current || + current.status !== (stage === "claim" ? "queued" : "running") + ) + return null; + const admission = readChatControlRecoveryAdmission(current); + const expectedAdmission = readChatControlRecoveryAdmission(run); + const admissionLost = + expectedAdmission !== "historical" && admission === "historical"; + if ( + stage === "dispatch" && + nativeRecovery && + !admissionLost && + (admission === "historical" || admission === "admitted") + ) + return onClear(tx as unknown as Db); + // A new warm child can inherit a prior runner's PID during preparation. + // Those fields do not prove this run's turn was dispatched. + // Exact admitted/historical recovery retains its existing ownership; + // a marked, unadmitted bootstrap must earn admission even after crash. + const proof = + admission === "invalid" || admissionLost + ? { kind: "unresolved" as const } + : await readChatControlRecoveryStop( + tx as unknown as Db, + { + companyId: run.companyId, + issueId, + agentId: run.agentId, + sourceRunId: run.id, + }, + true, + ); + if (proof.kind === "clear") { + if (stage === "claim" || admission === "required") + await tx + .update(heartbeatRuns) + .set({ + runnerProfileJson: { + ...parseObject(current.runnerProfileJson), + [CHAT_CONTROL_RECOVERY_ADMISSION_KEY]: + chatControlRecoveryAdmission( + current, + stage === "dispatch" + ? "admitted" + : admission === "admitted" + ? "admitted" + : "required", + ), + }, + updatedAt: new Date(), + }) + .where(eq(heartbeatRuns.id, current.id)); + return onClear(tx as unknown as Db); + } + if (proof.kind === "unresolved" && stage === "claim") return null; + const now = new Date(); + const stopped = proof.kind === "stopped"; + const code = stopped + ? CHAT_CONTROL_RECOVERY_STOP_CODE + : CHAT_CONTROL_RECOVERY_UNRESOLVED_CODE; + const error = stopped + ? "Automatic continuation stopped by the committed chat conversation close. Send a new request in chat or on the Board to start fresh work." + : "Automatic continuation source could not be verified before provider admission. Review the task and send a fresh request; this attempt will not automatically retry."; + await notifyChatRunMilestones(tx, { companyId: current.companyId, issueId }); + await notifyInstructionCollection(tx, { companyId: current.companyId, runId: current.id }); + await notifyExternalAgentSettlement(tx, { companyId: current.companyId, runId: current.id }); + await notifyCancelledWorkspacePreservation(tx, { companyId: current.companyId, runId: current.id }); + await notifyUnsafeWorkspaceExportRunChange(tx, current); + [terminal] = await tx + .update(heartbeatRuns) + .set({ + status: stopped ? "cancelled" : "failed", + errorCode: code, + error, + finishedAt: now, + resultJson: { + ...parseObject(current.resultJson), + automaticRecovery: { + code, + providerDispatched: false, + ...(stopped + ? { + sourceRunId: proof.sourceRunId, + conversationId: proof.conversationId, + publicationId: proof.publicationId, + } + : {}), + }, + }, + updatedAt: now, + }) + .where( + and( + eq(heartbeatRuns.id, current.id), + eq(heartbeatRuns.status, current.status), + ), + ) + .returning(); + if (!terminal) return null; + if (current.wakeupRequestId) + await tx + .update(agentWakeupRequests) + .set({ + status: stopped ? "skipped" : "failed", + error, + finishedAt: now, + updatedAt: now, + }) + .where( + and( + eq(agentWakeupRequests.id, current.wakeupRequestId), + eq(agentWakeupRequests.companyId, current.companyId), + eq(agentWakeupRequests.agentId, current.agentId), + eq(agentWakeupRequests.runId, current.id), + ne(agentWakeupRequests.status, "cancelled"), + ), + ); + await tx + .update(issues) + .set({ + executionRunId: null, + executionAgentNameKey: null, + executionLockedAt: null, + updatedAt: now, + }) + .where( + and( + eq(issues.companyId, current.companyId), + eq(issues.id, issueId), + eq(issues.executionRunId, current.id), + ), + ); + return null; + }); + const result = stage === "dispatch" + ? await retryChatControlAdmission(attempt) + : await attempt(); + if (terminal) { + const settled = terminal as typeof heartbeatRuns.$inferSelect; + publishLiveEvent({ + companyId: settled.companyId, + type: "heartbeat.run.status", + payload: { + runId: settled.id, + agentId: settled.agentId, + status: settled.status, + errorCode: settled.errorCode, + error: settled.error, + }, + }); + publishRunLifecyclePluginEvent(settled); + if (stage === "dispatch") + await finalizeAgentStatus(settled.agentId, "cancelled"); + } + return result; + } catch (error) { + if (!isExternalChatWaitAuthorizationContention(error)) throw error; + if (stage === "dispatch") { + // No effect was admitted. Let the existing setup-failure path settle + // this attempt distinctly; it must not become a successful close. + throw new ChatControlRecoveryUnresolvedError(); + } + return null; + } + } + + async function claimQueuedRun( + run: typeof heartbeatRuns.$inferSelect, + companyAgents?: AgentOrgRow[], + deferredPostCommitEffects?: WakeQueuePostCommitEffect[], + ) { + if (run.status !== "queued") return run; + const [caseHandoffReceipt] = run.wakeupRequestId ? await db.select().from(caseHandoffs).where(and( + eq(caseHandoffs.wakeupRequestId, run.wakeupRequestId), eq(caseHandoffs.companyId, run.companyId), + )).limit(1) : []; + if (caseHandoffReceipt) { + const [current] = await db.select({ version: cases.handoffVersion, issueId: cases.workIssueId, + agentId: issues.assigneeAgentId, statusVersion: issues.statusVersion }).from(cases) + .innerJoin(issues, eq(issues.id, cases.workIssueId)).where(and(eq(cases.id, caseHandoffReceipt.caseId), + eq(cases.companyId, run.companyId), eq(issues.companyId, run.companyId))).limit(1); + if (!["pending_dispatch", "delivered"].includes(caseHandoffReceipt.state) || caseHandoffReceipt.toAgentId !== run.agentId || + !current || current.version !== caseHandoffReceipt.version || current.issueId !== caseHandoffReceipt.issueId || + current.agentId !== run.agentId || (!run.startedAt && current.statusVersion !== caseHandoffReceipt.appliedIssueStatusVersion)) { + const cancelled = await db.transaction(async tx => { + const [currentRun] = await tx.select({ id: heartbeatRuns.id }).from(heartbeatRuns) + .where(and(eq(heartbeatRuns.id, run.id), eq(heartbeatRuns.companyId, run.companyId), eq(heartbeatRuns.status, "queued"))) + .for("update"); + if (!currentRun) return null; + await notifyChatRunMilestones(tx, { companyId: run.companyId, runId: run.id, + issueId: run.nativeIssueId ?? run.issueId ?? readNonEmptyString(parseObject(run.contextSnapshot).issueId) ?? undefined }); + await notifyInstructionCollection(tx, { companyId: run.companyId, runId: run.id }); + await notifyExternalAgentSettlement(tx, { companyId: run.companyId, runId: run.id }); + await notifyCancelledWorkspacePreservation(tx, { companyId: run.companyId, runId: run.id }); + await notifyUnsafeWorkspaceExportRunChange(tx, run); + const [row] = await tx.update(heartbeatRuns).set({ status: "cancelled", finishedAt: new Date(), + errorCode: "case_handoff_superseded", error: "This case handoff was cancelled or superseded.", updatedAt: new Date() }) + .where(and(eq(heartbeatRuns.id, run.id), eq(heartbeatRuns.companyId, run.companyId), eq(heartbeatRuns.status, "queued"))).returning(); + return row; + }); + if (cancelled) await setWakeupStatus(run.wakeupRequestId, "cancelled", { finishedAt: new Date(), error: cancelled.error }); + return null; + } + if (!(await instanceSettings.getExperimental()).enableCases) return null; + } + const caseIssueId = readNonEmptyString(parseObject(run.contextSnapshot).issueId); + if (caseIssueId) { + const [pending] = await db.select().from(caseHandoffs).where(and(eq(caseHandoffs.companyId, run.companyId), eq(caseHandoffs.issueId, caseIssueId), + inArray(caseHandoffs.state, ["awaiting_release", "pending_dispatch"]))).limit(1); + if (pending && (pending.wakeupRequestId !== run.wakeupRequestId || pending.state === "awaiting_release")) return null; + } + // Claim is the first execution boundary. startedAt survives same-run + // retries; restart recovery may also enter executeRun already running. + // Neither may reinterpret an admitted, already-started message as new work. + if (!run.startedAt && !(await slackMentionAllowsRunStart(db, run))) { + const now = new Date(); + const reason = "Message did not @mention the bot"; + // This runs under the agent start lock. Do not use cancelRunInternal, + // which recursively starts the next run and reacquires that lock. + const cancelled = await setRunStatus(run.id, "cancelled", { + finishedAt: now, + error: reason, + errorCode: "chat_mention_required", + }); + if (cancelled) { + await setWakeupStatus(run.wakeupRequestId, "skipped", { finishedAt: now, error: reason }); + await appendRunEvent(cancelled, { + eventType: "lifecycle", stream: "system", level: "warn", message: reason, + }); + await releaseIssueExecutionAndPromote(cancelled, { + suppressImmediateRecovery: true, deferredPostCommitEffects, + }); + } + return null; + } + + const agent = await getAgent(run.agentId); + if (!agent) { + await cancelRunInternal( + run.id, + "Cancelled because the agent no longer exists", + ); + return null; + } + // Keep accepted work in the durable queue until setup completes. + if (isAgentAwaitingSetup(agent)) return null; + const invokability = companyAgents + ? evaluateAgentInvokability(toAgentOrgRow(agent), companyAgents) + : await getAgentInvokability(agent); + if (!invokability.invokable) { + if (caseHandoffReceipt) return null; + await cancelRunInternal( + run.id, + `Cancelled because the agent is not invokable: ${invokability.reason}`, + ); + return null; + } + if (!run.startedAt && !await canStartNewExternalBotWork(agent)) return null; + + const context = parseObject(run.contextSnapshot); + const budgetBlock = await budgets.getInvocationBlock( + run.companyId, + run.agentId, + { + issueId: readNonEmptyString(context.issueId), + projectId: readNonEmptyString(context.projectId), + }, + ); + if (budgetBlock) { + if (caseHandoffReceipt) return null; + await cancelRunInternal(run.id, budgetBlock.reason); + return null; + } + + const dailyCapBlock = await getHeartbeatDailyCapBlock( + agent, + parseHeartbeatPolicy(agent), + { + excludeRunId: run.id, + checkRunCap: true, + checkCostCap: true, + }, + ); + if (dailyCapBlock) { + if (caseHandoffReceipt) return null; + await cancelQueuedRunForHeartbeatDailyCap(run, dailyCapBlock); + return null; + } + + const issueId = readNonEmptyString(context.issueId); + if (issueId && activeRunExecutions.size > 0) { + // Native finalization publishes success before workspace synchronization, + // provider suspension, and lease release finish. A queued comment must + // not acquire a fresh sandbox while its predecessor still owns that work. + // The executor's finally block retries this agent after removing its owner. + const [settlingOwner] = await db.select({ id: heartbeatRuns.id }) + .from(heartbeatRuns) + .where(and( + eq(heartbeatRuns.companyId, run.companyId), + eq(heartbeatRuns.agentId, run.agentId), + eq(heartbeatRuns.runtimeMode, "native"), + inArray(heartbeatRuns.id, [...activeRunExecutions]), + sql`${heartbeatRuns.contextSnapshot}->>'issueId' = ${issueId}`, + )) + .limit(1); + if (settlingOwner) return null; + } + if (issueId) { + const activePauseHold = await treeControlSvc.getActivePauseHoldGate( + run.companyId, + issueId, + ); + const treeHoldInteractionWake = + activePauseHold && + (await isVerifiedIssueTreeControlInteractionWake(db, { + companyId: run.companyId, + issueId, + agentId: run.agentId, + runId: run.id, + wakeupRequestId: run.wakeupRequestId, + contextSnapshot: context, + })); + if (activePauseHold && !treeHoldInteractionWake) { + await cancelRunInternal( + run.id, + "Cancelled because issue is held by an active subtree pause hold", + ); + await logActivity(db, { + companyId: run.companyId, + actorType: "system", + actorId: "system", + agentId: run.agentId, + runId: run.id, + action: "issue.tree_hold_run_interrupted", + entityType: "heartbeat_run", + entityId: run.id, + issueId: issueId, + details: { + issueId, + holdId: activePauseHold.holdId, + rootIssueId: activePauseHold.rootIssueId, + source: "heartbeat.claim_queued_run", + securityPrinciples: [ + "Complete Mediation", + "Fail Securely", + "Secure Defaults", + ], + }, + }); + return null; + } + + const dependencyReadiness = await issuesSvc.listDependencyReadiness( + run.companyId, + [issueId], + ); + const readiness = dependencyReadiness.get(issueId); + const unresolvedBlockerCount = readiness?.unresolvedBlockerCount ?? 0; + if ( + unresolvedBlockerCount > 0 && + !allowsIssueInteractionWake( + context, + ISSUE_TREE_CONTROL_INTERACTION_WAKE_REASONS, + ) + ) { + await cancelQueuedRunForBlockedDependencies( + run, + issueId, + readiness?.unresolvedBlockerIssueIds ?? [], + ); + logger.info( + { runId: run.id, issueId, unresolvedBlockerCount }, + "claimQueuedRun: cancelled blocked queued run", + ); + return null; + } + + const staleness = await runDispatch.cancelStaleQueuedRun({ + runId: run.id, + companyId: run.companyId, + expectedStatus: "queued", + }); + if (staleness.outcome === "cancelled" || staleness.outcome === "deferred") { + applyRunDispatchPostCommitEffects(staleness.postCommitEffects); + logger.info( + { + runId: run.id, + issueId, + outcome: staleness.outcome, + errorCode: staleness.outcome === "cancelled" ? staleness.errorCode : undefined, + }, + "claimQueuedRun: withheld queued run at the execution gate", + ); + return null; + } + } + + const claimedAt = new Date(); + const responsibleUserId = await resolveResponsibleUserIdForRun({ + run, + contextSnapshot: context, + issueContext: issueId + ? await getIssueExecutionContext(run.companyId, issueId) + : null, + routineEnvContext: { + routineId: null, + env: null, + responsibleUserId: null, + }, + }); + // All ordinary and comment claims use the same company-scoped issue + // lock. A batch may claim several runs before executeRun tracks any owner. + async function lockIssueExecutionClaim(tx: Db) { + const [owner] = issueId ? await tx.select({ + assigneeAgentId: issues.assigneeAgentId, + statusVersion: issues.statusVersion, + executionRunId: issues.executionRunId, + checkoutRunId: issues.checkoutRunId, + }).from(issues).where(and( + eq(issues.id, issueId), eq(issues.companyId, run.companyId), + )).for("update") : []; + const ownsIssue = owner?.assigneeAgentId === run.agentId && + context.wakeReason !== "source_scoped_recovery_action"; + const [handoff] = issueId ? await tx.select().from(caseHandoffs).where(and(eq(caseHandoffs.companyId, run.companyId), eq(caseHandoffs.issueId, issueId), + inArray(caseHandoffs.state, ["awaiting_release", "pending_dispatch"]))).limit(1) : []; + if (handoff && (handoff.wakeupRequestId !== run.wakeupRequestId || handoff.state === "awaiting_release")) + return { ownsIssue, blocked: true }; + if (caseHandoffReceipt) { + const [current] = await tx.select({ state: caseHandoffs.state, version: caseHandoffs.version, + currentVersion: cases.handoffVersion, workIssueId: cases.workIssueId, + appliedIssueStatusVersion: caseHandoffs.appliedIssueStatusVersion, + toAgentId: caseHandoffs.toAgentId, wakeupRequestId: caseHandoffs.wakeupRequestId }).from(caseHandoffs) + .innerJoin(cases, eq(cases.id, caseHandoffs.caseId)).where(and(eq(caseHandoffs.id, caseHandoffReceipt.id), + eq(caseHandoffs.companyId, run.companyId))).limit(1); + if (!current || !["pending_dispatch", "delivered"].includes(current.state) || current.version !== current.currentVersion || + current.workIssueId !== issueId || current.toAgentId !== run.agentId || current.wakeupRequestId !== run.wakeupRequestId || + !ownsIssue || (!run.startedAt && current.appliedIssueStatusVersion !== owner?.statusVersion)) + return { ownsIssue, blocked: true }; + } + if (ownsIssue && ["native_safe_replacement", "native_provider_overloaded"].includes(run.scheduledRetryReason ?? "") && + owner.checkoutRunId && owner.checkoutRunId !== run.id) { + return { ownsIssue, blocked: true }; + } + if (run.scheduledRetryReason === "native_provider_overloaded" && owner?.executionRunId && + owner.executionRunId !== run.id && owner.executionRunId !== run.retryOfRunId) { + return { ownsIssue, blocked: true }; + } + const previousRunId = run.scheduledRetryReason === "native_provider_overloaded" + ? run.retryOfRunId : ownsIssue ? owner?.executionRunId : null; + if (previousRunId && previousRunId !== run.id) { + const [previous] = await tx.select({ status: heartbeatRuns.status }) + .from(heartbeatRuns).where(and( + eq(heartbeatRuns.id, previousRunId), + eq(heartbeatRuns.companyId, run.companyId), + )); + // A terminal result can precede workspace/lease cleanup on this or + // another controller. Local absence alone is not a release receipt. + if (!isHeartbeatRunTerminalStatus(previous?.status) || + liveRunExecutions.has(previousRunId)) { + return { ownsIssue, blocked: true }; + } + if (await hasPendingRunCleanup(tx, run.companyId, previousRunId)) { + return { ownsIssue, blocked: true }; + } + } + return { ownsIssue, blocked: false }; + } + async function bindClaimedIssueExecution(tx: Db, ownsIssue: boolean, claimedRun: typeof heartbeatRuns.$inferSelect | null | undefined) { + if (!claimedRun || !issueId || !ownsIssue) return; + await notifyStatusCardIssueChange(tx, run.companyId); + await tx.update(issues).set({ + executionRunId: claimedRun.id, + executionAgentNameKey: normalizeAgentNameKey(agent.name), + executionLockedAt: claimedAt, + updatedAt: claimedAt, + }).where(and(eq(issues.id, issueId), eq(issues.companyId, run.companyId))); + } + const nativeReviewContext = readNativeReviewAssignmentContext(context); + const queuedCommentIds = queuedCommentIdsFromRunContext(context); + if ( + issueId && + run.invocationSource === "automation" && + queuedCommentIds.length > 0 + ) + await options.beforeChatControlRecoveryCheck?.({ + runId: run.id, + issueId, + stage: "claim", + }); + const queuedCommentClaim = + !nativeReviewContext && issueId && run.wakeupRequestId && queuedCommentIds.length > 0 + ? await db + .transaction(async (tx) => { + // Match the queue-edit lock order: issue, wake, then run. Once the + // run becomes running, a concurrent discard must observe the + // claimed wake and return an explicit conflict; if discard wins, + // this claim observes the cancelled queue and does no work. + const issueClaim = await lockIssueExecutionClaim(tx as unknown as Db); + if (issueClaim.blocked) return { kind: "stale" as const, run: null }; + const wake = await tx + .select() + .from(agentWakeupRequests) + .where( + and( + eq(agentWakeupRequests.id, run.wakeupRequestId!), + eq(agentWakeupRequests.companyId, run.companyId), + eq(agentWakeupRequests.agentId, run.agentId), + ), + ) + .for("update") + .limit(1) + .then((rows) => rows[0] ?? null); + const lockedRun = await tx + .select() + .from(heartbeatRuns) + .where( + and( + eq(heartbeatRuns.id, run.id), + eq(heartbeatRuns.companyId, run.companyId), + eq(heartbeatRuns.agentId, run.agentId), + ), + ) + .for("update") + .limit(1) + .then((rows) => rows[0] ?? null); + if ( + !wake || + wake.status !== "queued" || + wake.runId !== run.id || + !lockedRun || + lockedRun.status !== "queued" || + lockedRun.wakeupRequestId !== wake.id + ) { + return { kind: "stale" as const, run: null }; + } + + if (lockedRun.invocationSource === "automation") { + const admission = readChatControlRecoveryAdmission(lockedRun); + if (admission === "invalid") + return { kind: "stale" as const, run: null }; + const proof = await readChatControlRecoveryStop( + tx as unknown as Db, + { + companyId: lockedRun.companyId, + issueId, + agentId: lockedRun.agentId, + sourceRunId: lockedRun.id, + }, + true, + ); + if (proof.kind === "unresolved") + return { kind: "stale" as const, run: null }; + if (proof.kind === "stopped") { + const error = + "Automatic continuation stopped by the committed chat conversation close. Send a fresh request in chat or on the Board."; + await notifyChatRunMilestones(tx, { companyId: run.companyId, issueId }); + await notifyInstructionCollection(tx, { companyId: run.companyId, runId: run.id }); + await notifyExternalAgentSettlement(tx, { companyId: run.companyId, runId: run.id }); + await notifyCancelledWorkspacePreservation(tx, { companyId: run.companyId, runId: run.id }); + await notifyUnsafeWorkspaceExportRunChange(tx, run); + const [cancelled] = await tx + .update(heartbeatRuns) + .set({ + status: "cancelled", + errorCode: CHAT_CONTROL_RECOVERY_STOP_CODE, + error, + finishedAt: claimedAt, + resultJson: { + ...parseObject(lockedRun.resultJson), + automaticRecovery: { + code: CHAT_CONTROL_RECOVERY_STOP_CODE, + providerDispatched: false, + sourceRunId: proof.sourceRunId, + conversationId: proof.conversationId, + publicationId: proof.publicationId, + }, + }, + updatedAt: claimedAt, + }) + .where( + and( + eq(heartbeatRuns.id, lockedRun.id), + eq(heartbeatRuns.status, "queued"), + ), + ) + .returning(); + if (!cancelled) return { kind: "stale" as const, run: null }; + await tx + .update(agentWakeupRequests) + .set({ + status: "skipped", + error, + finishedAt: claimedAt, + updatedAt: claimedAt, + }) + .where(eq(agentWakeupRequests.id, wake.id)); + await tx + .update(issues) + .set({ + executionRunId: null, + executionAgentNameKey: null, + executionLockedAt: null, + updatedAt: claimedAt, + }) + .where( + and( + eq(issues.id, issueId), + eq(issues.companyId, run.companyId), + eq(issues.executionRunId, run.id), + ), + ); + return { kind: "cancelled" as const, run: cancelled }; + } + await tx + .update(heartbeatRuns) + .set({ + runnerProfileJson: { + ...parseObject(lockedRun.runnerProfileJson), + [CHAT_CONTROL_RECOVERY_ADMISSION_KEY]: + chatControlRecoveryAdmission( + lockedRun, + admission === "admitted" ? "admitted" : "required", + ), + }, + updatedAt: claimedAt, + }) + .where(eq(heartbeatRuns.id, lockedRun.id)); + } + const authoritativeIds = queuedCommentIdsFromWakePayload( + wake.payload, + ); + if (authoritativeIds.length === 0) { + // Legacy/direct comment wakes carry comment ids in their ordinary + // payload and context, not in the authoritative queued-message + // envelope. Preserve their established claim path; only an + // explicitly bound queued-message envelope is subject to the + // live-comment discard gate below. + await withExternalAdmissionGuard(tx, run.companyId, run.agentId, () => assertNoExternalOverlap(tx, run.companyId, run.agentId)); + await notifyChatRunMilestones(tx, { companyId: run.companyId, issueId }); + const [claimedRun] = await tx + .update(heartbeatRuns) + .set({ + status: "running", + runnerProfileJson: sql`(case when jsonb_typeof(${heartbeatRuns.runnerProfileJson}) = 'object' then ${heartbeatRuns.runnerProfileJson} else '{}'::jsonb end) || ${JSON.stringify({ adapterDispatch: { adapterType: agent.adapterType } })}::jsonb`, + ...legacyControllerClaim(run.runtimeMode), + responsibleUserId, + startedAt: lockedRun.startedAt ?? claimedAt, + updatedAt: claimedAt, + }) + .where( + and( + eq(heartbeatRuns.id, lockedRun.id), + eq(heartbeatRuns.status, "queued"), + ), + ) + .returning(); + await bindClaimedIssueExecution(tx as unknown as Db, issueClaim.ownsIssue, claimedRun); + return claimedRun + ? { kind: "claimed" as const, run: claimedRun } + : { kind: "stale" as const, run: null }; + } + const commentRows = await tx + .select({ + id: issueComments.id, + deletedAt: issueComments.deletedAt, + }) + .from(issueComments) + .where( + and( + eq(issueComments.companyId, run.companyId), + eq(issueComments.issueId, issueId), + inArray(issueComments.id, authoritativeIds), + ), + ); + const liveIds = authoritativeIds.filter((commentId) => { + const comment = commentRows.find((row) => row.id === commentId); + return Boolean(comment && !comment.deletedAt); + }); + if (liveIds.length === 0) { + const reason = "Queued messages were discarded before dispatch"; + await notifyChatRunMilestones(tx, { companyId: run.companyId, issueId }); + await notifyInstructionCollection(tx, { companyId: run.companyId, runId: run.id }); + await notifyExternalAgentSettlement(tx, { companyId: run.companyId, runId: run.id }); + await notifyCancelledWorkspacePreservation(tx, { companyId: run.companyId, runId: run.id }); + await notifyUnsafeWorkspaceExportRunChange(tx, run); + const [cancelled] = await tx + .update(heartbeatRuns) + .set({ + status: "cancelled", + finishedAt: claimedAt, + error: reason, + errorCode: "queued_comment_discarded", + updatedAt: claimedAt, + }) + .where( + and( + eq(heartbeatRuns.id, lockedRun.id), + eq(heartbeatRuns.status, "queued"), + ), + ) + .returning(); + await tx + .update(agentWakeupRequests) + .set({ + status: "cancelled", + finishedAt: claimedAt, + error: reason, + updatedAt: claimedAt, + }) + .where(eq(agentWakeupRequests.id, wake.id)); + await tx + .update(issues) + .set({ + executionRunId: null, + executionAgentNameKey: null, + executionLockedAt: null, + updatedAt: claimedAt, + }) + .where( + and( + eq(issues.id, issueId), + eq(issues.companyId, run.companyId), + eq(issues.executionRunId, run.id), + ), + ); + return { + kind: "cancelled" as const, + run: cancelled ?? lockedRun, + }; + } + + await withExternalAdmissionGuard(tx, run.companyId, run.agentId, () => assertNoExternalOverlap(tx, run.companyId, run.agentId)); + await notifyChatRunMilestones(tx, { companyId: run.companyId, issueId }); + await tx + .update(agentWakeupRequests) + .set({ + status: "claimed", + claimedAt, + payload: withQueuedCommentIdsInWakePayload( + wake.payload, + liveIds, + ), + updatedAt: claimedAt, + }) + .where(eq(agentWakeupRequests.id, wake.id)); + const [claimedRun] = await tx + .update(heartbeatRuns) + .set({ + status: "running", + runnerProfileJson: sql`(case when jsonb_typeof(${heartbeatRuns.runnerProfileJson}) = 'object' then ${heartbeatRuns.runnerProfileJson} else '{}'::jsonb end) || ${JSON.stringify({ adapterDispatch: { adapterType: agent.adapterType } })}::jsonb`, + ...legacyControllerClaim(run.runtimeMode), + responsibleUserId, + startedAt: lockedRun.startedAt ?? claimedAt, + contextSnapshot: withQueuedCommentIdsInRunContext( + lockedRun.contextSnapshot, + liveIds, + ), + updatedAt: claimedAt, + }) + .where( + and( + eq(heartbeatRuns.id, lockedRun.id), + eq(heartbeatRuns.status, "queued"), + ), + ) + .returning(); + await bindClaimedIssueExecution(tx as unknown as Db, issueClaim.ownsIssue, claimedRun); + return claimedRun + ? { kind: "claimed" as const, run: claimedRun } + : { kind: "stale" as const, run: null }; + }) + .catch((error) => { + if (isExternalChatWaitAuthorizationContention(error)) + return { kind: "stale" as const, run: null }; + throw error; + }) + : null; + if (queuedCommentClaim?.kind === "cancelled") { + await appendRunEvent(queuedCommentClaim.run, { + eventType: "lifecycle", + stream: "system", + level: "warn", + message: + queuedCommentClaim.run.error ?? + "Queued messages were discarded before dispatch", + }); + publishLiveEvent({ + companyId: queuedCommentClaim.run.companyId, + type: "heartbeat.run.status", + payload: { + runId: queuedCommentClaim.run.id, + agentId: queuedCommentClaim.run.agentId, + status: queuedCommentClaim.run.status, + invocationSource: queuedCommentClaim.run.invocationSource, + triggerDetail: queuedCommentClaim.run.triggerDetail, + error: queuedCommentClaim.run.error ?? null, + errorCode: queuedCommentClaim.run.errorCode ?? null, + startedAt: queuedCommentClaim.run.startedAt + ? new Date(queuedCommentClaim.run.startedAt).toISOString() + : null, + finishedAt: queuedCommentClaim.run.finishedAt + ? new Date(queuedCommentClaim.run.finishedAt).toISOString() + : null, + }, + }); + publishRunLifecyclePluginEvent(queuedCommentClaim.run); + // Fire-and-forget: nothing else in this path depends on the emission, + // so it must not delay the return. + void emitAgentTaskRun(db, queuedCommentClaim.run); + return null; + } + const claimed = queuedCommentClaim + ? queuedCommentClaim.run + : await withChatControlRecoveryGate(run, "claim", async (tx) => { + const claimValues = { + status: "running", + runnerProfileJson: sql`(case when jsonb_typeof(${heartbeatRuns.runnerProfileJson}) = 'object' then ${heartbeatRuns.runnerProfileJson} else '{}'::jsonb end) || ${JSON.stringify({ adapterDispatch: { adapterType: agent.adapterType } })}::jsonb`, + ...legacyControllerClaim(run.runtimeMode), + responsibleUserId, + startedAt: run.startedAt ?? claimedAt, + updatedAt: claimedAt, + }; + if (nativeReviewContext) { + if (run.scheduledRetryReason === "native_provider_overloaded") { + const predecessor = await lockIssueExecutionClaim(tx); + if (predecessor.blocked) return null; + } + return claimQueuedNativeReviewRun(tx, { + run, claimedAt, claimValues, + agentNameKey: normalizeAgentNameKey(agent.name), + }); + } + return tx.transaction(async (claimTx) => { + const issueClaim = await lockIssueExecutionClaim(claimTx as unknown as Db); + if (issueClaim.blocked) return null; + if (run.wakeupRequestId) await claimTx.select({ id: agentWakeupRequests.id }).from(agentWakeupRequests).where(and( + eq(agentWakeupRequests.id, run.wakeupRequestId), eq(agentWakeupRequests.companyId, run.companyId), eq(agentWakeupRequests.agentId, run.agentId), + )).for("update"); + const [lockedRun] = await claimTx.select().from(heartbeatRuns).where(and( + eq(heartbeatRuns.id, run.id), eq(heartbeatRuns.companyId, run.companyId), eq(heartbeatRuns.agentId, run.agentId), + )).for("update"); + if (!lockedRun || lockedRun.status !== "queued") return null; + await withExternalAdmissionGuard(claimTx, run.companyId, run.agentId, () => assertNoExternalOverlap(claimTx, run.companyId, run.agentId)); + await notifyChatRunMilestones(claimTx, { companyId: run.companyId, runId: run.id, issueId: issueId ?? undefined }); + const claimedRun = await claimTx.update(heartbeatRuns).set(claimValues).where(and( + eq(heartbeatRuns.id, run.id), eq(heartbeatRuns.status, "queued"), + )).returning().then((rows) => rows[0] ?? null); + await bindClaimedIssueExecution(claimTx as unknown as Db, issueClaim.ownsIssue, claimedRun); + return claimedRun; + }); + }); + if (!claimed) return null; + + publishLiveEvent({ + companyId: claimed.companyId, + type: "heartbeat.run.status", + payload: { + runId: claimed.id, + agentId: claimed.agentId, + status: claimed.status, + invocationSource: claimed.invocationSource, + triggerDetail: claimed.triggerDetail, + error: claimed.error ?? null, + errorCode: claimed.errorCode ?? null, + startedAt: claimed.startedAt + ? new Date(claimed.startedAt).toISOString() + : null, + finishedAt: claimed.finishedAt + ? new Date(claimed.finishedAt).toISOString() + : null, + }, + }); + publishRunLifecyclePluginEvent(claimed); + + if (!nativeReviewContext) { + await setWakeupStatus(claimed.wakeupRequestId, "claimed", { claimedAt }); + } + + // Fix A (lazy locking): stamp executionRunId now that the run is actually running, + // not at queue time. Guard is idempotent — safe if called more than once. + const claimedContext = parseObject(claimed.contextSnapshot); + const claimedIssueId = readNonEmptyString(claimedContext.issueId); + const claimedWakeReason = readNonEmptyString(claimedContext.wakeReason); + if ( + !nativeReviewContext && claimedIssueId && + claimedWakeReason !== "source_scoped_recovery_action" + ) { + const claimedAgent = await getAgent(claimed.agentId); + await db + .update(issues) + .set({ + executionRunId: claimed.id, + executionAgentNameKey: normalizeAgentNameKey(claimedAgent?.name), + executionLockedAt: claimedAt, + updatedAt: claimedAt, + }) + .where( + and( + eq(issues.id, claimedIssueId), + eq(issues.companyId, claimed.companyId), + // Mention/context runs can touch an issue, but only the current assignee + // owns the issue execution lock shown as the active run. + eq(issues.assigneeAgentId, claimed.agentId), + ["native_safe_replacement", "native_provider_overloaded"].includes(claimed.scheduledRetryReason ?? "") + ? or( + isNull(issues.checkoutRunId), + eq(issues.checkoutRunId, claimed.id), + ) + : undefined, + or( + isNull(issues.executionRunId), + eq(issues.executionRunId, claimed.id), + ), + ), + ); + } + + return claimed; + } + + // startNextQueuedRunForAgent checks admission suppression once, then claims + // runs (sets status "running"), then dispatches each to executeRun, which + // checks suppression again before it does any work. Suppression (task + // drain, worktree mode, a database restore) can start in the gap between + // those two checks. When executeRun's check catches that, the run is + // already claimed — release it back to "queued" so it does not keep a + // running execution lock that nothing will ever process. This runs inside + // the same promise startNextQueuedRunForAgent already tracks in + // activeRunExecutionPromises, so getTaskDrainStatus() keeps reporting the + // run as active until the release finishes. + // + // The run row, the wakeup request, and the issue execution lock all guard + // the same claim, so one transaction commits all three writes together. A + // partial write (for example the run flips to "queued" but the wakeup or + // issue update then fails) would let the execution promise clear from + // activeRunExecutionPromises while the wakeup stayed "claimed" or the + // issue stayed locked to a queued run — task-drain status would then read + // quiescent while the database still held part of the old claim. + async function releaseRunClaimedJustBeforeSuppression(runId: string) { + const now = new Date(); + await db.transaction(async (tx) => { + const [candidate] = await tx.select({ companyId: heartbeatRuns.companyId, contextSnapshot: heartbeatRuns.contextSnapshot }) + .from(heartbeatRuns).where(and(eq(heartbeatRuns.id, runId), eq(heartbeatRuns.status, "running"))).for("update"); + if (!candidate) return; + await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers); + await notifyChatRunMilestones(tx, { companyId: candidate.companyId, runId, + issueId: readNonEmptyString(parseObject(candidate.contextSnapshot).issueId) ?? undefined }); + const released = await tx + .update(heartbeatRuns) + .set({ + status: "queued", + startedAt: null, + responsibleUserId: null, + updatedAt: now, + }) + .where( + and(eq(heartbeatRuns.id, runId), eq(heartbeatRuns.status, "running")), + ) + .returning() + .then((rows) => rows[0] ?? null); + if (!released) return; + + if (released.wakeupRequestId) { + await tx + .update(agentWakeupRequests) + .set({ status: "queued", claimedAt: null, updatedAt: now }) + .where(and(eq(agentWakeupRequests.id, released.wakeupRequestId), ne(agentWakeupRequests.status, "cancelled"))); + } + + const context = parseObject(released.contextSnapshot); + const issueId = readNonEmptyString(context.issueId); + if (issueId) { + await tx + .update(issues) + .set({ + executionRunId: null, + executionAgentNameKey: null, + executionLockedAt: null, + updatedAt: now, + }) + .where( + and( + eq(issues.id, issueId), + eq(issues.companyId, released.companyId), + eq(issues.executionRunId, released.id), + ), + ); + } + }); + } + + async function cancelQueuedRunForBlockedDependencies( + run: typeof heartbeatRuns.$inferSelect, + issueId: string, + unresolvedBlockerIssueIds: string[], + ) { + const now = new Date(); + const reason = + "Cancelled because issue dependencies are still blocked; Paperclip will wake the assignee when blockers resolve"; + const cancelled = await setRunStatus(run.id, "cancelled", { + finishedAt: now, + error: reason, + errorCode: "issue_dependencies_blocked", + resultJson: { + ...parseObject(run.resultJson), + stopReason: "issue_dependencies_blocked", + effectiveTimeoutSec: 0, + timeoutConfigured: false, + timeoutSource: "dependency_gate", + timeoutFired: false, + }, + }); + if (!cancelled) return null; + + await setWakeupStatus(run.wakeupRequestId, "skipped", { + finishedAt: now, + error: reason, + }); + + await db.transaction(async tx => { + await notifyUnsafeWorkspaceExportRecovery(tx, { companyId: run.companyId, issueId }); + await notifyNativeStatusReassessment(tx, { companyId: run.companyId, issueId }); + await tx + .update(issues) + .set({ + executionRunId: null, + executionAgentNameKey: null, + executionLockedAt: null, + updatedAt: now, + }) + .where( + and( + eq(issues.companyId, run.companyId), + eq(issues.id, issueId), + eq(issues.executionRunId, run.id), + ), + ); + }); + + await appendRunEvent(cancelled, { + eventType: "lifecycle", + stream: "system", + level: "warn", + message: reason, + payload: { + issueId, + unresolvedBlockerIssueIds, + }, + }); + + return cancelled; + } + + async function resumeQueuedRuns() { + if ((await getSchedulingSuppression()).suppressed) return; + await resumeExecutionWaitComments(); + const cutoff = await getWorktreeExecutionCutoff(); + const pendingInterrupts = await db.select({ id: agentWakeupRequests.id, companyId: agentWakeupRequests.companyId }) + .from(agentWakeupRequests).innerJoin(companies, eq(companies.id, agentWakeupRequests.companyId)) + .where(and(eq(agentWakeupRequests.status, "deferred_issue_execution"), + eq(companies.status, "active"), + sql`${agentWakeupRequests.payload}->'queuedCommentInterrupt' is not null`, + lte(agentWakeupRequests.updatedAt, new Date(Date.now() - 30_000)), + cutoff ? gte(agentWakeupRequests.requestedAt, cutoff) : undefined)) + .orderBy(asc(agentWakeupRequests.updatedAt)).limit(50); + for (const wake of pendingInterrupts) { + await db.update(agentWakeupRequests).set({ updatedAt: new Date() }).where(and( + eq(agentWakeupRequests.id, wake.id), eq(agentWakeupRequests.status, "deferred_issue_execution"), + )); + await resumeQueuedCommentInterrupt(wake.companyId, wake.id).catch(err => { + logger.warn({ err, queueId: wake.id }, "failed to resume interrupted comment queue"); + }); + } + // A server restart or a message/cleanup race can leave a deferred wake + // after its owner has released the issue lock. Revisit it through the same + // release admission, so recovery holds and operator Stops still apply. + const strandedQueues = await db.select({ wake: agentWakeupRequests }) + .from(agentWakeupRequests) + .innerJoin(issues, and(eq(issues.companyId, agentWakeupRequests.companyId), + sql`${issues.id}::text = ${agentWakeupRequests.payload}->>'issueId'`, + or(eq(issues.assigneeAgentId, agentWakeupRequests.agentId), + sql`${agentWakeupRequests.payload} #>> '{_paperclipWakeContext,wakeReason}' = 'issue_comment_mentioned'`))) + .innerJoin(companies, and(eq(companies.id, issues.companyId), eq(companies.status, "active"))) + .where(and(eq(agentWakeupRequests.status, "deferred_issue_execution"), + isNull(issues.executionRunId), + or(and( + sql`jsonb_typeof(${agentWakeupRequests.payload} #> '{_paperclipWakeContext,wakeCommentIds}') = 'array'`, + sql`${agentWakeupRequests.payload} #> '{_paperclipWakeContext,wakeCommentIds}' <> '[]'::jsonb`, + ), sql`${agentWakeupRequests.payload}->>'mutation' = 'interaction'`), + sql`${agentWakeupRequests.payload}->'queuedCommentInterrupt' is null`, + sql`NOT EXISTS (SELECT 1 FROM heartbeat_runs live_issue_run + WHERE live_issue_run.company_id = ${issues.companyId} + AND (live_issue_run.context_snapshot->>'issueId' = ${issues.id}::text + OR live_issue_run.native_issue_id = ${issues.id}) + AND live_issue_run.status IN ('queued', 'running', 'scheduled_retry'))`, + cutoff ? gte(agentWakeupRequests.requestedAt, cutoff) : undefined)) + .orderBy(asc(agentWakeupRequests.updatedAt)).limit(50); + const releasedNativeSources = new Set(); + for (const { wake } of strandedQueues) { + if (!queuedCommentIdsFromWakePayload(wake.payload).length && + !await readQueuedInteractionResponse(db, wake.companyId, String(wake.payload?.issueId), wake.payload)) continue; + const [latest] = await db.select({ + id: heartbeatRuns.id, + companyId: heartbeatRuns.companyId, + status: heartbeatRuns.status, + finishedAt: heartbeatRuns.finishedAt, + startedAt: heartbeatRuns.startedAt, + runtimeMode: heartbeatRuns.runtimeMode, + errorCode: heartbeatRuns.errorCode, + sourceVersion: sql`${heartbeatRuns}.xmin::text`, + }).from(heartbeatRuns).where(and( + eq(heartbeatRuns.companyId, wake.companyId), + sql`${heartbeatRuns.contextSnapshot}->>'issueId' = ${String(wake.payload?.issueId)}`, + or(isNull(heartbeatRuns.nativeIssueId), sql`${heartbeatRuns.nativeIssueId}::text = ${String(wake.payload?.issueId)}`), + )).orderBy(desc(heartbeatRuns.createdAt), desc(heartbeatRuns.id)).limit(1); + await db.update(agentWakeupRequests).set({ updatedAt: new Date() }).where(and( + eq(agentWakeupRequests.id, wake.id), eq(agentWakeupRequests.status, "deferred_issue_execution"), + )); + if (!latest || !latest.finishedAt || !isHeartbeatRunTerminalStatus(latest.status)) continue; + if (latest.runtimeMode !== "legacy" && latest.runtimeMode !== "native") continue; + if (wake.idempotencyKey?.startsWith("chat-inbound:")) { + await resumeDeferredChatWake(db, { companyId: wake.companyId, agentId: wake.agentId, + issueId: String(wake.payload?.issueId), requestId: wake.id }).catch(() => { + logger.warn({ requestId: wake.id }, "saved chat source could not resume after recovery"); + }); + continue; + } + const releaseSource = `${latest.companyId}:${String(wake.payload?.issueId)}:${latest.id}:${latest.sourceVersion}`; + if (latest.runtimeMode === "native" && releasedNativeSources.has(releaseSource)) continue; + if (latest.runtimeMode === "legacy") { + const cancelledAdmission = latest.status === "cancelled" && !latest.startedAt && + latest.errorCode === "execution_reconciliation_required"; + if ((latest.status !== "cancelled" || cancelledAdmission) && await getExecutionBlocker(db, wake.companyId, String(wake.payload?.issueId))) { + await resumeSavedLegacyComments(wake.companyId, wake.id).catch(err => { + logger.warn({ err, queueId: wake.id }, "failed to deliver saved legacy comment after recovery stopped"); + }); + } + } + await releaseIssueExecutionAndPromote(latest, { suppressImmediateRecovery: true, resumeStrandedQueue: true }).then(() => { + if (latest.runtimeMode === "native") releasedNativeSources.add(releaseSource); + }).catch(err => { + logger.warn({ err, queueId: wake.id }, "failed to promote stranded comments"); + }); + } + + // The cancellation marker is durable intent. Retry while its exact queue + // is still deferred, including after a failed cleanup promotion or restart. + // Normal admission still checks process ownership, leases, pauses, and scope. + const interruptedQueues = await db + .select({ id: heartbeatRuns.id, companyId: heartbeatRuns.companyId }) + .from(agentWakeupRequests) + .innerJoin(heartbeatRuns, and( + sql`${heartbeatRuns.resultJson}->>'queuedCommentInterruptQueueId' = ${agentWakeupRequests.id}::text`, + eq(heartbeatRuns.companyId, agentWakeupRequests.companyId), + eq(heartbeatRuns.agentId, agentWakeupRequests.agentId), + )) + .innerJoin(companies, eq(companies.id, heartbeatRuns.companyId)) + .where(and( + eq(agentWakeupRequests.status, "deferred_issue_execution"), + eq(heartbeatRuns.status, "cancelled"), + eq(heartbeatRuns.runtimeMode, "legacy"), + eq(companies.status, "active"), + cutoff ? gte(heartbeatRuns.createdAt, cutoff) : undefined, + )); + for (const run of interruptedQueues) { + await releaseIssueExecutionAndPromote(run, { suppressImmediateRecovery: true }).catch((err) => { + logger.error({ err, runId: run.id }, "failed to retry interrupted comment queue"); + }); + } + + await resumeQueuedRunsAfterDrain(); + } + + // Release admission promptly without first scanning unrelated recovery work. + // Every run still passes the ordinary per-agent claim and dispatch gates. + async function resumeQueuedRunsAfterDrain(scope?: { companyId: string }) { + if ((await getSchedulingSuppression()).suppressed) return; + const cutoff = await getWorktreeExecutionCutoff(); + const queuedRuns = await db + .select({ agentId: heartbeatRuns.agentId }) + .from(heartbeatRuns) + .innerJoin(companies, eq(companies.id, heartbeatRuns.companyId)) + .where( + and( + eq(heartbeatRuns.status, "queued"), + scope ? eq(heartbeatRuns.companyId, scope.companyId) : undefined, + eq(companies.status, "active"), + cutoff ? gte(heartbeatRuns.createdAt, cutoff) : undefined, + ), + ); + + const agentIds = [...new Set(queuedRuns.map((r) => r.agentId))]; + for (const agentId of agentIds) { + await startNextQueuedRunForAgent(agentId); + } + } + + // A 403 from claimQueuedRun comes from the run's own persisted identity + // (an unverifiable interrupt receipt, a manual wake with no user). Those rows + // do not change, so the claim fails the same way on every pass and restart. + function isPermanentClaimRejection(err: unknown): err is HttpError { + return err instanceof HttpError && err.status === 403; + } + + // Other 4xx rejections can clear later (a responsible user gets assigned, a + // conflicting claim finishes). Keep the run queued, but do not let it stop + // the rest of the queue or startup recovery. + function isDeferrableClaimRejection(err: unknown): err is HttpError { + return err instanceof HttpError && err.status >= 400 && err.status < 500; + } + + // Settle runs that can never be claimed. Letting the error escape stalls the + // agent's queue and, during startup recovery, stops the server from booting. + async function cancelRejectedQueuedRuns( + rejected: Array<{ run: typeof heartbeatRuns.$inferSelect; err: HttpError }>, + ) { + for (const { run, err } of rejected) { + logger.warn( + { err, runId: run.id, agentId: run.agentId, companyId: run.companyId }, + "cancelling queued heartbeat run whose claim was rejected", + ); + try { + await cancelRunInternal( + run.id, + `Cancelled because the queued run cannot be claimed: ${err.message}`, + { errorCode: "queued_run_claim_rejected" }, + ); + } catch (cancelErr) { + logger.error( + { err: cancelErr, runId: run.id }, + "failed to cancel queued heartbeat run whose claim was rejected; it stays queued for the next recovery pass", + ); + } + } + } + + async function startNextQueuedRunForAgent(agentId: string) { + if ((await getSchedulingSuppression()).suppressed) return []; + const cutoff = await getWorktreeExecutionCutoff(); + // Cancelled after the start lock is released: cancelRunInternal promotes the + // agent's next queued run, which takes this same lock. + const rejectedClaims: Array<{ run: typeof heartbeatRuns.$inferSelect; err: HttpError }> = []; + const deferredPostCommitEffects: WakeQueuePostCommitEffect[] = []; + let cancellationReason: string | undefined; + + return withAgentStartLock(agentId, async () => { + const agent = await getAgent(agentId); + if (!agent) return []; + const invokability = await getAgentInvokability(agent); + if (!invokability.invokable) { + if (shouldCancelRunsForNonInvokableAgent(invokability)) { + cancellationReason = `Cancelled because the agent is not invokable: ${invokability.reason}`; + } + return []; + } + const policy = parseHeartbeatPolicy(agent); + const runningCount = await countRunningRunsForAgent(agentId); + const availableSlots = Math.max( + 0, + policy.maxConcurrentRuns - runningCount, + ); + if (availableSlots <= 0) return []; + + const queuedRuns = await db + .select() + .from(heartbeatRuns) + .where( + and( + eq(heartbeatRuns.agentId, agentId), + eq(heartbeatRuns.status, "queued"), + cutoff ? gte(heartbeatRuns.createdAt, cutoff) : undefined, + ), + ) + .orderBy(asc(heartbeatRuns.createdAt)); + if (queuedRuns.length === 0) return []; + if (queuedRuns.every(run => !run.startedAt) && !await canStartNewExternalBotWork(agent)) return []; + + const dependencyReadiness = await listQueuedRunDependencyReadiness( + agent.companyId, + queuedRuns, + ); + const queuedIssueIds = [ + ...new Set( + queuedRuns + .map((run) => + readNonEmptyString(parseObject(run.contextSnapshot).issueId), + ) + .filter((issueId): issueId is string => Boolean(issueId)), + ), + ]; + const issueRows = await db + .select({ + id: issues.id, + status: issues.status, + priority: issues.priority, + }) + .from(issues) + .where( + queuedIssueIds.length > 0 + ? and( + eq(issues.companyId, agent.companyId), + inArray(issues.id, queuedIssueIds), + ) + : sql`false`, + ); + const issueById = new Map(issueRows.map((row) => [row.id, row])); + const companyAgents = await listCompanyAgentOrgRows(agent.companyId); + const prioritizedRuns = [...queuedRuns].sort((left, right) => { + const leftIssueId = readNonEmptyString( + parseObject(left.contextSnapshot).issueId, + ); + const rightIssueId = readNonEmptyString( + parseObject(right.contextSnapshot).issueId, + ); + const leftReadiness = leftIssueId + ? dependencyReadiness.get(leftIssueId) + : null; + const rightReadiness = rightIssueId + ? dependencyReadiness.get(rightIssueId) + : null; + const leftReady = leftIssueId + ? (leftReadiness?.isDependencyReady ?? true) + : true; + const rightReady = rightIssueId + ? (rightReadiness?.isDependencyReady ?? true) + : true; + const leftIssue = leftIssueId ? issueById.get(leftIssueId) : null; + const rightIssue = rightIssueId ? issueById.get(rightIssueId) : null; + const leftRank = leftIssueId + ? leftReady + ? leftIssue?.status === "in_progress" + ? 0 + : 1 + : 3 + : 2; + const rightRank = rightIssueId + ? rightReady + ? rightIssue?.status === "in_progress" + ? 0 + : 1 + : 3 + : 2; + if (leftRank !== rightRank) return leftRank - rightRank; + const leftPriorityRank = issueRunPriorityRank(leftIssue?.priority); + const rightPriorityRank = issueRunPriorityRank(rightIssue?.priority); + if (leftPriorityRank !== rightPriorityRank) + return leftPriorityRank - rightPriorityRank; + return left.createdAt.getTime() - right.createdAt.getTime(); + }); + + const claimedRuns: Array = []; + for (const queuedRun of prioritizedRuns) { + if (claimedRuns.length >= availableSlots) break; + let claimed: typeof heartbeatRuns.$inferSelect | null; + try { + claimed = await claimQueuedRun(queuedRun, companyAgents, deferredPostCommitEffects); + } catch (err) { + if (isPermanentClaimRejection(err)) { + rejectedClaims.push({ run: queuedRun, err }); + continue; + } + if (!isDeferrableClaimRejection(err)) throw err; + logger.warn( + { err, runId: queuedRun.id, agentId: queuedRun.agentId, companyId: queuedRun.companyId }, + "queued heartbeat run claim was rejected; leaving it queued for the next recovery pass", + ); + continue; + } + if (claimed) claimedRuns.push(claimed); + } + if (claimedRuns.length === 0) return []; + + for (const claimedRun of claimedRuns) { + const execution = executeRun(claimedRun.id).catch((err) => { + logger.error( + { err, runId: claimedRun.id }, + "queued heartbeat execution failed", + ); + }); + // Register the in-flight execution so drainActiveRunExecutions() can await + // it. executeRun resolves only after its finally block finishes flushing + // run rows/events, so awaiting this promise guarantees the run's writes + // have landed before a caller (e.g. a test's afterEach) mutates the DB. + activeRunExecutionPromises.add(execution); + void execution.finally(() => { + // drainActiveRunExecutions loops on activeRunExecutionPromises.size, + // so an entry that never clears here would hang it forever. + activeRunExecutionPromises.delete(execution); + }); + } + return claimedRuns; + }).finally(async () => { + // Dispatch promoted inputs after the agent start lock is released. + try { + await applyWakeQueuePostCommitEffects(deferredPostCommitEffects); + } finally { + if (cancellationReason) await cancelActiveForAgentInternal(agentId, cancellationReason); + await cancelRejectedQueuedRuns(rejectedClaims); + } + }); + } + + // Public wakeup entry point. Callers dispatch it fire-and-forget, so register + // the promise in activeWakeupPromises before it starts its asynchronous + // prologue. drainActiveRunExecutions can then await a wake that is still before + // run registration. Internal callers reference enqueueWakeup directly and + // already await it, so they do not need this registration. + function trackWakeup( + agentId: string, + opts: WakeupOptions = {}, + ): ReturnType { + const promise = enqueueWakeup(agentId, opts); + activeWakeupPromises.add(promise); + void promise + .catch(() => {}) + .finally(() => { + activeWakeupPromises.delete(promise); + }); + return promise; + } + + async function enqueueWakeup(agentId: string, opts: WakeupOptions = {}, executionWaitRequestId?: string) { + const source = opts.source ?? "on_demand"; + const triggerDetail = opts.triggerDetail ?? null; + const contextSnapshot: Record = { + ...(opts.contextSnapshot ?? {}), + }; + const reason = opts.reason ?? null; + let payload = opts.payload ? { ...opts.payload } : null; + // Only the board queue route can record interruption authority on an + // existing receipt. Never accept this internal marker from a wake caller. + if (payload) { + delete payload.queuedCommentInterrupt; + delete payload.manualUserWake; + delete payload.taskRetry; + } + if (opts.manualUserWake) { + if (opts.requestedByActorType !== "user" || !opts.requestedByActorId || opts.failedRunId) { + throw new HttpError(403, "Manual wake requires an authenticated user"); + } + payload = { ...payload, manualUserWake: true }; + } + const executionReconciliationWake = + contextSnapshot.source === "execution.reconciled" || + opts.idempotencyKey?.startsWith("execution-reconciliation:") === true; + const { + contextSnapshot: enrichedContextSnapshot, + issueIdFromPayload, + taskKey, + wakeCommentId, + } = enrichWakeContextSnapshot({ + contextSnapshot, + reason, + source, + triggerDetail, + payload, + }); + // Keep each request's own server-derived origin, including coalesced wakes. + // A run's merged context cannot establish which caller authored one receipt. + // Overwrite caller-supplied nested context rather than trusting it. + payload = { ...payload, [DEFERRED_WAKE_CONTEXT_KEY]: { ...enrichedContextSnapshot } }; + let issueId = + readNonEmptyString(enrichedContextSnapshot.issueId) ?? issueIdFromPayload; + if (executionReconciliationWake && !issueId) return null; + const durableRequest = opts.durableChatRequest; + let taskRetryActorId: string | null = null; + delete enrichedContextSnapshot.boardMessageRetry; + if (opts.taskRetryRequestId) { + const [receipt] = await db.select().from(agentWakeupRequests).where(and( + eq(agentWakeupRequests.id, opts.taskRetryRequestId), eq(agentWakeupRequests.agentId, agentId), + eq(agentWakeupRequests.status, "deferred_issue_execution"), + eq(agentWakeupRequests.requestedByActorType, "user"), + sql`${agentWakeupRequests.payload}->>'issueId' = ${issueId ?? ""}`, + )); + const retry = readTaskRetryIntent(receipt?.payload); + if (!receipt || retry?.generationId !== receipt.id || retry.canonicalIntentId || + retry.actorId !== receipt.requestedByActorId) return null; + const chatRetry = retry.chatActionId && durableRequest?.id === retry.chatActionId && + durableRequest.failedRunRetry?.failedRunId === retry.sourceRunId; + if (retry.chatActionId ? !chatRetry : + opts.requestedByActorType !== "user" || retry.actorId !== opts.requestedByActorId) return null; + taskRetryActorId = retry.actorId; + enrichedContextSnapshot.forceFreshSession = true; + enrichedContextSnapshot.previousRunId = retry.sourceRunId; + enrichedContextSnapshot.taskRetryRequestId = receipt.id; + if (retry.boardMessageOrigin) enrichedContextSnapshot.boardMessageRetry = true; + } + + + let agent = await getAgent(agentId); + if (!agent) throw notFound("Agent not found"); + if (issueId) { + const [pending] = await db.select().from(caseHandoffs).where(and(eq(caseHandoffs.companyId, agent.companyId), eq(caseHandoffs.issueId, issueId), + inArray(caseHandoffs.state, ["awaiting_release", "pending_dispatch"]))).limit(1); + if (pending && opts.durableCaseHandoffRequest?.handoffId !== pending.id) return null; + } + // Mentions only annotate comments. Ignore legacy callers before creating + // a run or deferred request; assignment and review have their own wakes. + if (reason === "issue_comment_mentioned" || enrichedContextSnapshot.wakeReason === "issue_comment_mentioned") return null; + if (issueId) { + const conversation = await getIssueExecutionContext(agent.companyId, issueId); + if (taskRetryActorId && conversation) await assertTaskRetryConversationAllowed(db, conversation, taskRetryActorId); + if (reason === "issue_children_completed" && conversation?.originKind === "onboarding_first_task") enrichedContextSnapshot.onboardingCompletion = true; + if (isConversation(conversation)) { + if (opts.manualUserWake && conversation!.conversationUserId !== opts.requestedByActorId) { + throw new HttpError(403, "Only the conversation owner can start a chat run"); + } + if (isConversationExecutionWake(conversation, reason ?? readNonEmptyString(enrichedContextSnapshot.wakeReason))) return null; + if (agent.id !== conversation!.conversationAgentId) return null; + if (!(await instanceSettings.getExperimental()).enableAgentChat) return null; + if (!wakeCommentId && !opts.taskRetryRequestId && isWaitingConversation(conversation) && !hasInteractionContinuationWakeContext(enrichedContextSnapshot) && reason !== CHAT_COMPLETION_WAKE_REASON) return null; + } + } + if (agent.adapterType === "paperclip_runner") { + const oldConfig = parseObject(agent.adapterConfig); + const nextConfig = normalizeLegacyRunnerProvider(oldConfig); + if (nextConfig !== oldConfig) { + await agentService(db).update( + agent.id, + { adapterConfig: nextConfig }, + { + recordRevision: { + source: "normalize_runner_provider", + createdByAgentId: null, + createdByUserId: null, + }, + }, + ); + await logActivity(db, { + companyId: agent.companyId, + actorType: "system", + actorId: "heartbeat", + action: "agent.updated", + entityType: "agent", + entityId: agent.id, + details: { provider: "codex", reason: "native_codex_provider" }, + }); + agent = (await getAgent(agentId))!; + } + } + + if (opts.failedRunId) { + const failed = await getRun(opts.failedRunId, { includeExecutionEvidence: true }); + if (opts.requestedByActorType !== "user" || !opts.requestedByActorId || + reason !== "retry_failed_run" || source !== "on_demand" || triggerDetail !== "manual" || + !failed || failed.companyId !== agent.companyId || failed.agentId !== agentId || + !(await canRetryStoppedRun(db, failed)) || + (failed.nativeIssueId ?? readNonEmptyString(failed.contextSnapshot?.issueId)) !== issueId) { + throw conflict("The selected failed run cannot be retried for this task."); + } + if (!activeRunExecutions.has(failed.id) && !adapterExecutionControls.has(failed.id)) { + await sweepPendingCleanupLeases({ explicitRetry: { + companyId: failed.companyId, runId: failed.id, actorId: opts.requestedByActorId, + } }); + } + if (isConversationAdapter(agent.adapterType) || agent.adapterType === "paperclip_runner") { + enrichedContextSnapshot.previousRunId = failed.id; + enrichedContextSnapshot.forceFreshSession = true; + } + } + + const caseRequest = opts.durableCaseHandoffRequest; + const waitForCaseAdmission = (reason: string, message: string): never => { + // The case ledger is the outbox. Roll back this admission attempt rather + // than consume its one receipt ID in a generic skipped/deferred queue. + throw conflict(message, { code: "case_handoff_admission_wait", reason }); + }; + let caseReceipt: { id: string; requestedAt: Date } | undefined; + if (caseRequest) { + if (opts.taskRetryRequestId || opts.durableChatRequest || opts.durableDotRequest || opts.durableMuseRequest || source !== "assignment" || reason !== "case_handoff") + throw conflict("Invalid case handoff wake authority"); + const [handoff] = await db.select().from(caseHandoffs).where(and(eq(caseHandoffs.id, caseRequest.handoffId), + eq(caseHandoffs.companyId, agent.companyId), eq(caseHandoffs.issueId, caseRequest.issueId), + eq(caseHandoffs.toAgentId, agentId), eq(caseHandoffs.wakeupRequestId, caseRequest.wakeupRequestId), + inArray(caseHandoffs.state, ["pending_dispatch", "delivered"]))).limit(1); + if (!handoff || issueId !== handoff.issueId || caseRequest.expectedIssueStatusVersion !== handoff.appliedIssueStatusVersion) + throw conflict("Case handoff no longer matches its task assignment"); + caseReceipt = { id: handoff.wakeupRequestId, requestedAt: handoff.createdAt }; + opts = { ...opts, allowRunCoalescing: false, idempotencyKey: `case-handoff:${handoff.id}`, + issueStateGuard: { assigneeAgentId: agentId, statuses: ["todo", "in_progress"], statusVersion: caseRequest.expectedIssueStatusVersion } }; + } + if (durableRequest) { + assertDurableChatWakeupRequest(durableRequest, { + agentId, + companyId: agent.companyId, + issueId, + commentId: wakeCommentId ?? null, + requestedByActorType: opts.requestedByActorType, + requestedByActorId: opts.requestedByActorId, + }); + opts = { ...opts, idempotencyKey: durableRequest.idempotencyKey }; + } + if ( + Object.hasOwn(enrichedContextSnapshot, "chatFailedRunRetry") && + !durableRequest?.failedRunRetry + ) { + throw new FailedChatRunRetryAuthorizationError(); + } + if (durableRequest?.failedRunRetry) { + opts = { ...opts, allowRunCoalescing: false }; + } + const dotRequest = opts.durableDotRequest; + const museRequest = opts.durableMuseRequest; + if (dotRequest && (durableRequest || museRequest || dotRequest.agentId !== agentId || + dotRequest.companyId !== agent.companyId || dotRequest.issueId !== issueId || + dotRequest.requestId !== payload?.dotRequestId || source !== "assignment" || + opts.requestedByActorType !== "agent" || opts.requestedByActorId !== agentId || + agent.adapterType !== "paperclip_runner" || !["openai_dot", "grok_bot"].includes(String(parseObject(agent.adapterConfig).provider)) || + opts.idempotencyKey !== dotRequest.idempotencyKey)) { + throw conflict("Dot work request does not match its admission authority."); + } + if (museRequest && (durableRequest || dotRequest || museRequest.agentId !== agentId || + museRequest.companyId !== agent.companyId || museRequest.issueId !== issueId || + museRequest.requestId !== payload?.museRequestId || source !== "assignment" || + opts.requestedByActorType !== "agent" || opts.requestedByActorId !== agentId || + agent.adapterType !== "paperclip_runner" || parseObject(agent.adapterConfig).provider !== "muse" || + parseObject(agent.adapterConfig).museBindingId !== museRequest.bindingId || + !Number.isSafeInteger(museRequest.bindingGeneration) || museRequest.bindingGeneration < 1 || + payload?.museBindingId !== museRequest.bindingId || payload?.museBindingGeneration !== museRequest.bindingGeneration || + payload?.museRequestDigest !== museRequest.requestDigest || !/^sha256:[a-f0-9]{64}$/.test(museRequest.requestDigest) || + opts.idempotencyKey !== museRequest.idempotencyKey)) { + throw conflict("Muse work request does not match its admission authority."); + } + const receiptRequest = durableRequest ?? dotRequest ?? museRequest ?? caseReceipt; + const durableReceiptFields = receiptRequest + ? { id: receiptRequest.id, requestedAt: receiptRequest.requestedAt } + : {}; + const existingDurableReceipt = async (queryDb: Db) => { + if (!receiptRequest) return null; + const receipt = await queryDb + .select() + .from(agentWakeupRequests) + .where(eq(agentWakeupRequests.id, receiptRequest.id)) + .limit(1) + .then((rows) => rows[0] ?? null); + if (receipt && durableRequest) assertDurableChatWakeupReceipt(durableRequest, receipt); + if (receipt && caseRequest && (receipt.companyId !== agent.companyId || receipt.agentId !== agentId || + receipt.payload?.issueId !== caseRequest.issueId || receipt.idempotencyKey !== `case-handoff:${caseRequest.handoffId}`)) + throw conflict("Case handoff wake receipt identity changed"); + if (receipt && dotRequest && (receipt.companyId !== dotRequest.companyId || + receipt.agentId !== dotRequest.agentId || receipt.source !== "assignment" || + receipt.requestedByActorType !== "agent" || receipt.requestedByActorId !== dotRequest.agentId || + receipt.idempotencyKey !== dotRequest.idempotencyKey || receipt.payload?.issueId !== dotRequest.issueId || + receipt.payload?.dotRequestId !== dotRequest.requestId)) { + throw conflict("requestId was reused for another task."); + } + if (receipt && museRequest && (receipt.companyId !== museRequest.companyId || + receipt.agentId !== museRequest.agentId || receipt.source !== "assignment" || + receipt.requestedByActorType !== "agent" || receipt.requestedByActorId !== museRequest.agentId || + receipt.idempotencyKey !== museRequest.idempotencyKey || receipt.payload?.issueId !== museRequest.issueId || + receipt.payload?.museRequestId !== museRequest.requestId || receipt.payload?.museBindingId !== museRequest.bindingId || + receipt.payload?.museBindingGeneration !== museRequest.bindingGeneration || receipt.payload?.museRequestDigest !== museRequest.requestDigest)) { + throw conflict("Request ID reused with changed input."); + } + if (receipt && durableRequest && receipt.status === "deferred_issue_execution" && !receipt.runId && + ((durableRequest.failedRunRetry && opts.taskRetryRequestId && + receipt.payload?.taskRetryRequestId === opts.taskRetryRequestId) || + (durableRequest.resumeDeferred && !durableRequest.failedRunRetry && + receipt.payload?.issueId === durableRequest.issueId && + queuedCommentIdsFromWakePayload(receipt.payload).includes(durableRequest.commentId)))) { + // The live inbound owner or a bound Retry may reconsider the same saved receipt. + // The issue lock, live chat authority and execution holds below still + // decide whether it can dispatch; this never creates another wake. + executionWaitRequestId = receipt.id; + return null; + } + return receipt; + }; + const priorReceipt = await existingDurableReceipt(db); + if (priorReceipt) { + // Replaying an admission receipt is not fresh authority to dispatch it. + // The normal queue owns dispatch and its current execution-policy checks. + return priorReceipt.runId ? getRun(priorReceipt.runId) : null; + } + + const agentDebug = parseObject(parseObject(agent.runtimeConfig).debug); + const runDebug = parseObject(enrichedContextSnapshot.debug); + if ( + agentDebug.providerTrace === "raw" && + runDebug.providerTrace !== "raw" + ) { + enrichedContextSnapshot.debug = { + ...runDebug, + providerTrace: "raw", + }; + enrichedContextSnapshot.providerTraceRequestedBy = `agent:${agent.id}:debug-setting`; + enrichedContextSnapshot.providerTraceRequestSource = + "agent_debug_setting"; + } + + // Automatic signals are replaceable; user input and interaction delivery + // retain distinct durable receipts even when the same gate blocks them. + const coalesceExecutionWait = + opts.requestedByActorType === "system" && + !receiptRequest && + !wakeCommentId && + queuedCommentIdsFromRunContext(enrichedContextSnapshot).length === 0 && + !isInteractionResolutionWakePayload(payload ?? {}) && + !hasInteractionContinuationWakeContext(enrichedContextSnapshot); + const writeSkippedRequest = async ( + skipReason: string, + patch: Partial = {}, + waitCondition?: Record, + ) => { + // A temporary refusal must leave the handoff retryable, without burning + // its one durable receipt ID on a skipped wake. + if (caseRequest) return { created: false }; + if (executionWaitRequestId) { + await db.update(agentWakeupRequests).set({ + payload: sql`jsonb_set(coalesce(${agentWakeupRequests.payload}, '{}'::jsonb), '{executionWait}', + coalesce(${agentWakeupRequests.payload}->'executionWait', '{}'::jsonb) || ${JSON.stringify({ + reason: skipReason, message: patch.error ?? (skipReason === "issue_tree_hold_active" + ? "This task is paused. Resume it to send your saved message." + : "Waiting for task execution to be enabled. Your message is saved."), + })}::jsonb)`, + updatedAt: new Date(), + }).where(and(eq(agentWakeupRequests.id, executionWaitRequestId), + eq(agentWakeupRequests.companyId, agent.companyId), eq(agentWakeupRequests.agentId, agentId), + eq(agentWakeupRequests.status, "deferred_issue_execution"))); + return { created: false }; + } + const request = { + ...durableReceiptFields, + companyId: agent.companyId, + agentId, + source, + triggerDetail, + reason: skipReason, + payload, + status: "skipped", + requestedByActorType: opts.requestedByActorType ?? null, + requestedByActorId: opts.requestedByActorId ?? null, + idempotencyKey: opts.idempotencyKey ?? null, + finishedAt: new Date(), + ...patch, + }; + if (waitCondition && issueId && isUuidLike(issueId)) { + const waitIssueId = issueId; + return db.transaction(async (tx) => { + await tx.execute(sql`select id from issues where id = ${waitIssueId} and company_id = ${agent.companyId} for update`); + if (durableRequest) await notifyChatInboundWakeNotices(tx, { + companyId: durableRequest.companyId, issueId: durableRequest.issueId, + }); + return recordExecutionWait(tx as unknown as Db, { + issueId: waitIssueId, request, condition: waitCondition, coalesce: coalesceExecutionWait, + }); + }); + } + if (durableRequest) { + await db.transaction(async tx => { + await notifyChatInboundWakeNotices(tx, { + companyId: durableRequest.companyId, issueId: durableRequest.issueId, + }); + await tx.insert(agentWakeupRequests).values(request); + }); + } else { + await db.insert(agentWakeupRequests).values(request); + } + return { created: true }; + }; + const writeSkippedHeartbeatRequest = async ( + skipReason: string, + details: Record, + ) => { + await writeSkippedRequest(skipReason, { + payload: { + ...(payload ?? {}), + heartbeatSkip: details, + }, + }); + }; + + const schedulingSuppression = await getSchedulingSuppression(); + // A task drain holds ADMISSION, not the request. The drain is a + // process-local pre-restart hold, so a wake that arrives while it is + // active still names real work that must run once the process comes + // back: leave it in the durable queue and let the dispatch-side checks + // (startNextQueuedRunForAgent / executeRun) keep it from starting until + // the drain lifts or the restart clears it. Writing it as `skipped` + // here dropped the wake permanently — an accepted plan whose + // continuation wake landed mid-drain left its issue in `todo` with no + // run and no path until a person noticed. + if ( + schedulingSuppression.suppressed && + schedulingSuppression.reason !== "task_drain" + ) { + await writeSkippedHeartbeatRequest("heartbeat.scheduling_suppressed", { + reason: schedulingSuppression.reason, + }); + return null; + } + + const worktreeExecutionCutoff = + opts.requestedByActorType === "user" + ? null + : await getWorktreeExecutionCutoff(); + + const company = await db + .select({ status: companies.status }) + .from(companies) + .where(eq(companies.id, agent.companyId)) + .then((rows) => rows[0] ?? null); + + if (!company || company.status !== "active") { + if (caseRequest) return null; + const companyStatus = company?.status ?? "missing"; + if (opts.requestedByActorType === "user") { + throw conflict("Company is not active", { status: companyStatus }); + } + await writeSkippedRequest("company.inactive", { + error: `Wake suppressed because company status is ${companyStatus}`, + }, { companyStatus }); + return null; + } + + const explicitResumeSession = await resolveExplicitResumeSessionOverride( + agent, + payload, + taskKey, + ); + if (explicitResumeSession) { + enrichedContextSnapshot.resumeFromRunId = + explicitResumeSession.resumeFromRunId; + enrichedContextSnapshot.resumeSessionDisplayId = + explicitResumeSession.sessionDisplayId; + enrichedContextSnapshot.resumeSessionParams = + explicitResumeSession.sessionParams; + if ( + !readNonEmptyString(enrichedContextSnapshot.issueId) && + explicitResumeSession.issueId + ) { + enrichedContextSnapshot.issueId = explicitResumeSession.issueId; + } + if ( + !readNonEmptyString(enrichedContextSnapshot.taskId) && + explicitResumeSession.taskId + ) { + enrichedContextSnapshot.taskId = explicitResumeSession.taskId; + } + if ( + !readNonEmptyString(enrichedContextSnapshot.taskKey) && + explicitResumeSession.taskKey + ) { + enrichedContextSnapshot.taskKey = explicitResumeSession.taskKey; + } + issueId = readNonEmptyString(enrichedContextSnapshot.issueId) ?? issueId; + } + const effectiveTaskKey = + readNonEmptyString(enrichedContextSnapshot.taskKey) ?? taskKey; + const genericTimerWake = + source === "timer" && + !issueId && + !wakeCommentId && + !readNonEmptyString(enrichedContextSnapshot.taskId) && + !effectiveTaskKey; + // Old or direct timer producers cannot create a doomed taskless Runner + // run. Explicit session restoration above can supply a real task scope. + if (genericTimerWake && !supportsUnscopedTimerHeartbeat(agent)) return null; + const sessionBefore = + explicitResumeSession?.sessionDisplayId ?? + (await resolveSessionBeforeForWakeup(agent, effectiveTaskKey)); + let hasResolvablePriorSessionWorkspace: boolean | null = null; + const resolveHasResolvablePriorSessionWorkspace = async () => { + if (hasResolvablePriorSessionWorkspace !== null) + return hasResolvablePriorSessionWorkspace; + hasResolvablePriorSessionWorkspace = issueId + ? await hasResolvablePriorSessionWorkspaceForWake({ + agent, + contextSnapshot: enrichedContextSnapshot, + taskKey: effectiveTaskKey, + explicitResumeSession, + }) + : false; + return hasResolvablePriorSessionWorkspace; + }; + const continuationAttempt = readContinuationAttempt( + enrichedContextSnapshot.livenessContinuationAttempt, + ); + + let projectId = readNonEmptyString(enrichedContextSnapshot.projectId); + if (!projectId && issueId) { + // Look up by either UUID or identifier (e.g. "ENV-13"), but always scope + // by companyId so a row from another tenant can never be returned even + // when identifiers collide across companies. Guard the UUID arm because + // issues.id is a Postgres uuid column — passing "ENV-13" into eq(issues.id, …) + // would fail with an invalid-input-syntax cast error before the OR is + // evaluated. + const lookupIsUuid = isUuidLike(issueId); + const idMatch = lookupIsUuid + ? or( + eq(issues.id, issueId), + eq(issues.identifier, issueId.toUpperCase()), + ) + : eq(issues.identifier, issueId.toUpperCase()); + const resolvedIssue = await db + .select({ + id: issues.id, + projectId: issues.projectId, + createdAt: issues.createdAt, + }) + .from(issues) + .where(and(eq(issues.companyId, agent.companyId), idMatch)) + .then((rows) => rows[0] ?? null); + if (resolvedIssue) { + if ( + worktreeExecutionCutoff && + resolvedIssue.createdAt < worktreeExecutionCutoff + ) { + await writeSkippedHeartbeatRequest( + "heartbeat.worktree_execution_cutoff", + { + reason: "worktree_execution_cutoff", + cutoff: worktreeExecutionCutoff.toISOString(), + issueId: resolvedIssue.id, + }, + ); + return null; + } + projectId = resolvedIssue.projectId ?? null; + // Canonicalize context to the UUID so downstream lookups always use UUID + if (resolvedIssue.id !== issueId) { + issueId = resolvedIssue.id; + enrichedContextSnapshot.issueId = issueId; + if (readNonEmptyString(enrichedContextSnapshot.taskId)) { + enrichedContextSnapshot.taskId = issueId; + } + } + } + } + // Propagate projectId into context so resolveWorkspaceForRun can bind the + // project workspace even when context.projectId wasn't set by the caller. + if (projectId && !readNonEmptyString(enrichedContextSnapshot.projectId)) { + enrichedContextSnapshot.projectId = projectId; + } + const isolatedWorkspacesEnabled = issueId + ? (await instanceSettings.getExperimental()).enableIsolatedWorkspaces + : false; + let operatorResponsibleUserId: string | null = opts.manualUserWake ? opts.requestedByActorId! : null; + let queuedResponsibleUserIdPromise: Promise | null = null; + const resolveQueuedResponsibleUserId = () => { + if (operatorResponsibleUserId) return Promise.resolve(operatorResponsibleUserId); + queuedResponsibleUserIdPromise ??= (async () => { + const queuedIssueContext = issueId + ? await getIssueExecutionContext(agent.companyId, issueId) + : null; + const queuedRoutineEnvContext = await getRoutineEnvForExecutionIssue( + agent.companyId, + queuedIssueContext, + ); + const queuedResponsibleUserId = + await resolveResponsibleUserIdForRunSeed({ + companyId: agent.companyId, + contextSnapshot: enrichedContextSnapshot, + issueContext: queuedIssueContext, + routineEnvContext: queuedRoutineEnvContext, + requestedByActorType: opts.requestedByActorType ?? null, + requestedByActorId: opts.requestedByActorId ?? null, + source, + triggerDetail, + }); + if (!queuedResponsibleUserId) { + throw new HttpError( + 422, + "Unable to resolve responsible user for heartbeat run dispatch", + { + code: "responsible_user_unresolved", + agentId, + companyId: agent.companyId, + issueId: issueId ?? null, + source, + triggerDetail, + wakeReason: readNonEmptyString( + enrichedContextSnapshot.wakeReason, + ), + }, + ); + } + return queuedResponsibleUserId; + })(); + return queuedResponsibleUserIdPromise; + }; + + const budgetBlock = await budgets.getInvocationBlock( + agent.companyId, + agentId, + { + issueId, + projectId, + }, + ); + if (budgetBlock) { + await writeSkippedRequest("budget.blocked", { error: budgetBlock.reason }, { + scopeType: budgetBlock.scopeType, scopeId: budgetBlock.scopeId, + }); + if (caseRequest) return null; + throw conflict(budgetBlock.reason, { + scopeType: budgetBlock.scopeType, + scopeId: budgetBlock.scopeId, + }); + } + + // Setup delays execution, but must not discard work accepted by a caller. + const invokability = await getAgentInvokability(isAgentAwaitingSetup(agent) ? { ...agent, status: "idle" } : agent); + if (!invokability.invokable) { + if (caseRequest) return null; + if (opts.requestedByActorType !== "user" || executionWaitRequestId) { + await writeSkippedRequest("agent.not_invokable", { + error: invokability.message, + }, { status: agent.status, reason: invokability.reason }); + } + throw conflict(invokability.message, { + status: agent.status, + reason: invokability.reason, + invalidOrgChain: invokability.invalidOrgChain, + ...invokability.details, + }); + } + + const policy = parseHeartbeatPolicy(agent); + + if (source === "timer" && !policy.enabled) { + await writeSkippedRequest("heartbeat.disabled", {}, { enabled: false }); + return null; + } + if (source !== "timer" && !policy.wakeOnDemand) { + await writeSkippedRequest("heartbeat.wakeOnDemand.disabled", {}, { wakeOnDemand: false }); + return null; + } + + if ( + policy.skipTimerWhenNoActionableWork && + genericTimerWake && + !(await hasActionableTimerWork(agent)) + ) { + await writeSkippedHeartbeatRequest("heartbeat.timer.no_actionable_work", { + reason: + "No assigned todo or in_progress issue requires this agent before timer adapter invocation.", + }); + await markTimerHeartbeatChecked(agentId, source); + return null; + } + + if (issueId) { + const activePauseHold = await treeControlSvc.getActivePauseHoldGate( + agent.companyId, + issueId, + ); + if (activePauseHold) { + const treeHoldInteractionWake = + await isVerifiedIssueTreeControlInteractionWake(db, { + companyId: agent.companyId, + issueId, + agentId, + contextSnapshot: enrichedContextSnapshot, + requestedByActorType: opts.requestedByActorType, + requestedByActorId: opts.requestedByActorId, + }); + + if (!treeHoldInteractionWake) { + const wait = await writeSkippedRequest("issue_tree_hold_active", {}, { + holdId: activePauseHold.holdId, + }); + if (wait.created) await logActivity(db, { + companyId: agent.companyId, + actorType: "system", + actorId: "system", + agentId, + runId: null, + action: "issue.tree_hold_wakeup_deferred", + entityType: "issue", + entityId: issueId, + details: { + holdId: activePauseHold.holdId, + rootIssueId: activePauseHold.rootIssueId, + requestedReason: reason, + source, + triggerDetail, + securityPrinciples: [ + "Complete Mediation", + "Fail Securely", + "Secure Defaults", + ], + }, + }); + return null; + } + + enrichedContextSnapshot.treeHoldInteraction = true; + enrichedContextSnapshot.activeTreeHold = { + holdId: activePauseHold.holdId, + rootIssueId: activePauseHold.rootIssueId, + mode: activePauseHold.mode, + reason: activePauseHold.reason, + releasePolicy: activePauseHold.releasePolicy, + interaction: true, + }; + } + } + + if (issueId) { + // Mention-triggered wakes can request input from another agent, but they must + // still respect the issue execution lock so a second agent cannot start on the + // same issue workspace while the assignee already has a live run. + const agentNameKey = normalizeAgentNameKey(agent.name); + + const cancelledRunsToEmit: (typeof heartbeatRuns.$inferSelect)[] = []; + + const outcome = await db + .transaction(withDurableChatAdmission(durableRequest, async (tx) => { + await tx.execute( + sql`select id from issues where id = ${issueId} and company_id = ${agent.companyId} for update`, + ); + + if (executionWaitRequestId) { + const [pending] = await tx.select().from(agentWakeupRequests).where(and( + eq(agentWakeupRequests.id, executionWaitRequestId), eq(agentWakeupRequests.companyId, agent.companyId), + eq(agentWakeupRequests.agentId, agentId), eq(agentWakeupRequests.status, "deferred_issue_execution"), + // A user message can join a queue originally created by a + // system wake. Admission validates the saved user comment or board click. + (opts.queuedCommentInterruptId ?? opts.queuedCommentRequestId) === executionWaitRequestId + ? undefined : eq(agentWakeupRequests.requestedByActorType, + durableRequest ? opts.requestedByActorType ?? "" : "user"), + opts.queuedCommentInterruptId === executionWaitRequestId + ? sql`${agentWakeupRequests.payload}->'queuedCommentInterrupt'->>'actorId' = ${opts.requestedByActorId ?? ""}` + : opts.queuedCommentRequestId === executionWaitRequestId ? undefined + : eq(agentWakeupRequests.requestedByActorId, opts.requestedByActorId ?? ""), + sql`${agentWakeupRequests.payload}->>'issueId' = ${issueId}`, + )); + // The issue lock serializes cleanup callbacks and periodic workers. + // An adopted, discarded, or edited receipt is no longer authority. + if (!pending || (!opts.taskRetryRequestId && !(wakeCommentId && queuedCommentIdsFromWakePayload(pending.payload).includes(wakeCommentId)) && + !(opts.queuedCommentInterruptId && await readQueuedInteractionResponse(tx as unknown as Db, + agent.companyId, issueId, pending.payload)))) { + return { kind: "deferred" as const }; + } + if (opts.taskRetryRequestId && durableRequest?.failedRunRetry) { + if (pending.id !== durableRequest.id || pending.payload?.taskRetryRequestId !== opts.taskRetryRequestId) + return { kind: "deferred" as const }; + // The parent board Retry is checked separately below after the + // connected conversation reauthorizes this exact child receipt. + } else if (opts.taskRetryRequestId) { + const retry = readTaskRetryIntent(pending.payload); + if (pending.id !== opts.taskRetryRequestId || retry?.generationId !== pending.id || retry.chatActionId || + retry.actorId !== opts.requestedByActorId || retry.sourceRunId !== enrichedContextSnapshot.previousRunId) + return { kind: "deferred" as const }; + if (!await taskRetryMutationAllowed(tx as unknown as Db, { companyId: agent.companyId, issueId, + actorId: retry.actorId, actorSource: retry.actorSource })) return { kind: "deferred" as const }; + if (retry.boardMessageOrigin) { + const origin = await readTaskRetryBoardMessageOrigin(tx as unknown as Db, { companyId: agent.companyId, + issueId, agentId, sourceRunId: retry.sourceRunId, actorId: retry.actorId }); + if (!origin || origin.commentId !== retry.boardMessageOrigin.commentId || + origin.identityContextId !== retry.boardMessageOrigin.identityContextId) return { kind: "deferred" as const }; + } + } + if (opts.queuedCommentRequestId) { + const ids = await undeliveredLegacyUserCommentIds(tx as unknown as Db, + agent.companyId, issueId, agentId, queuedCommentIdsFromWakePayload(pending.payload)); + if (!wakeCommentId || !ids.includes(wakeCommentId)) return { kind: "deferred" as const }; + pending.payload = withQueuedCommentIdsInWakePayload(parseObject(pending.payload), ids); + await tx.update(agentWakeupRequests).set({ payload: pending.payload }).where(and( + eq(agentWakeupRequests.id, pending.id), eq(agentWakeupRequests.companyId, agent.companyId), + )); + } + if (!opts.queuedCommentInterruptId && !opts.queuedCommentRequestId && pending.payload?.manualUserWake === true) { + // A persisted manual wake keeps its actor when an execution wait + // resumes. The locked receipt above has revalidated that actor. + payload = { ...payload, manualUserWake: true }; + operatorResponsibleUserId = opts.requestedByActorId!; + } + if (opts.queuedCommentInterruptId) { + // The locked board receipt supplies execution authority even when + // another user authored the messages. Dispatch revalidates the receipt. + operatorResponsibleUserId = opts.requestedByActorId!; + } + if (opts.queuedCommentInterruptId || opts.queuedCommentRequestId) { + // Edits/discards between the click and dispatch remain authoritative. + Object.assign(enrichedContextSnapshot, withQueuedCommentIdsInRunContext( + enrichedContextSnapshot, queuedCommentIdsFromWakePayload(pending.payload), + )); + } + } + let automaticParentRunId: string | null = null; + if ( + source === "automation" && + opts.requestedByActorType === "system" + ) { + const nativeParent = await readChatControlNativeParent( + tx as unknown as Db, + { companyId: agent.companyId, issueId, agentId }, + opts.requestedByActorId ?? null, + ); + const genericParent = + !opts.requestedByActorId && reason === "issue_continuation_needed" + ? readNonEmptyString(enrichedContextSnapshot.retryOfRunId) + : null; + automaticParentRunId = + nativeParent.kind === "parent" + ? nativeParent.runId + : genericParent; + if (automaticParentRunId) { + const [parent] = await tx + .select({ + agentId: heartbeatRuns.agentId, + context: heartbeatRuns.contextSnapshot, + }) + .from(heartbeatRuns) + .where( + and( + eq(heartbeatRuns.companyId, agent.companyId), + eq(heartbeatRuns.id, automaticParentRunId), + ), + ) + .limit(1); + if ( + parent && + (parseObject(parent.context).issueId ?? + parseObject(parent.context).taskId) === issueId && + parent.agentId !== agentId + ) + automaticParentRunId = null; + } + const proof = + nativeParent.kind === "unresolved" + ? { kind: "unresolved" as const } + : automaticParentRunId + ? await readChatControlRecoveryStop( + tx as unknown as Db, + { + companyId: agent.companyId, + issueId, + agentId, + sourceRunId: automaticParentRunId, + }, + true, + ) + : { kind: "clear" as const }; + if (proof.kind !== "clear") { + await tx + .insert(agentWakeupRequests) + .values({ + companyId: agent.companyId, + agentId, + source, + triggerDetail, + reason: + proof.kind === "stopped" + ? CHAT_CONTROL_RECOVERY_STOP_CODE + : CHAT_CONTROL_RECOVERY_UNRESOLVED_CODE, + payload: { issueId }, + status: "skipped", + requestedByActorType: "system", + requestedByActorId: opts.requestedByActorId ?? null, + idempotencyKey: opts.idempotencyKey ?? null, + finishedAt: new Date(), + }); + return { kind: "skipped" as const }; + } + } + + const durableReceipt = await existingDurableReceipt( + tx as unknown as Db, + ); + if (durableReceipt) + return { kind: "durable" as const, receipt: durableReceipt }; + if (caseRequest) { + const [handoff] = await tx.select().from(caseHandoffs).where(and(eq(caseHandoffs.id, caseRequest.handoffId), + eq(caseHandoffs.companyId, agent.companyId), + eq(caseHandoffs.issueId, caseRequest.issueId), eq(caseHandoffs.state, "pending_dispatch"), + eq(caseHandoffs.toAgentId, agentId), eq(caseHandoffs.wakeupRequestId, caseRequest.wakeupRequestId))).for("update"); + if (!handoff || handoff.appliedIssueStatusVersion !== caseRequest.expectedIssueStatusVersion) + throw conflict("Case handoff delivery was superseded"); + } + const failedChatRetry = await authorizeFailedChatRunRetryWake( + db, + tx as unknown as Db, + { + phase: "admission", + wakeupRequestId: durableRequest?.id ?? null, + companyId: agent.companyId, + agentId, + issueId, + contextSnapshot: enrichedContextSnapshot, + }, + ); + if ( + failedChatRetry !== Boolean(durableRequest?.failedRunRetry) || + (failedChatRetry && + durableRequest?.failedRunRetry?.failedRunId !== + readNonEmptyString(enrichedContextSnapshot.retryOfRunId)) + ) { + throw new FailedChatRunRetryAuthorizationError(); + } + if (durableRequest) { + if (issueId !== durableRequest.issueId) + throw new Error("chat_inbound_wakeup_binding_denied"); + await durableRequest.authorize(tx as unknown as Db); + if (opts.taskRetryRequestId) { + const [retryWake] = await tx.select().from(agentWakeupRequests).where(and( + eq(agentWakeupRequests.companyId, agent.companyId), eq(agentWakeupRequests.id, opts.taskRetryRequestId), + eq(agentWakeupRequests.agentId, agentId), eq(agentWakeupRequests.status, "deferred_issue_execution"), + )).for("update"); + const retry = readTaskRetryIntent(retryWake?.payload); + if (!failedChatRetry || retry?.chatActionId !== durableRequest.id || + retry.actorId !== taskRetryActorId || retry.sourceRunId !== durableRequest.failedRunRetry?.failedRunId) + throw new FailedChatRunRetryAuthorizationError(); + if (!await taskRetryMutationAllowed(tx as unknown as Db, { companyId: agent.companyId, issueId, + actorId: retry.actorId, actorSource: retry.actorSource })) return { kind: "deferred" as const }; + } + } else { + if ( + source === "on_demand" && + triggerDetail === "manual" && + reason === "retry_failed_run" + ) { + // Generic Board retry does not carry server-authorized failed-run + // lineage. Caller-supplied source/comment/retry markers cannot + // restore it, including for a retired conversation generation. + const [chatBinding] = await tx + .select({ id: chatConversations.id }) + .from(chatConversations) + .innerJoin(chatEndpoints, eq(chatEndpoints.id, chatConversations.endpointId)) + .where( + and( + eq(chatEndpoints.externalExecutionPolicy, "restricted"), + eq(chatConversations.companyId, agent.companyId), + eq(chatConversations.issueId, issueId), + ), + ) + .limit(1); + if (chatBinding) { + throw conflict( + "Retry needs the exact failed chat request and current access. Send the request again in the current connected conversation.", + { code: "chat_failed_run_retry_requires_authorized_context" }, + ); + } + } + const [held] = await tx + .select({ id: issues.id }) + .from(issues) + .where( + and( + eq(issues.id, issueId), + eq(issues.companyId, agent.companyId), + unadmittedChatWakeupCondition(issues.id, issues.companyId), + ), + ) + .limit(1); + if (held) { + throw conflict( + "This task has external chat input that has not been admitted. Check the connection's Activity; let pending input finish, or restore access and send a new authorized message before starting the task.", + { code: "chat_inbound_wakeup_unadmitted", issueId }, + ); + } + } + + const issue = await tx + .select({ + id: issues.id, + companyId: issues.companyId, + identifier: issues.identifier, + conversationAgentId: issues.conversationAgentId, + conversationUserId: issues.conversationUserId, + conversationState: issues.conversationState, + status: issues.status, + statusVersion: issues.statusVersion, + projectId: issues.projectId, + projectWorkspaceId: issues.projectWorkspaceId, + executionWorkspaceId: issues.executionWorkspaceId, + executionWorkspacePreference: issues.executionWorkspacePreference, + executionWorkspaceSettings: issues.executionWorkspaceSettings, + assigneeAgentId: issues.assigneeAgentId, + assigneeUserId: issues.assigneeUserId, + monitorNextCheckAt: issues.monitorNextCheckAt, + monitorWakeRequestedAt: issues.monitorWakeRequestedAt, + executionRunId: issues.executionRunId, + executionAgentNameKey: issues.executionAgentNameKey, + createdAt: issues.createdAt, + }) + .from(issues) + .where( + and( + eq(issues.id, issueId), + eq(issues.companyId, agent.companyId), + ), + ) + .then((rows) => rows[0] ?? null); + + if (!issue) { + if (caseRequest) throw conflict("Case handoff task no longer exists"); + await tx.insert(agentWakeupRequests).values({ + ...durableReceiptFields, + companyId: agent.companyId, + agentId, + source, + triggerDetail, + reason: "issue_execution_issue_not_found", + payload, + status: "skipped", + requestedByActorType: opts.requestedByActorType ?? null, + requestedByActorId: opts.requestedByActorId ?? null, + idempotencyKey: opts.idempotencyKey ?? null, + finishedAt: new Date(), + }); + return { kind: "skipped" as const }; + } + + if (taskRetryActorId) { + await assertTaskRetryConversationAllowed(tx as unknown as Db, issue, taskRetryActorId); + if (await retireSupersededTaskRetry(tx as unknown as Db, { + companyId: issue.companyId, issueId: issue.id, requestId: opts.taskRetryRequestId!, + sourceRunId: readNonEmptyString(enrichedContextSnapshot.previousRunId)!, + })) return { kind: "deferred" as const }; + } + const issueStateGuard = opts.issueStateGuard; + const activeMonitorRun = issueStateGuard?.monitorNextCheckAt === undefined ? null + : await tx.select({ id: heartbeatRuns.id }).from(heartbeatRuns).where(and( + eq(heartbeatRuns.companyId, issue.companyId), eq(heartbeatRuns.nativeIssueId, issue.id), + eq(heartbeatRuns.runtimeMode, "native"), inArray(heartbeatRuns.status, ["queued", "running", "scheduled_retry"]), + )).limit(1).then(rows => rows[0] ?? null); + if ( + issueStateGuard && + (!issueStateGuard.statuses.includes(issue.status) || + issue.assigneeAgentId !== issueStateGuard.assigneeAgentId || + (issueStateGuard.statusVersion !== undefined && issue.statusVersion !== issueStateGuard.statusVersion) || + (issueStateGuard.monitorNextCheckAt !== undefined && ( + activeMonitorRun !== null || issue.assigneeUserId !== null || + issue.monitorNextCheckAt?.toISOString() !== issueStateGuard.monitorNextCheckAt || + issue.monitorWakeRequestedAt?.toISOString() !== issueStateGuard.monitorWakeRequestedAt + ))) + ) { + if (caseRequest) throw conflict("Case handoff task changed before admission"); + // A deferred monitor retains its schedule; do not create a receipt + // that could suppress its next admission attempt. + if (issueStateGuard.monitorNextCheckAt !== undefined) return { kind: "skipped" as const }; + await tx.insert(agentWakeupRequests).values({ + ...durableReceiptFields, + companyId: agent.companyId, + agentId, + source, + triggerDetail, + reason: "issue_state_guard_mismatch", + payload: { + ...(payload ?? {}), + heartbeatSkip: { + reason: + "Issue status or assignee changed before the wake could be queued.", + issueId: issue.id, + expectedStatuses: issueStateGuard.statuses, + actualStatus: issue.status, + expectedAssigneeAgentId: issueStateGuard.assigneeAgentId, + actualAssigneeAgentId: issue.assigneeAgentId, + }, + }, + status: "skipped", + requestedByActorType: opts.requestedByActorType ?? null, + requestedByActorId: opts.requestedByActorId ?? null, + idempotencyKey: opts.idempotencyKey ?? null, + finishedAt: new Date(), + }); + return { kind: "skipped" as const }; + } + + if (opts.failedRunId) { + // The issue lock makes double-clicks and network retries adopt the + // same successor, including after it has already finished. + const [previousRetry] = await tx.select().from(heartbeatRuns).where(and( + eq(heartbeatRuns.companyId, issue.companyId), eq(heartbeatRuns.agentId, agentId), + eq(heartbeatRuns.retryOfRunId, opts.failedRunId), + sql`${heartbeatRuns.contextSnapshot}->>'wakeReason' = 'retry_failed_run'`, + )).orderBy(desc(heartbeatRuns.createdAt)).limit(1); + if (previousRetry) return { kind: "replayed" as const, run: previousRetry }; + } + + let reconciledSourceRunId: string | null = null; + let reconciledRestoreRetryCount: number | null = null; + if (executionReconciliationWake) { + const actionId = readNonEmptyString( + enrichedContextSnapshot.recoveryActionId, + ); + if ( + !actionId || + !isUuidLike(actionId) || + source !== "automation" || + triggerDetail !== "system" || + reason !== "issue_recovery_action_restored" || + opts.requestedByActorType !== "system" || + opts.requestedByActorId !== "execution-recovery" || + opts.idempotencyKey !== `execution-reconciliation:${actionId}` || + enrichedContextSnapshot.source !== "execution.reconciled" || + enrichedContextSnapshot.forceFreshSession !== true || + payload?.issueId !== issue.id || + payload?.recoveryActionId !== actionId || + issue.assigneeAgentId !== agentId || + ["done", "cancelled"].includes(issue.status) + ) + return { kind: "skipped" as const }; + + // The issue lock serializes all admissions for this source. Validate + // the durable operator decision, then reconcile a prior queue commit + // before considering a new wake (including a now-terminal successor). + const [action] = await tx + .select() + .from(issueRecoveryActions) + .where( + and( + eq(issueRecoveryActions.companyId, issue.companyId), + eq(issueRecoveryActions.sourceIssueId, issue.id), + eq(issueRecoveryActions.id, actionId), + ), + ) + .for("update"); + const decision = parseObject( + action?.evidence.executionReconciliation, + ); + const sourceRunId = readNonEmptyString(decision.runId); + if ( + !action || + action.status !== "resolved" || + action.kind !== "active_run_watchdog" || + action.returnOwnerAgentId !== agentId || + !sourceRunId || + !isUuidLike(sourceRunId) || + decision.providerStopped !== true || + !["completed", "not_performed", "mixed"].includes( + String(decision.actionOutcome), + ) || + !readNonEmptyString(decision.outcomeEvidence) || + enrichedContextSnapshot.previousRunId !== sourceRunId || + enrichedContextSnapshot.retryOfRunId !== sourceRunId || + !["pending", "delivered"].includes( + String(action.evidence.continuationDelivery), + ) + ) + return { kind: "skipped" as const }; + + const [existingWake] = await tx + .select() + .from(agentWakeupRequests) + .where( + and( + eq(agentWakeupRequests.companyId, issue.companyId), + eq(agentWakeupRequests.agentId, agentId), + eq(agentWakeupRequests.idempotencyKey, opts.idempotencyKey), + ne(agentWakeupRequests.status, "skipped"), + ), + ) + .orderBy(asc(agentWakeupRequests.requestedAt)) + .limit(1); + if (existingWake) { + if ( + existingWake.payload?.issueId !== issue.id || + existingWake.payload?.recoveryActionId !== action.id || + existingWake.requestedByActorType !== "system" || + existingWake.requestedByActorId !== "execution-recovery" || + !existingWake.runId + ) + return { kind: "deferred" as const }; + const [existingRun] = await tx + .select() + .from(heartbeatRuns) + .where( + and( + eq(heartbeatRuns.companyId, issue.companyId), + eq(heartbeatRuns.agentId, agentId), + eq(heartbeatRuns.id, existingWake.runId), + ), + ); + if ( + !existingRun || + existingRun.contextSnapshot?.issueId !== issue.id || + existingRun.contextSnapshot?.recoveryActionId !== action.id || + existingRun.contextSnapshot?.previousRunId !== sourceRunId + ) + return { kind: "deferred" as const }; + return { kind: "replayed" as const, run: existingRun }; + } + if (action.evidence.continuationDelivery !== "pending") + return { kind: "skipped" as const }; + const [reconciledRun] = await tx.select().from(heartbeatRuns).where(and( + eq(heartbeatRuns.companyId, issue.companyId), eq(heartbeatRuns.id, sourceRunId), + )); + if (decision.workspaceReconstruction) { + await validateNativeWorkspaceReconstruction(tx as unknown as Db, { + companyId: issue.companyId, issueId: issue.id, runId: sourceRunId, actionId, + }); + reconciledRestoreRetryCount = executionFailureRetryCount(reconciledRun!); + enrichedContextSnapshot.workspaceReconstructionActionId = action.id; + } + if (hasWorkspaceRestoreFailure(reconciledRun?.resultJson)) { + if ((readNonEmptyString(decision.workspaceRepairEvidence)?.length ?? 0) < 20) + return { kind: "skipped" as const }; + // Repair does not reset the remaining automatic retry budget. + reconciledRestoreRetryCount = executionFailureRetryCount(reconciledRun!); + } + reconciledSourceRunId = sourceRunId; + } + + let continuationWait = { reason: "execution_recovery", message: "Waiting for execution recovery. Your message is saved." }; + let continuationBlocked = false; + const deferBlockedExecution = async ( + executionBlocker: Pick>>, "recoveryActionId" | "cause" | "nextAction">, + ) => { + if (caseRequest) waitForCaseAdmission("execution_reconciliation_required", executionBlocker.nextAction); + opts.issueStateGuard?.onExecutionDeferred?.(executionBlocker.cause); + const condition = { recoveryActionId: executionBlocker.recoveryActionId, ...continuationWait }; + if (executionWaitRequestId) { + await tx.update(agentWakeupRequests).set({ + payload: sql`jsonb_set(coalesce(${agentWakeupRequests.payload}, '{}'::jsonb), '{executionWait}', ${JSON.stringify(condition)}::jsonb)`, + updatedAt: new Date(), + }).where(eq(agentWakeupRequests.id, executionWaitRequestId)); + return { kind: "deferred" as const, admitted: true }; + } + if (durableRequest || wakeCommentId || + hasInteractionContinuationWakeContext(enrichedContextSnapshot) || + readNonEmptyString(enrichedContextSnapshot.nativeStatusWakeIntentId)) { + await tx.insert(agentWakeupRequests).values({ + ...durableReceiptFields, + companyId: agent.companyId, agentId, source, triggerDetail, reason, + payload: withQueuedCommentIdsInWakePayload({ + ...payload, + issueId: issue.id, + [DEFERRED_WAKE_CONTEXT_KEY]: enrichedContextSnapshot, + executionWait: condition, + }, [...new Set([ + ...queuedCommentIdsFromRunContext(enrichedContextSnapshot), + ...(wakeCommentId ? [wakeCommentId] : []), + ])]), + status: "deferred_issue_execution", + requestedByActorType: opts.requestedByActorType ?? null, + requestedByActorId: opts.requestedByActorId ?? null, + idempotencyKey: opts.idempotencyKey ?? null, + }); + } else { + await recordExecutionWait(tx as unknown as Db, { + issueId: issue.id, condition, coalesce: coalesceExecutionWait, + request: { + ...durableReceiptFields, + companyId: agent.companyId, agentId, source, triggerDetail, + reason: "execution_reconciliation_required", + error: executionBlocker.nextAction, + payload, + requestedByActorType: opts.requestedByActorType ?? null, + requestedByActorId: opts.requestedByActorId ?? null, + idempotencyKey: opts.idempotencyKey ?? null, + }, + }); + } + return { kind: "deferred" as const, admitted: true }; + }; + const explicitContinuationRunId = randomUUID(); + const [resetComment] = isConversation(issue) && opts.requestedByActorType === "user" && opts.requestedByActorId && wakeCommentId + ? await tx.select({ body: issueComments.body }).from(issueComments).where(and( + eq(issueComments.companyId, issue.companyId), eq(issueComments.issueId, issue.id), + eq(issueComments.id, wakeCommentId), eq(issueComments.authorUserId, opts.requestedByActorId), + eq(issueComments.authorType, "user"), isNull(issueComments.deletedAt), isNull(issueComments.createdByRunId), + )).limit(1) + : []; + const verifiedConversationReset = Boolean(resetComment && isConversationReset(resetComment.body)); + const executionBlocker = await getExecutionBlocker( + tx as unknown as Db, issue.companyId, issue.id, + { conversationResetCommentId: verifiedConversationReset ? wakeCommentId : null }, + ); + // Prove eligibility without retiring the hold. Later gates can still + // decline this wake; hold retirement and successor creation stay atomic. + // A bound chat request has already rechecked its current principal + // above. Treat its new user message like a board comment, but keep + // failed-run retry actions on their separate exact-request path. + const ownsReconstructionContinuation = reconciledSourceRunId && executionBlocker?.runId === reconciledSourceRunId + && executionBlocker.recoveryActionId === enrichedContextSnapshot.recoveryActionId; + const continuationHold = () => executionBlocker ?? { + recoveryActionId: null, cause: "native_continuation_requires_reconciliation" as const, nextAction: continuationWait.message, + }; + if ((executionBlocker || opts.queuedCommentInterruptId) && !ownsReconstructionContinuation && !(await admitExplicitNativeContinuation({ + db: tx as unknown as Db, companyId: issue.companyId, issueId: issue.id, + agentId, actorType: taskRetryActorId ? "user" : opts.requestedByActorType, actorId: taskRetryActorId ?? opts.requestedByActorId, + reason: taskRetryActorId ? "task_retry" : durableRequest && !failedChatRetry ? "issue_commented" : reason, + commentId: wakeCommentId ?? null, failedRunId: opts.failedRunId, successorRunId: explicitContinuationRunId, + queuedCommentInterruptId: opts.queuedCommentInterruptId, + queuedCommentRequestId: opts.queuedCommentRequestId, + taskRetryRequestId: opts.taskRetryRequestId, + ...(durableRequest?.resumeDeferred && !failedChatRetry ? { durableChatRequest: durableRequest } : {}), + dryRun: true, + onBlocked: (reason, message) => { continuationBlocked = true; continuationWait = { reason, message }; }, + })) && (executionBlocker || (continuationBlocked && !verifiedConversationReset))) return deferBlockedExecution(continuationHold()); + + + if ( + worktreeExecutionCutoff && + issue.createdAt < worktreeExecutionCutoff + ) { + if (caseRequest) waitForCaseAdmission("heartbeat.worktree_execution_cutoff", "Case handoff is waiting for workspace execution to be enabled"); + await tx.insert(agentWakeupRequests).values({ + ...durableReceiptFields, + companyId: agent.companyId, + agentId, + source, + triggerDetail, + reason: "heartbeat.worktree_execution_cutoff", + payload: { + ...(payload ?? {}), + heartbeatSkip: { + reason: "worktree_execution_cutoff", + cutoff: worktreeExecutionCutoff.toISOString(), + issueId: issue.id, + }, + }, + status: "skipped", + requestedByActorType: opts.requestedByActorType ?? null, + requestedByActorId: opts.requestedByActorId ?? null, + idempotencyKey: opts.idempotencyKey ?? null, + finishedAt: new Date(), + }); + return { kind: "skipped" as const }; + } + + const cancelStaleScheduledRetry = async ( + scheduledRun: typeof heartbeatRuns.$inferSelect, + ) => { + const issueCancelled = issue.status === "cancelled"; + if ( + scheduledRun.status !== "scheduled_retry" || + (scheduledRun.agentId === issue.assigneeAgentId && + !issueCancelled) + ) { + return false; + } + + const now = new Date(); + const reason = issueCancelled + ? "Cancelled because the issue was cancelled before the scheduled retry became due" + : "Cancelled because the issue was reassigned before the scheduled retry became due"; + await notifyInstructionCollection(tx, { companyId: scheduledRun.companyId, runId: scheduledRun.id }); + await notifyExternalAgentSettlement(tx, { companyId: scheduledRun.companyId, runId: scheduledRun.id }); + await notifyCancelledWorkspacePreservation(tx, { companyId: scheduledRun.companyId, runId: scheduledRun.id }); + await notifyUnsafeWorkspaceExportRunChange(tx, scheduledRun); + await notifyScheduledRetryCancellation(tx, { companyId: scheduledRun.companyId, runId: scheduledRun.id }); + await notifyChatRunMilestones(tx, { companyId: scheduledRun.companyId, issueId: issue.id }); + const cancelled = await tx + .update(heartbeatRuns) + .set({ + status: "cancelled", + finishedAt: now, + error: reason, + errorCode: issueCancelled + ? "issue_cancelled" + : "issue_reassigned", + updatedAt: now, + }) + .where( + and( + eq(heartbeatRuns.id, scheduledRun.id), + eq(heartbeatRuns.status, "scheduled_retry"), + ), + ) + .returning() + .then((rows) => rows[0] ?? null); + + if (!cancelled) return false; + + if (scheduledRun.wakeupRequestId) { + await tx + .update(agentWakeupRequests) + .set({ + status: "cancelled", + finishedAt: now, + error: reason, + updatedAt: now, + }) + .where( + eq(agentWakeupRequests.id, scheduledRun.wakeupRequestId), + ); + } + + if (issue.executionRunId === scheduledRun.id) { + await tx + .update(issues) + .set({ + executionRunId: null, + executionAgentNameKey: null, + executionLockedAt: null, + updatedAt: now, + }) + .where( + and( + eq(issues.id, issue.id), + eq(issues.executionRunId, scheduledRun.id), + ), + ); + } + + const eventSeq = await allocateHeartbeatRunEventSeq( + tx as unknown as Db, + cancelled.id, + ); + + await tx.insert(heartbeatRunEvents).values({ + companyId: cancelled.companyId, + runId: cancelled.id, + agentId: cancelled.agentId, + seq: eventSeq, + eventType: "lifecycle", + stream: "system", + level: "warn", + message: issueCancelled + ? "Scheduled retry cancelled because issue was cancelled before it became due" + : "Scheduled retry cancelled because issue ownership changed before it became due", + payload: { + issueId: issue.id, + issueStatus: issue.status, + scheduledRetryAttempt: cancelled.scheduledRetryAttempt, + scheduledRetryAt: cancelled.scheduledRetryAt + ? new Date(cancelled.scheduledRetryAt).toISOString() + : null, + scheduledRetryReason: cancelled.scheduledRetryReason, + previousRetryAgentId: cancelled.agentId, + currentAssigneeAgentId: issue.assigneeAgentId, + }, + }); + await tx + .update(heartbeatRuns) + .set({ nextEventSeq: eventSeq + 1, updatedAt: now }) + .where(eq(heartbeatRuns.id, cancelled.id)); + + cancelledRunsToEmit.push(cancelled); + + return true; + }; + + let activeExecutionRun = issue.executionRunId + ? await tx + .select() + .from(heartbeatRuns) + .where(eq(heartbeatRuns.id, issue.executionRunId)) + .then((rows) => rows[0] ?? null) + : null; + + if ( + activeExecutionRun && + !EXECUTION_PATH_HEARTBEAT_RUN_STATUSES.includes( + activeExecutionRun.status as (typeof EXECUTION_PATH_HEARTBEAT_RUN_STATUSES)[number], + ) + ) { + activeExecutionRun = null; + } + + if ( + activeExecutionRun && + (await cancelStaleScheduledRetry(activeExecutionRun)) + ) { + activeExecutionRun = null; + } + + // A queued/scheduled run holding the lock for an agent that is + // no longer the issue's assignee is stale by design — the issue + // has been re-routed (e.g. blocked → in_review with a different + // assignee). Cancel it and release the lock; otherwise the new + // assignee's wake gets parked in `deferred_issue_execution` + // forever, because the original queued holder will never run + // (the issue's status / target now belongs to someone else). + // + // Race guard: pin the cancel UPDATE to the exact non-running + // status we read above. A worker could transition the holder + // from `queued` → `running` between the SELECT and this UPDATE; + // the status predicate ensures we never clobber a freshly- + // claimed running run. If zero rows matched, leave + // `activeExecutionRun` populated so the defer path runs + // normally against the now-running holder. + if ( + activeExecutionRun && + activeExecutionRun.status !== "running" && + issue.assigneeAgentId && + activeExecutionRun.agentId !== issue.assigneeAgentId + ) { + await notifyInstructionCollection(tx, { companyId: activeExecutionRun.companyId, runId: activeExecutionRun.id }); + await notifyExternalAgentSettlement(tx, { companyId: activeExecutionRun.companyId, runId: activeExecutionRun.id }); + await notifyCancelledWorkspacePreservation(tx, { companyId: activeExecutionRun.companyId, runId: activeExecutionRun.id }); + await notifyUnsafeWorkspaceExportRunChange(tx, activeExecutionRun); + await notifyChatRunMilestones(tx, { companyId: activeExecutionRun.companyId, issueId: issue.id }); + const cancelled = await tx + .update(heartbeatRuns) + .set({ + status: "cancelled", + finishedAt: new Date(), + error: + "Execution lock released after issue reassigned to a different agent", + errorCode: "lock_released_on_reassignment", + updatedAt: new Date(), + }) + .where( + and( + eq(heartbeatRuns.id, activeExecutionRun.id), + eq(heartbeatRuns.status, activeExecutionRun.status), + ), + ) + .returning(); + if (cancelled.length > 0) { + cancelledRunsToEmit.push(cancelled[0]); + if (activeExecutionRun.wakeupRequestId) { + await tx + .update(agentWakeupRequests) + .set({ + status: "cancelled", + finishedAt: new Date(), + error: + "Execution lock released after issue reassigned to a different agent", + updatedAt: new Date(), + }) + .where( + eq( + agentWakeupRequests.id, + activeExecutionRun.wakeupRequestId, + ), + ); + } + activeExecutionRun = null; + } + } + + if (!activeExecutionRun && issue.executionRunId) { + await notifyNativeStatusReassessment(tx, { companyId: issue.companyId, issueId: issue.id }); + await notifyUnsafeWorkspaceExportRecovery(tx, { companyId: issue.companyId, issueId: issue.id }); + await tx + .update(issues) + .set({ + executionRunId: null, + executionAgentNameKey: null, + executionLockedAt: null, + updatedAt: new Date(), + }) + .where(eq(issues.id, issue.id)); + } + + if (!activeExecutionRun) { + const legacyRun = await tx + .select() + .from(heartbeatRuns) + .where( + and( + eq(heartbeatRuns.companyId, issue.companyId), + inArray(heartbeatRuns.status, [ + ...EXECUTION_PATH_HEARTBEAT_RUN_STATUSES, + ]), + sql`${heartbeatRuns.contextSnapshot} ->> 'issueId' = ${issue.id}`, + ), + ) + .orderBy( + sql`case when ${heartbeatRuns.status} = 'running' then 0 else 1 end`, + asc(heartbeatRuns.createdAt), + ) + .limit(1) + .then((rows) => rows[0] ?? null); + + if (legacyRun) { + if (await cancelStaleScheduledRetry(legacyRun)) { + activeExecutionRun = null; + } else { + activeExecutionRun = legacyRun; + const legacyAgent = await tx + .select({ name: agents.name }) + .from(agents) + .where(eq(agents.id, legacyRun.agentId)) + .then((rows) => rows[0] ?? null); + await notifyStatusCardIssueChange(tx, issue.companyId); + await tx + .update(issues) + .set({ + executionRunId: legacyRun.id, + executionAgentNameKey: normalizeAgentNameKey( + legacyAgent?.name, + ), + executionLockedAt: new Date(), + updatedAt: new Date(), + }) + .where(eq(issues.id, issue.id)); + } + } + } + + const dependencyReadiness = await issuesSvc + .listDependencyReadiness(issue.companyId, [issue.id], tx) + .then((rows) => rows.get(issue.id) ?? null); + + // Blocked descendants should stay idle until the final blocker resolves. + // Human comment/mention wakes are the exception: they may run in a + // bounded interaction mode so the assignee can answer or triage. + const blockedInteractionWake = + dependencyReadiness && + !dependencyReadiness.isDependencyReady && + allowsIssueInteractionWake( + enrichedContextSnapshot, + ISSUE_TREE_CONTROL_INTERACTION_WAKE_REASONS, + ); + + if (blockedInteractionWake) { + enrichedContextSnapshot.dependencyBlockedInteraction = true; + enrichedContextSnapshot.unresolvedBlockerIssueIds = + dependencyReadiness.unresolvedBlockerIssueIds; + enrichedContextSnapshot.unresolvedBlockerCount = + dependencyReadiness.unresolvedBlockerCount; + enrichedContextSnapshot.unresolvedBlockerSummaries = + await listUnresolvedBlockerSummaries( + tx, + issue.companyId, + issue.id, + dependencyReadiness.unresolvedBlockerIssueIds, + ); + } + + if ( + !activeExecutionRun && + dependencyReadiness && + !dependencyReadiness.isDependencyReady && + !blockedInteractionWake + ) { + if (caseRequest) waitForCaseAdmission("issue_dependencies_blocked", "Case handoff is waiting for task dependencies to resolve"); + await recordExecutionWait(tx as unknown as Db, { + issueId: issue.id, + coalesce: coalesceExecutionWait, + condition: { unresolvedBlockerIssueIds: [...dependencyReadiness.unresolvedBlockerIssueIds].sort() }, + request: { + ...durableReceiptFields, + companyId: agent.companyId, + agentId, + source, + triggerDetail, + reason: "issue_dependencies_blocked", + payload: { + ...(payload ?? {}), + issueId, + unresolvedBlockerIssueIds: + dependencyReadiness.unresolvedBlockerIssueIds, + }, + status: "skipped", + requestedByActorType: opts.requestedByActorType ?? null, + requestedByActorId: opts.requestedByActorId ?? null, + idempotencyKey: opts.idempotencyKey ?? null, + finishedAt: new Date(), + }, + }); + return { kind: "skipped" as const }; + } + + if ( + isolatedWorkspacesEnabled && + !activeExecutionRun && + issue.status !== "done" && + issue.status !== "cancelled" + ) { + const issueSettings = parseIssueExecutionWorkspaceSettings( + issue.executionWorkspaceSettings, + ); + const resolvedMode = resolveExecutionWorkspaceMode({ + projectPolicy: null, + issueSettings, + legacyUseProjectWorkspace: null, + }); + const workspaceManagedConfig = buildExecutionWorkspaceAdapterConfig( + { + agentConfig: parseObject(agent.adapterConfig), + projectPolicy: null, + issueSettings, + mode: resolvedMode, + legacyUseProjectWorkspace: null, + }, + ); + const resolvedStrategy = resolveEffectiveWorkspaceStrategyType( + resolvedMode, + workspaceManagedConfig, + ); + const existingExecutionWorkspaceStatus = issue.executionWorkspaceId + ? await tx + .select({ status: executionWorkspaces.status }) + .from(executionWorkspaces) + .where( + and( + eq(executionWorkspaces.id, issue.executionWorkspaceId), + eq(executionWorkspaces.companyId, issue.companyId), + ), + ) + .then((rows) => rows[0]?.status ?? null) + : null; + const reuseRequest = resolveExecutionWorkspaceReuseRequestForIssue({ + issueExecutionWorkspaceId: issue.executionWorkspaceId, + issueExecutionWorkspacePreference: + issue.executionWorkspacePreference, + existingExecutionWorkspaceStatus, + }); + const hasResolvablePriorSessionWorkspace = + await resolveHasResolvablePriorSessionWorkspace(); + + if ( + isUnrunnableWorktreeCombo({ + issue: { + projectId: issue.projectId ?? projectId ?? null, + projectWorkspaceId: issue.projectWorkspaceId, + executionWorkspaceId: issue.executionWorkspaceId, + executionWorkspacePreference: + issue.executionWorkspacePreference, + }, + resolvedMode, + resolvedStrategy, + reusableExecutionWorkspaceAvailable: + reuseRequest.existingExecutionWorkspaceAvailable, + hasResolvablePriorSessionWorkspace, + }) + ) { + if (caseRequest) waitForCaseAdmission(WORKSPACE_WORKTREE_REQUIRES_PROJECT_CODE, WORKSPACE_WORKTREE_REQUIRES_PROJECT_MESSAGE); + const now = new Date(); + const issueLabel = formatIssueIdentifierLink( + issue.identifier, + issue.id, + ); + const blockedComment = [ + `Paperclip blocked ${issueLabel} before dispatch because its workspace settings are not runnable.`, + "", + `- Code: \`${WORKSPACE_WORKTREE_REQUIRES_PROJECT_CODE}\``, + `- Reason: ${WORKSPACE_WORKTREE_REQUIRES_PROJECT_MESSAGE}`, + `- Next action: ${WORKSPACE_WORKTREE_REQUIRES_PROJECT_REMEDIATION}`, + ].join("\n"); + await notifyUnsafeWorkspaceExportRecovery(tx, { companyId: issue.companyId, issueId: issue.id }); + await notifyNativeStatusReassessment(tx, { companyId: issue.companyId, issueId: issue.id }); + await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors); + await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers); + if (issue.status !== "blocked") await notifyChatResponseAvailability(tx, { companyId: issue.companyId, issueId: issue.id }); + await tx + .update(issues) + .set({ + status: "blocked", + checkoutRunId: null, + executionRunId: null, + executionAgentNameKey: null, + executionLockedAt: null, + updatedAt: now, + }) + .where(eq(issues.id, issue.id)); + await insertIssueComment(tx, { + companyId: issue.companyId, + issueId: issue.id, + body: blockedComment, + createdAt: now, + updatedAt: now, + }); + await tx.insert(agentWakeupRequests).values({ + ...durableReceiptFields, + companyId: agent.companyId, + agentId, + source, + triggerDetail, + reason: WORKSPACE_WORKTREE_REQUIRES_PROJECT_CODE, + payload: { + ...(payload ?? {}), + issueId, + heartbeatSkip: { + code: WORKSPACE_WORKTREE_REQUIRES_PROJECT_CODE, + reason: WORKSPACE_WORKTREE_REQUIRES_PROJECT_MESSAGE, + remediation: + WORKSPACE_WORKTREE_REQUIRES_PROJECT_REMEDIATION, + }, + }, + status: "skipped", + requestedByActorType: opts.requestedByActorType ?? null, + requestedByActorId: opts.requestedByActorId ?? null, + idempotencyKey: opts.idempotencyKey ?? null, + finishedAt: now, + }); + await logActivity(tx as unknown as Db, { + companyId: issue.companyId, + actorType: "system", + actorId: "system", + agentId, + runId: null, + action: "issue.workspace_preflight_blocked", + entityType: "issue", + entityId: issue.id, + details: { + code: WORKSPACE_WORKTREE_REQUIRES_PROJECT_CODE, + reason: WORKSPACE_WORKTREE_REQUIRES_PROJECT_MESSAGE, + remediation: WORKSPACE_WORKTREE_REQUIRES_PROJECT_REMEDIATION, + requestedReason: reason, + source, + triggerDetail, + resolvedMode, + resolvedStrategy, + hasResolvablePriorSessionWorkspace, + }, + }); + return { kind: "skipped" as const }; + } + } + + if (activeExecutionRun) { + // The case ledger owns retry. Do not place its exact receipt in a + // generic deferred queue that cannot retain handoff authority. + if (caseRequest) return { kind: "deferred" as const }; + // The resolved action is already a durable retry outbox. Do not merge + // its fresh-session contract into unrelated work or create a second + // deferred wake that could later replay the same reconciliation. + if (reconciledSourceRunId) return { kind: "deferred" as const }; + + // Offer the input to the active external agent, but keep a durable queued wake + // until it reads the comment. A finish racing the event tick must + // leave unread input available for the next assignment. + const followUpProvider = parseObject(agent.adapterConfig).provider; + const museFollowUp = followUpProvider === "muse"; + const bindingId = readNonEmptyString(parseObject(agent.adapterConfig)[museFollowUp ? "museBindingId" : followUpProvider === "grok_bot" ? "externalBindingId" : "dotBindingId"]); + const followUpPayloadKey = museFollowUp ? "museAssignmentFollowUp" : "dotAssignmentFollowUp"; + let followUp: false | { assignmentId: string; consumed: boolean } = false; + let pendingFollowUps: typeof agentWakeupRequests.$inferSelect[] = []; + if (agent.adapterType === "paperclip_runner" && ["openai_dot", "grok_bot", "muse"].includes(String(followUpProvider)) + && bindingId && activeExecutionRun.agentId === agentId && issue.assigneeAgentId === agentId + && reason === "issue_commented" && wakeCommentId && opts.allowRunCoalescing !== false + && enrichedContextSnapshot.forceFreshSession !== true && !explicitResumeSession && !opts.manualUserWake + && !enrichedContextSnapshot.interactionId && !payload?.interactionId && !receiptRequest) { + // Receipt consumption uses the same issue -> wake -> run -> guard -> + // binding order. Lock before publishing, then reuse these wake rows. + pendingFollowUps = await tx.select().from(agentWakeupRequests).where(and( + eq(agentWakeupRequests.companyId, agent.companyId), eq(agentWakeupRequests.agentId, agentId), + isNull(agentWakeupRequests.runId), eq(agentWakeupRequests.status, "deferred_issue_execution"), + sql`${agentWakeupRequests.payload}->>'issueId' = ${issue.id}`, + sql`${agentWakeupRequests.payload}->>${followUpPayloadKey} is not null`, + )).orderBy(asc(agentWakeupRequests.createdAt), asc(agentWakeupRequests.id)).for("update"); + const [currentRun] = await tx.select().from(heartbeatRuns).where(and( + eq(heartbeatRuns.id, activeExecutionRun.id), eq(heartbeatRuns.companyId, agent.companyId), + eq(heartbeatRuns.agentId, agentId), eq(heartbeatRuns.status, "running"), + eq(heartbeatRuns.runtimeMode, "native"), eq(heartbeatRuns.nativeIssueId, issue.id), + )).for("update"); + if (currentRun) followUp = await withExternalAdmissionGuard(tx, agent.companyId, agentId, + () => (museFollowUp ? publishActiveMuseComment : publishActiveDotComment)(tx as unknown as Db, { + companyId: agent.companyId, agentId, bindingId, runId: currentRun.id, + issueId: issue.id, commentId: wakeCommentId, + })); + } + if (followUp) { + const assignmentId = followUp.assignmentId; + const pending = followUp.consumed ? undefined : pendingFollowUps.find(wake => + wake.payload?.[followUpPayloadKey] === assignmentId && + wake.requestedByActorType === (opts.requestedByActorType ?? null) && + wake.requestedByActorId === (opts.requestedByActorId ?? null)); + if (pending) await tx.update(agentWakeupRequests).set({ + payload: withQueuedCommentIdsInWakePayload(pending.payload, + [...new Set([...queuedCommentIdsFromWakePayload(pending.payload), wakeCommentId!])]), + coalescedCount: pending.coalescedCount + 1, updatedAt: new Date(), + }).where(and(eq(agentWakeupRequests.id, pending.id), eq(agentWakeupRequests.companyId, agent.companyId), + eq(agentWakeupRequests.agentId, agentId), eq(agentWakeupRequests.status, "deferred_issue_execution"))); + else await tx.insert(agentWakeupRequests).values({ companyId: agent.companyId, agentId, source, triggerDetail, + reason, payload: withQueuedCommentIdsInWakePayload({ ...payload, issueId: issue.id, + [followUpPayloadKey]: followUp.assignmentId, + [DEFERRED_WAKE_CONTEXT_KEY]: enrichedContextSnapshot }, [wakeCommentId!]), + status: followUp.consumed ? "coalesced" : "deferred_issue_execution", + runId: followUp.consumed ? activeExecutionRun.id : null, + requestedByActorType: opts.requestedByActorType ?? null, + requestedByActorId: opts.requestedByActorId ?? null, idempotencyKey: opts.idempotencyKey ?? null, + finishedAt: followUp.consumed ? new Date() : null }); + return { kind: "coalesced" as const, run: activeExecutionRun }; + } + + const admissionScope = wakeQueue.createAdmissionTransactionScope( + agent.companyId, + tx as unknown as Db, + ); + const admission = await wakeQueue.admitWakeBehindIssueExecution( + admissionScope, + { + companyId: agent.companyId, + issueId: issue.id, + agentId, + agentNameKey, + issueExecutionAgentNameKey: issue.executionAgentNameKey, + activeExecutionRun: { + id: activeExecutionRun.id, + agentId: activeExecutionRun.agentId, + status: activeExecutionRun.status, + contextSnapshot: activeExecutionRun.contextSnapshot, + wakeupRequestId: activeExecutionRun.wakeupRequestId, + }, + allowRunCoalescing: isConversation(issue) ? false : opts.allowRunCoalescing, + durableReceipt: receiptRequest + ? { + id: receiptRequest.id, + requestedAt: receiptRequest.requestedAt, + } + : undefined, + reason, + liveRunExecutions, + wakeCommentId, + forceFreshSession: + enrichedContextSnapshot.forceFreshSession === true, + contextSnapshot: enrichedContextSnapshot, + source, + triggerDetail, + payload, + requestedByActorType: opts.requestedByActorType ?? null, + requestedByActorId: opts.requestedByActorId ?? null, + idempotencyKey: opts.idempotencyKey ?? null, + }, + ); + + if (admission.kind === "coalesced") { + return { + kind: "coalesced" as const, + run: admission.run as typeof heartbeatRuns.$inferSelect, + }; + } + if (admission.kind === "deferred") { + return { kind: "deferred" as const, admitted: true }; + } + // admission.kind === "proceed": no active run absorbed this wake, + // so fall through to the ordinary queue path below. + } + + // PAP-13775: no live run holds the lock, so this wake would start a + // fresh adapter session. If this agent's recent runs on this issue + // keep succeeding without any issue-visible progress and the wake + // carries no new information, hold it back for an escalating cooldown + // so external pollers/reconcilers can't storm full-price sessions. + // Server-side recovery retries insert runs directly and never reach + // this gate. A verified durable case receipt carries a new phase; + // its agent-authored instruction comment is not an event-free wake. + if ( + !caseRequest && + isThrottleCandidateIssueRewake({ + reason, + wakeCommentId: wakeCommentId ?? null, + requestedByActorType: opts.requestedByActorType ?? null, + forceFreshSession: + enrichedContextSnapshot.forceFreshSession === true, + hasExplicitResume: Boolean(explicitResumeSession), + }) + ) { + const throttleNow = new Date(); + const recentTerminalRuns = await tx + .select({ + id: heartbeatRuns.id, + status: heartbeatRuns.status, + finishedAt: heartbeatRuns.finishedAt, + }) + .from(heartbeatRuns) + .where( + and( + eq(heartbeatRuns.companyId, agent.companyId), + eq(heartbeatRuns.agentId, agentId), + sql`${heartbeatRuns.finishedAt} is not null`, + gte( + heartbeatRuns.finishedAt, + new Date(throttleNow.getTime() - ISSUE_REWAKE_LOOKBACK_MS), + ), + sql`${heartbeatRuns.contextSnapshot} ->> 'issueId' = ${issue.id}`, + ), + ) + .orderBy(desc(heartbeatRuns.finishedAt)) + .limit(ISSUE_REWAKE_RUN_SAMPLE_LIMIT); + + if (recentTerminalRuns.length > 0) { + const sampleRunIds = recentTerminalRuns.map( + (sampleRun) => sampleRun.id, + ); + const progressRows = await tx + .select({ runId: activityLog.runId }) + .from(activityLog) + .where( + and( + eq(activityLog.companyId, agent.companyId), + eq(activityLog.entityType, "issue"), + eq(activityLog.entityId, issue.id), + inArray(activityLog.runId, sampleRunIds), + inArray( + activityLog.action, + ISSUE_PROGRESS_ACTIVITY_ACTIONS, + ), + ), + ); + const lastRunFinishedAt = + recentTerminalRuns[0]?.finishedAt ?? null; + const newInputRows = lastRunFinishedAt + ? await tx + .select({ id: activityLog.id }) + .from(activityLog) + .where( + and( + eq(activityLog.companyId, agent.companyId), + eq(activityLog.entityType, "issue"), + eq(activityLog.entityId, issue.id), + gt(activityLog.createdAt, lastRunFinishedAt), + sql`coalesce(${activityLog.details}->>'origin', '') <> 'fast_response'`, + inArray( + activityLog.action, + ISSUE_NEW_INPUT_ACTIVITY_ACTIONS, + ), + wakeCommentId && opts.requestedByActorType === "agent" + ? ne(activityLog.actorType, "agent") + : undefined, + ), + ) + .limit(1) + : []; + + const throttleDecision = evaluateIssueRewakeThrottle({ + now: throttleNow, + recentTerminalRuns, + runIdsWithIssueProgress: new Set( + progressRows + .map((row) => row.runId) + .filter((runId): runId is string => Boolean(runId)), + ), + // For an agent comment wake, the query excludes agent-authored + // activity while preserving genuinely new user/system input. + // Presentation/author metadata therefore cannot smuggle human + // wake privilege, nor can it mask an actual human response. + hasNewIssueInputSinceLastRun: newInputRows.length > 0, + }); + + if (throttleDecision.blocked) { + await tx.insert(agentWakeupRequests).values({ + ...durableReceiptFields, + companyId: agent.companyId, + agentId, + source, + triggerDetail, + reason: "issue_rewake_throttled", + payload: { + ...(payload ?? {}), + issueId, + heartbeatSkip: { + reason: "issue_rewake_throttled", + requestedReason: reason, + noProgressStreak: throttleDecision.noProgressStreak, + cooldownMs: throttleDecision.cooldownMs, + lastRunFinishedAt: + throttleDecision.lastRunFinishedAt.toISOString(), + nextAllowedAt: + throttleDecision.nextAllowedAt.toISOString(), + }, + }, + status: "skipped", + requestedByActorType: opts.requestedByActorType ?? null, + requestedByActorId: opts.requestedByActorId ?? null, + idempotencyKey: opts.idempotencyKey ?? null, + finishedAt: throttleNow, + }); + return { kind: "skipped" as const }; + } + } + } + + const dailyCapBlock = await getHeartbeatDailyCapBlock( + agent, + policy, + {}, + tx, + ); + if (dailyCapBlock) { + if (caseRequest) waitForCaseAdmission(dailyCapBlock.reason, "Case handoff is waiting for the agent's daily limit to clear"); + if (executionWaitRequestId && executionBlocker) { + continuationWait = { reason: dailyCapBlock.reason, + message: "The agent has reached its daily limit. Your message is saved until work can resume." }; + return deferBlockedExecution(executionBlocker); + } + const now = new Date(); + await tx.insert(agentWakeupRequests).values({ + ...durableReceiptFields, + companyId: agent.companyId, + agentId, + source, + triggerDetail, + reason: dailyCapBlock.reason, + payload: { + ...(payload ?? {}), + heartbeatSkip: { + reason: + "Per-agent heartbeat daily cap reached before adapter invocation.", + observed: dailyCapBlock.observed, + limit: dailyCapBlock.limit, + }, + }, + status: "skipped", + requestedByActorType: opts.requestedByActorType ?? null, + requestedByActorId: opts.requestedByActorId ?? null, + idempotencyKey: opts.idempotencyKey ?? null, + finishedAt: now, + }); + if (source === "timer") { + await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers); + await tx + .update(agents) + .set({ + lastHeartbeatAt: now, + updatedAt: now, + }) + .where(eq(agents.id, agentId)); + } + return { kind: "skipped" as const }; + } + + const explicitContinuation = await admitExplicitNativeContinuation({ + db: tx as unknown as Db, companyId: issue.companyId, issueId: issue.id, + agentId, actorType: taskRetryActorId ? "user" : opts.requestedByActorType, actorId: taskRetryActorId ?? opts.requestedByActorId, + reason: taskRetryActorId ? "task_retry" : durableRequest && !failedChatRetry ? "issue_commented" : reason, + commentId: wakeCommentId ?? null, failedRunId: opts.failedRunId, successorRunId: explicitContinuationRunId, + queuedCommentInterruptId: opts.queuedCommentInterruptId, + queuedCommentRequestId: opts.queuedCommentRequestId, + taskRetryRequestId: opts.taskRetryRequestId, + ...(durableRequest?.resumeDeferred && !failedChatRetry ? { durableChatRequest: durableRequest } : {}), + onBlocked: (reason, message) => { continuationBlocked = true; continuationWait = { reason, message }; }, + }); + if (!explicitContinuation && (executionBlocker || (continuationBlocked && !verifiedConversationReset)) && !ownsReconstructionContinuation) + return deferBlockedExecution(continuationHold()); + if (explicitContinuation) { + enrichedContextSnapshot.forceFreshSession = true; + enrichedContextSnapshot.previousRunId = explicitContinuation.previousRunId; + enrichedContextSnapshot.explicitUserContinuation = explicitContinuation; + } + + const wakeupValues = { + ...durableReceiptFields, + companyId: agent.companyId, + agentId, + source, + triggerDetail, + reason, + payload, + status: "queued" as const, + requestedByActorType: opts.requestedByActorType ?? null, + requestedByActorId: opts.requestedByActorId ?? null, + idempotencyKey: opts.idempotencyKey ?? null, + }; + // A deferred durable chat wake already owns this ID and source. + // Promote that receipt in the same transaction as its successor. + const wakeupRequest = durableRequest && executionWaitRequestId === durableRequest.id + ? await tx.update(agentWakeupRequests).set({ + payload, status: "queued", error: null, finishedAt: null, updatedAt: new Date(), + }).where(and(eq(agentWakeupRequests.id, durableRequest.id), + eq(agentWakeupRequests.companyId, agent.companyId), + eq(agentWakeupRequests.status, "deferred_issue_execution"), isNull(agentWakeupRequests.runId))) + .returning().then(rows => rows[0]) + : await tx.insert(agentWakeupRequests).values(wakeupValues).returning().then(rows => rows[0]); + if (!wakeupRequest) return { kind: "deferred" as const }; + + // A handoff changes the executor, not the owner of saved user input. + // Validate its exact stopped source while the issue row is locked; + // unrelated agents and dedicated continuations keep their own wakes. + const interruptedRunId = readNonEmptyString(enrichedContextSnapshot.interruptedRunId); + const handoffSource = source === "assignment" && reason === "issue_assigned" && + issue.assigneeAgentId === agentId && interruptedRunId && isUuidLike(interruptedRunId) + ? await tx.select({ agentId: heartbeatRuns.agentId }).from(heartbeatRuns).where(and( + eq(heartbeatRuns.id, interruptedRunId), eq(heartbeatRuns.companyId, issue.companyId), + eq(heartbeatRuns.status, "cancelled"), eq(heartbeatRuns.errorCode, "issue_reassigned"), + ne(heartbeatRuns.agentId, agentId), + sql`${heartbeatRuns.contextSnapshot}->>'issueId' = ${issue.id}`, + or(isNull(heartbeatRuns.nativeIssueId), eq(heartbeatRuns.nativeIssueId, issue.id)), + )).then(rows => rows[0] ?? null) + : null; + const canCoalesceComments = !isConversation(issue) && opts.allowRunCoalescing !== false; + // A resumed receipt already passed admission under this issue lock. + // Consume it with its successor even when chat keeps other messages + // in separate turns, or finalization will deliver it a second time. + const pendingComments = + (executionWaitRequestId || canCoalesceComments) && + (ownsReconstructionContinuation || !(await getExecutionBlocker(tx as unknown as Db, issue.companyId, issue.id))) + ? await tx + .select() + .from(agentWakeupRequests) + .where( + and( + eq(agentWakeupRequests.companyId, issue.companyId), + inArray(agentWakeupRequests.agentId, handoffSource ? [agentId, handoffSource.agentId] : [agentId]), + eq(agentWakeupRequests.status, "deferred_issue_execution"), + sql`${agentWakeupRequests.payload}->>'issueId' = ${issue.id}`, + // Chat Retry already pins its complete authorized source + // batch. Keep separate board input for its own admission. + canCoalesceComments || (opts.taskRetryRequestId && !failedChatRetry) + ? undefined : eq(agentWakeupRequests.id, executionWaitRequestId!), + ), + ) + .orderBy(asc(agentWakeupRequests.requestedAt)) + : []; + const adoptedComments = pendingComments.filter((wake) => { + if (wake.id === executionWaitRequestId) return true; + if (wake.id === opts.queuedCommentInterruptId || wake.id === opts.queuedCommentRequestId) return true; + const deferredPayload = parseObject(wake.payload); + const deferredContext = parseObject( + deferredPayload[DEFERRED_WAKE_CONTEXT_KEY], + ); + // Dedicated interaction wakes carry their own source and session + // contract. ID-only adoption must not erase that continuation. + return ( + // Durable chat work must keep its receipt, actor, source, and + // session contract through normal promotion and authorization. + !wake.idempotencyKey?.startsWith("chat-inbound:") && + !isInteractionResolutionWakePayload(deferredPayload) && + !hasInteractionContinuationWakeContext(deferredContext) && + ["issue_commented", "issue_reopened_via_comment"].includes(String(deferredContext.wakeReason ?? wake.reason)) && + queuedCommentIdsFromWakePayload(wake.payload).length > 0 + ); + }); + let adoptedCommentIds = [ + ...new Set([ + ...adoptedComments.flatMap((wake) => + queuedCommentIdsFromWakePayload(wake.payload), + ), + ...queuedCommentIdsFromRunContext(enrichedContextSnapshot), + ]), + ]; + if (opts.queuedCommentRequestId || (opts.taskRetryRequestId && !failedChatRetry) || handoffSource) { + adoptedCommentIds = await undeliveredLegacyUserCommentIds(tx as unknown as Db, + agent.companyId, issueId, agentId, adoptedCommentIds); + } + await notifyChatRunMilestones(tx, { companyId: agent.companyId, issueId: issueId ?? undefined }); + const newRun = await tx + .insert(heartbeatRuns) + .values({ + ...(explicitContinuation ? { id: explicitContinuationRunId } : {}), + companyId: agent.companyId, + agentId, + scopeKind: readNonEmptyString(enrichedContextSnapshot.issueId) ? "issue" : "company", + issueId: readNonEmptyString(enrichedContextSnapshot.issueId), + invocationSource: source, + triggerDetail, + status: "queued", + responsibleUserId: await resolveQueuedResponsibleUserId(), + wakeupRequestId: wakeupRequest.id, + retryOfRunId: failedChatRetry + ? durableRequest!.failedRunRetry!.failedRunId + : opts.failedRunId ?? automaticParentRunId, + contextSnapshot: adoptedComments.length + ? withQueuedCommentIdsInRunContext( + enrichedContextSnapshot, + adoptedCommentIds, + adoptedComments.map((wake) => wake.id), + ) + : enrichedContextSnapshot, + sessionIdBefore: explicitContinuation ? null : sessionBefore, + continuationAttempt, + ...(reconciledSourceRunId + ? { retryOfRunId: reconciledSourceRunId } + : {}), + ...(reconciledRestoreRetryCount !== null ? { + scheduledRetryAttempt: reconciledRestoreRetryCount, + scheduledRetryReason: "transient_failure", + } : {}), + }) + .returning() + .then((rows) => rows[0]); + + if (ownsReconstructionContinuation) { + await tx.update(issueRecoveryActions).set({ evidence: sql`${issueRecoveryActions.evidence} || ${JSON.stringify({ + continuationDelivery: "delivered", continuationRunId: newRun.id, + })}::jsonb` }).where(and(eq(issueRecoveryActions.companyId, issue.companyId), + eq(issueRecoveryActions.id, String(enrichedContextSnapshot.recoveryActionId)), + sql`${issueRecoveryActions.evidence}->>'continuationDelivery' = 'pending'`)); + } + + await tx + .update(agentWakeupRequests) + .set({ + runId: newRun.id, + updatedAt: new Date(), + }) + .where(eq(agentWakeupRequests.id, wakeupRequest.id)); + + if (failedChatRetry && opts.taskRetryRequestId) { + await tx.update(agentWakeupRequests).set({ + status: "coalesced", runId: newRun.id, finishedAt: new Date(), updatedAt: new Date(), + }).where(and(eq(agentWakeupRequests.companyId, issue.companyId), + eq(agentWakeupRequests.id, opts.taskRetryRequestId), + eq(agentWakeupRequests.status, "deferred_issue_execution"))); + } + + if (adoptedComments.length) { + await tx + .update(agentWakeupRequests) + .set({ + status: "coalesced", + runId: newRun.id, + finishedAt: new Date(), + updatedAt: new Date(), + }) + .where( + inArray( + agentWakeupRequests.id, + adoptedComments.map((wake) => wake.id), + ), + ); + await tx + .update(agentWakeupRequests) + .set({ + payload: withQueuedCommentIdsInWakePayload(payload, adoptedCommentIds), + }) + .where(eq(agentWakeupRequests.id, wakeupRequest.id)); + } + + // executionRunId is NOT stamped here (enqueueWakeup queues the run but + // doesn't start it). It will be stamped in claimQueuedRun() once the run + // transitions to "running" — Fix A (lazy locking). + + return { kind: "queued" as const, run: newRun }; + })) + .catch((error) => { + if (isExternalChatWaitAuthorizationContention(error)) + return { kind: "deferred" as const }; + throw error; + }); + + // Telemetry for the cancelled runs is best-effort background work. + // Fire it here and never await it: none of the lifecycle work below, + // nor this function's return, depends on it, so a slow telemetry + // lookup must not delay them. + for (const cancelledRun of cancelledRunsToEmit) { + void emitAgentTaskRun(db, cancelledRun); + } + + if (outcome.kind === "durable") { + return outcome.receipt.runId ? getRun(outcome.receipt.runId) : null; + } + if (outcome.kind === "deferred" || outcome.kind === "skipped") { + return null; + } + if (outcome.kind === "coalesced") { + await startNextQueuedRunForAgent(agent.id); + return outcome.run; + } + if (outcome.kind === "replayed") { + if (outcome.run.status === "queued") + await startNextQueuedRunForAgent(agent.id); + return outcome.run; + } + + const newRun = outcome.run; + publishLiveEvent({ + companyId: newRun.companyId, + type: "heartbeat.run.queued", + payload: { + runId: newRun.id, + agentId: newRun.agentId, + invocationSource: newRun.invocationSource, + triggerDetail: newRun.triggerDetail, + wakeupRequestId: newRun.wakeupRequestId, + }, + }); + + await startNextQueuedRunForAgent(agent.id); + return newRun; + } + + if (durableRequest) throw new Error("chat_inbound_wakeup_binding_denied"); + + const activeRuns = await db + .select() + .from(heartbeatRuns) + .where( + and( + eq(heartbeatRuns.agentId, agentId), + inArray(heartbeatRuns.status, [ + ...EXECUTION_PATH_HEARTBEAT_RUN_STATUSES, + ]), + ), + ) + .orderBy(desc(heartbeatRuns.createdAt)); + + const sameScopeQueuedRun = activeRuns.find( + (candidate) => + candidate.status === "queued" && + isSameTaskScope(runTaskKey(candidate), taskKey), + ); + const sameScopeScheduledRetryRun = activeRuns.find( + (candidate) => + candidate.status === "scheduled_retry" && + isSameTaskScope(runTaskKey(candidate), taskKey), + ); + const sameScopeRunningRun = activeRuns.find( + (candidate) => + candidate.status === "running" && + isSameTaskScope(runTaskKey(candidate), taskKey), + ); + const shouldQueueFollowupForRunningWake = + Boolean(sameScopeRunningRun) && + !sameScopeQueuedRun && + shouldQueueFollowupForRunningIssueWake({ + contextSnapshot: enrichedContextSnapshot, + wakeCommentId, + }); + // Unscoped manual wakes need their own receipt and execution identity too. + const rawCoalescedTarget = + opts.allowRunCoalescing === false || opts.manualUserWake + ? null + : (sameScopeQueuedRun ?? + sameScopeScheduledRetryRun ?? + (shouldQueueFollowupForRunningWake + ? null + : (sameScopeRunningRun ?? null))); + + // A reviewed chat run attests one conversation, so never fold another + // conversation's reply into it; that reply queues its own run instead. + const coalescedTargetRun = rawCoalescedTarget && + wakeChatConversationsConflict(rawCoalescedTarget.contextSnapshot, enrichedContextSnapshot) + ? null + : filterZombieCoalesceTarget(rawCoalescedTarget, liveRunExecutions); + + if (coalescedTargetRun) { + const mergedContextSnapshot = mergeCoalescedContextSnapshot( + coalescedTargetRun.contextSnapshot, + enrichedContextSnapshot, + { + preserveExistingInteractionContinuation: + coalescedTargetRun.status === "queued" || + coalescedTargetRun.status === "scheduled_retry", + }, + ); + const mergedRun = await db + .update(heartbeatRuns) + .set({ + contextSnapshot: mergedContextSnapshot, + updatedAt: new Date(), + }) + .where(eq(heartbeatRuns.id, coalescedTargetRun.id)) + .returning() + .then((rows) => rows[0] ?? coalescedTargetRun); + + await db.insert(agentWakeupRequests).values({ + ...durableReceiptFields, + companyId: agent.companyId, + agentId, + source, + triggerDetail, + reason, + payload, + status: "coalesced", + coalescedCount: 1, + requestedByActorType: opts.requestedByActorType ?? null, + requestedByActorId: opts.requestedByActorId ?? null, + idempotencyKey: opts.idempotencyKey ?? null, + runId: mergedRun.id, + finishedAt: new Date(), + }); + return mergedRun; + } + + const queueOutcome = await db.transaction(async (tx) => { + await notifyRunDispatch(tx, { companyId: agent.companyId }); + await tx.execute( + sql`select id from agents where id = ${agentId} and company_id = ${agent.companyId} for update`, + ); + + const dailyCapBlock = await getHeartbeatDailyCapBlock( + agent, + policy, + {}, + tx, + ); + if (dailyCapBlock) { + const now = new Date(); + await tx.insert(agentWakeupRequests).values({ + ...durableReceiptFields, + companyId: agent.companyId, + agentId, + source, + triggerDetail, + reason: dailyCapBlock.reason, + payload: { + ...(payload ?? {}), + heartbeatSkip: { + reason: + "Per-agent heartbeat daily cap reached before adapter invocation.", + observed: dailyCapBlock.observed, + limit: dailyCapBlock.limit, + }, + }, + status: "skipped", + requestedByActorType: opts.requestedByActorType ?? null, + requestedByActorId: opts.requestedByActorId ?? null, + idempotencyKey: opts.idempotencyKey ?? null, + finishedAt: now, + }); + if (source === "timer") { + await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers); + await tx + .update(agents) + .set({ + lastHeartbeatAt: now, + updatedAt: now, + }) + .where(eq(agents.id, agentId)); + } + return { kind: "skipped" as const }; + } + + const wakeupRequest = await tx + .insert(agentWakeupRequests) + .values({ + ...durableReceiptFields, + companyId: agent.companyId, + agentId, + source, + triggerDetail, + reason, + payload, + status: "queued", + requestedByActorType: opts.requestedByActorType ?? null, + requestedByActorId: opts.requestedByActorId ?? null, + idempotencyKey: opts.idempotencyKey ?? null, + }) + .returning() + .then((rows) => rows[0]); + + const newRun = await tx + .insert(heartbeatRuns) + .values({ + companyId: agent.companyId, + agentId, + scopeKind: readNonEmptyString(enrichedContextSnapshot.issueId) ? "issue" : "company", + issueId: readNonEmptyString(enrichedContextSnapshot.issueId), + invocationSource: source, + triggerDetail, + status: "queued", + responsibleUserId: await resolveQueuedResponsibleUserId(), + wakeupRequestId: wakeupRequest.id, + contextSnapshot: enrichedContextSnapshot, + sessionIdBefore: sessionBefore, + continuationAttempt, + }) + .returning() + .then((rows) => rows[0]); + + await tx + .update(agentWakeupRequests) + .set({ + runId: newRun.id, + updatedAt: new Date(), + }) + .where(eq(agentWakeupRequests.id, wakeupRequest.id)); + + return { kind: "queued" as const, run: newRun }; + }); + + if (queueOutcome.kind === "skipped") return null; + const newRun = queueOutcome.run; + + publishLiveEvent({ + companyId: newRun.companyId, + type: "heartbeat.run.queued", + payload: { + runId: newRun.id, + agentId: newRun.agentId, + invocationSource: newRun.invocationSource, + triggerDetail: newRun.triggerDetail, + wakeupRequestId: newRun.wakeupRequestId, + }, + }); + + await startNextQueuedRunForAgent(agent.id); + + return newRun; + } + + /** + * Native status commitment deliberately persists dependency/parent wake + * intents in the same transaction as the authoritative status projection. + * Those rows are not runnable until the heartbeat scheduler has applied its + * normal policy, workspace, concurrency, and responsible-user checks. Bridge + * the durable intent into that scheduler here instead of treating a bare + * `agent_wakeup_requests` row as if it were already a queued heartbeat run. + * + * The intent is claimed before dispatch. A deterministic dispatcher actor id + * lets a later sweep recover the narrow process-crash window after the real + * wake was inserted but before the intent was linked to it. Dispatch failure + * only requeues the intent; it never changes the provider run outcome. + */ + async function dispatchPendingNativeStatusWakeups( + input: { + companyId?: string; + limit?: number; + staleClaimMs?: number; + } = {}, + ) { + const now = new Date(); + const staleClaimMs = Math.max(1_000, input.staleClaimMs ?? 60_000); + const candidates = await db + .select() + .from(agentWakeupRequests) + .where( + and( + input.companyId + ? eq(agentWakeupRequests.companyId, input.companyId) + : undefined, + eq(agentWakeupRequests.requestedByActorType, "system"), + eq(agentWakeupRequests.requestedByActorId, "native-status-committer"), + inArray(agentWakeupRequests.status, ["queued", "claimed"]), + isNull(agentWakeupRequests.runId), + ), + ) + .orderBy(asc(agentWakeupRequests.requestedAt)) + .limit(Math.max(1, Math.min(input.limit ?? 100, 500))); + + let dispatched = 0; + let recovered = 0; + let deferred = 0; + const deliveredByIssueScope = new Map< + string, + { runId: string | null; status: string } + >(); + + for (const candidate of candidates) { + const payload = parseObject(candidate.payload); + const wakeContext = parseObject(payload._paperclipWakeContext); + const issueId = + readNonEmptyString(payload.issueId) ?? + readNonEmptyString(payload.taskId) ?? + readNonEmptyString(wakeContext.issueId) ?? + null; + const dispatchActorId = `native-status-wake-dispatch:${candidate.id}`; + const existingDispatch = await db + .select() + .from(agentWakeupRequests) + .where( + and( + eq(agentWakeupRequests.companyId, candidate.companyId), + eq(agentWakeupRequests.requestedByActorType, "system"), + eq(agentWakeupRequests.requestedByActorId, dispatchActorId), + ), + ) + .orderBy(desc(agentWakeupRequests.requestedAt)) + .limit(1) + .then((rows) => rows[0] ?? null); + + if (existingDispatch) { + const recoveredStatus = existingDispatch.runId + ? "coalesced" + : existingDispatch.status === "deferred_issue_execution" + ? "coalesced" + : existingDispatch.status; + await db.transaction(async tx => { + await notifyChatRunMilestones(tx, { companyId: candidate.companyId, issueId: issueId ?? undefined }); + await tx + .update(agentWakeupRequests) + .set({ + status: recoveredStatus, + runId: existingDispatch.runId, + finishedAt: + (existingDispatch.runId ?? existingDispatch.finishedAt) + ? (existingDispatch.finishedAt ?? now) + : null, + error: existingDispatch.error, + updatedAt: now, + }) + .where( + and( + eq(agentWakeupRequests.id, candidate.id), + isNull(agentWakeupRequests.runId), + ), + ); + }); + recovered += 1; + continue; + } + + if ( + candidate.status === "claimed" && + candidate.claimedAt && + now.getTime() - candidate.claimedAt.getTime() < staleClaimMs + ) { + deferred += 1; + continue; + } + + const claimed = await db + .update(agentWakeupRequests) + .set({ + status: "claimed", + claimedAt: now, + error: null, + updatedAt: now, + }) + .where( + and( + eq(agentWakeupRequests.id, candidate.id), + isNull(agentWakeupRequests.runId), + candidate.status === "claimed" + ? eq(agentWakeupRequests.status, "claimed") + : eq(agentWakeupRequests.status, "queued"), + ), + ) + .returning({ id: agentWakeupRequests.id }); + if (claimed.length === 0) continue; + + const scopeKey = issueId + ? `${candidate.companyId}:${candidate.agentId}:${issueId}:${readNativeReviewAssignmentContext(wakeContext)?.nativeReviewInteractionId ?? ""}` + : null; + const priorDelivery = scopeKey + ? deliveredByIssueScope.get(scopeKey) + : null; + if (priorDelivery) { + await db.transaction(async tx => { + await notifyChatRunMilestones(tx, { companyId: candidate.companyId, issueId: issueId ?? undefined }); + await tx + .update(agentWakeupRequests) + .set({ + status: "coalesced", + runId: priorDelivery.runId, + finishedAt: new Date(), + error: null, + updatedAt: new Date(), + }) + .where(eq(agentWakeupRequests.id, candidate.id)); + }); + recovered += 1; + continue; + } + + let completedOnboardingGuard: WakeupOptions["issueStateGuard"]; + if (issueId) { + const targetIssue = await db + .select({ + status: issues.status, + statusVersion: issues.statusVersion, + assigneeAgentId: issues.assigneeAgentId, + }) + .from(issues) + .where( + and( + eq(issues.id, issueId), + eq(issues.companyId, candidate.companyId), + ), + ) + .limit(1) + .then((rows) => rows[0] ?? null); + const nativeReview = candidate.reason === "native_completion_review" + ? await getNativeReviewAssignment(db, { + companyId: candidate.companyId, issueId, agentId: candidate.agentId, + contextSnapshot: wakeContext, + }) + : null; + const onboardingResultReport = targetIssue?.status === "done" && await isCompletedOnboardingHandoffWake(db, { + companyId: candidate.companyId, issueId, agentId: candidate.agentId, + reason: candidate.reason, contextSnapshot: wakeContext, + }); + if (onboardingResultReport) completedOnboardingGuard = { assigneeAgentId: candidate.agentId, statuses: ["done"], statusVersion: targetIssue!.statusVersion }; + if ( + !targetIssue || + (["done", "cancelled"].includes(targetIssue.status) && !onboardingResultReport) || + (targetIssue.assigneeAgentId !== candidate.agentId && !nativeReview) || + (candidate.reason === "native_completion_review" && !nativeReview) + ) { + await db.transaction(async tx => { + await notifyChatInboundWakeNotices(tx, { companyId: candidate.companyId, issueId }); + await tx + .update(agentWakeupRequests) + .set({ + status: "skipped", + finishedAt: new Date(), + error: !targetIssue + ? "Native status wake target no longer exists" + : ["done", "cancelled"].includes(targetIssue.status) + ? `Native status wake target is already ${targetIssue.status}` + : "Native status wake target has a different assignee", + updatedAt: new Date(), + }) + .where(eq(agentWakeupRequests.id, candidate.id)); + }); + recovered += 1; + continue; + } + } + + try { + const wakeRun = await enqueueWakeup(candidate.agentId, { + allowRunCoalescing: false, + source: candidate.source as WakeupOptions["source"], + triggerDetail: (candidate.triggerDetail ?? + "system") as WakeupOptions["triggerDetail"], + reason: candidate.reason, + payload, + idempotencyKey: candidate.idempotencyKey, + requestedByActorType: "system", + requestedByActorId: dispatchActorId, + ...(completedOnboardingGuard ? { issueStateGuard: completedOnboardingGuard } : {}), + contextSnapshot: { + ...wakeContext, + ...(issueId ? { issueId, taskId: issueId } : {}), + wakeReason: candidate.reason, + source: "native_status_decision", + statusDecisionSource: "native_status_decision", + nativeStatusWakeIntentId: candidate.id, + }, + }); + + const delivered = await db + .select() + .from(agentWakeupRequests) + .where( + and( + eq(agentWakeupRequests.companyId, candidate.companyId), + eq(agentWakeupRequests.requestedByActorType, "system"), + eq(agentWakeupRequests.requestedByActorId, dispatchActorId), + ), + ) + .orderBy(desc(agentWakeupRequests.requestedAt)) + .limit(1) + .then((rows) => rows[0] ?? null); + + // The committer intent is the durable outbox entry. When admission is + // blocked, enqueueWakeup creates a separate deferred dispatch receipt; + // leave the original intent coalesced so the release drain cannot + // promote both rows (the original has no nativeStatusWakeIntentId + // provenance and would otherwise run once before the dispatch receipt). + const deliveredStatus = delivered?.status ?? "queued"; + await db.transaction(async tx => { + await notifyChatRunMilestones(tx, { companyId: candidate.companyId, issueId: issueId ?? undefined }); + await tx + .update(agentWakeupRequests) + .set({ + status: wakeRun || deliveredStatus === "deferred_issue_execution" + ? "coalesced" + : deliveredStatus, + runId: wakeRun?.id ?? delivered?.runId ?? null, + finishedAt: + wakeRun || delivered?.finishedAt + ? (delivered?.finishedAt ?? new Date()) + : null, + error: delivered?.error ?? null, + updatedAt: new Date(), + }) + .where(eq(agentWakeupRequests.id, candidate.id)); + }); + + if (scopeKey) { + deliveredByIssueScope.set(scopeKey, { + runId: wakeRun?.id ?? delivered?.runId ?? null, + status: wakeRun ? "coalesced" : (delivered?.status ?? "queued"), + }); + } + + if (wakeRun) dispatched += 1; + else deferred += 1; + } catch (error) { + await db + .update(agentWakeupRequests) + .set({ + status: "queued", + claimedAt: null, + error: + error instanceof Error + ? error.message.slice(0, 1_000) + : String(error).slice(0, 1_000), + updatedAt: new Date(), + }) + .where( + and( + eq(agentWakeupRequests.id, candidate.id), + eq(agentWakeupRequests.status, "claimed"), + isNull(agentWakeupRequests.runId), + ), + ); + logger.warn( + { + err: error, + wakeupRequestId: candidate.id, + agentId: candidate.agentId, + }, + "failed to dispatch persisted native status wake intent", + ); + deferred += 1; + } + } + + return { scanned: candidates.length, dispatched, recovered, deferred }; + } + + return { + enqueueWakeup, + dispatchPendingNativeStatusWakeups, + startNextQueuedRunForAgent, + countRunningRunsForAgent, + releaseRunClaimedJustBeforeSuppression, + claimQueuedRun, + withChatControlRecoveryGate, + trackWakeup, + resumeQueuedRuns, + resumeQueuedRunsAfterDrain, + parseHeartbeatPolicy, + nextTimerHeartbeatAt, + claimDueTimerHeartbeat, + }; +} diff --git a/server/src/services/heartbeat/retries.ts b/server/src/services/heartbeat/retries.ts index 0e6a67a4dcb..d2e72ab0b6e 100644 --- a/server/src/services/heartbeat/retries.ts +++ b/server/src/services/heartbeat/retries.ts @@ -1 +1,1934 @@ +import { notifyRunDispatch } from "../run-dispatch-notifications.js"; import { notifyStatusCardIssueChange } from "../status-card-notifications.js"; +import { deriveTaskKeyWithHeartbeatFallback } from "./run-state.js"; +import { infrastructureRetryDelayMs } from "../system-recovery-state.js"; +import { + WORKSPACE_VALIDATION_FAILURE_CODE, + isWorkspaceValidationFailedRun, + readWorkspaceValidationPayloadFromRun, +} from "./workspaces.js"; +import { admitExplicitContinuationRetry } from "../explicit-native-continuation.js"; +import { + CONVERSATION_CONTINUATION_POLICY, + hasConversationContinuationPolicy, +} from "../conversation-continuation.js"; +import { legacyExecutionNeedsReconciliationWithEvidence } from "../legacy-execution-recovery.js"; +import { + executionFailureRetryCount, + executionRetryAttemptCount, + accountingForScheduledRetry, +} from "../execution-recovery-attempt.js"; +import { WORKSPACE_GIT_SCAN_ERROR_CODES } from "../workspace-git-operation-scheduler.js"; +import { CONTROLLER_LEASE_LOST_FAILURE_CODE } from "../legacy-controller-lease.js"; +import { randomUUID } from "node:crypto"; +import { + and, + asc, + desc, + eq, + inArray, + ne, + notInArray, + or, + sql, +} from "drizzle-orm"; +import type { Db } from "@paperclipai/db"; +import { + agents, + agentWakeupRequests, + executionWorkspaces, + heartbeatRuns, + issues, +} from "@paperclipai/db"; +import { + HttpError, + notFound, +} from "../../errors.js"; +import { logger } from "../../middleware/logger.js"; +import { parseObject } from "../../adapters/utils.js"; +import { logActivity } from "../activity-log.js"; +import { readContinuationAttempt } from "../recovery/index.js"; +import { withRecoveryContext } from "../recovery/status-only-context.js"; +import { + MAX_TURN_CONTINUATION_RETRY_REASON, + WORKSPACE_BUSY_RETRY_REASON, + AI_CONNECTION_BUSY_RETRY_REASON, + AI_CONNECTION_POOL_WAIT_RETRY_REASON, + INTERACTION_CONTINUATION_INFRA_RETRY_REASON, + INTERACTION_CONTINUATION_INFRA_WAKE_REASON, + isNonAssigneeWorkspaceBusyRetry, + isResolvedInteractionContinuationWakeContext, +} from "../../modules/run-dispatch/index.js"; + +import type { AppendHeartbeatRunEventInput } from "../heartbeat-run-events.js"; +import type { evaluateAgentInvokabilityFromDb } from "../agent-invokability.js"; +import type { createHeartbeatRunState } from "./run-state.js"; +import type { createHeartbeatRunPreparation } from "./run-preparation.js"; +import type { createRunDispatch, PostCommitEffect } from "../../modules/run-dispatch/index.js"; + +type HeartbeatRun = typeof heartbeatRuns.$inferSelect; +type HeartbeatRunState = ReturnType; +type PlanApprovalResumeFailure = { + run: HeartbeatRun; + issueId: string | null; + attempt: number; + maxAttempts: number; +}; + +/** The retry module owns scheduling writes; the service owns lifecycle effects. */ +export interface HeartbeatRetryDependencies { + hasCaseHandoffForRun: (run: Pick) => Promise; + runDispatch: Pick, + "evaluateScheduledRetryGate" | "promoteDueScheduledRetries" | "promoteScheduledRetry" | "nextScheduledRetryAt">; + getAgent: (agentId: string) => Promise; + getRun: ( + runId: string, + options?: Parameters[1], + ) => Promise> | null | undefined>; + resolveSessionBeforeForWakeup: HeartbeatRunState["resolveSessionBeforeForWakeup"]; + resolveResponsibleUserIdForRunContext: ReturnType["resolveResponsibleUserIdForRunContext"]; + getAgentInvokability: (agent: typeof agents.$inferSelect | null | undefined) => ReturnType; + appendRunEvent: (run: HeartbeatRun, event: { + eventType: string; + stream?: "system" | "stdout" | "stderr"; + level?: "info" | "warn" | "error"; + color?: string; + message?: string; + payload?: Record; + retryExhaustion?: AppendHeartbeatRunEventInput["retryExhaustion"]; + }) => Promise; + escalatePlanApprovalResumeFailureNeedsAttention: (input: PlanApprovalResumeFailure) => Promise; + recordPlanApprovalResumeFailureRetry: (input: PlanApprovalResumeFailure & { retryRunId: string | null }) => Promise; + setRunStatusIfRunning: (runId: string, status: string, patch?: Partial) => Promise<{ + updated: boolean; + run: HeartbeatRun | null; + }>; + setWakeupStatus: (wakeupRequestId: string | null | undefined, status: string, patch?: Partial) => Promise; + releaseIssueExecutionAndPromote: (run: Pick) => Promise; + finalizeAgentStatus: ( + agentId: string, + outcome: "succeeded" | "interrupted" | "failed" | "cancelled" | "timed_out", + failureReason?: string | null, + options?: { keepIdleOnFailure?: boolean; wasFirstHeartbeat?: boolean }, + ) => Promise; + getWorktreeExecutionCutoff: () => Promise; + applyRunDispatchPostCommitEffects: (effects: PostCommitEffect[]) => void; +} + +export const BOUNDED_TRANSIENT_HEARTBEAT_RETRY_DELAYS_MS = [ + 30_000, 60_000, +] as const; + +const BOUNDED_TRANSIENT_HEARTBEAT_RETRY_REASON = "transient_failure"; + +const BOUNDED_TRANSIENT_HEARTBEAT_RETRY_WAKE_REASON = "transient_failure_retry"; + +export function isTransientWorkspaceGitScanCode(code: string | null | undefined): boolean { + return code === WORKSPACE_GIT_SCAN_ERROR_CODES.timeout || code === WORKSPACE_GIT_SCAN_ERROR_CODES.saturated; +} + +export const BOUNDED_TRANSIENT_HEARTBEAT_RETRY_MAX_ATTEMPTS = + BOUNDED_TRANSIENT_HEARTBEAT_RETRY_DELAYS_MS.length; + +const INTERACTION_CONTINUATION_INFRA_MAX_ATTEMPTS = 2; + +const MAX_TURN_CONTINUATION_LIVE_RUN_STATUSES = [ + "scheduled_retry", + "queued", + "running", +] as const; + +export const WORKSPACE_BUSY_RETRY_WAKE_REASON = "workspace_busy_retry"; + +export const WORKSPACE_BUSY_ERROR_CODE = "workspace_busy"; + +export const WORKSPACE_BUSY_RETRY_BASE_DELAY_MS = 60 * 1000; + +export const WORKSPACE_BUSY_RETRY_JITTER_MS = 60 * 1000; + +// Preserve the one-hour shared-workspace holder cutoff independently of the +// informational output-silence warnings. Warning sooner must not let another +// run overtake a quiet holder and mutate its shared workspace. +export const WORKSPACE_BUSY_HOLDER_STALE_AFTER_MS = 60 * 60 * 1000; + +// Issue-level executionWorkspaceSettings.mode values that unambiguously opt an +// issue's runs out of the shared project workspace, and therefore out of +// shared-workspace serialization ("isolated" is the legacy alias +// parseIssueExecutionWorkspaceSettings normalizes to isolated_workspace). Any +// other value — including agent_default and an absent mode — may still resolve +// to the shared workspace and counts as a holder. +const ISOLATED_EXECUTION_WORKSPACE_MODES = [ + "isolated_workspace", + "operator_branch", + "isolated", +] as const; + +type CodexTransientFallbackMode = + | "same_session" + | "safer_invocation" + | "fresh_session" + | "fresh_session_safer_invocation"; + +export interface SharedWorkspaceHolder { + runId: string; + agentId: string; + issueId: string; + issueIdentifier: string | null; +} + +export interface ExecutionWorkspaceHolder extends SharedWorkspaceHolder { + // Silent past WORKSPACE_BUSY_HOLDER_STALE_AFTER_MS but still running. + stale: boolean; +} + +// Pre-dispatch gate outcome: another running run currently holds the issue's +// shared project workspace, or the shared execution workspace it reuses. Not a +// failure — the run is parked as a bounded scheduled retry and re-attempted +// once the holder finishes, so two agents never mutate the same working tree +// concurrently. +export class WorkspaceBusyDeferral extends Error { + code = WORKSPACE_BUSY_ERROR_CODE; + holder: SharedWorkspaceHolder; + projectWorkspaceId: string | null; + executionWorkspaceId: string | null; + deferralAttempt: number; + wasIssueAssignee: boolean; + + constructor(input: { + holder: SharedWorkspaceHolder; + projectWorkspaceId?: string | null; + executionWorkspaceId?: string | null; + deferralAttempt: number; + wasIssueAssignee: boolean; + }) { + super( + `Shared ${input.executionWorkspaceId ? "execution" : "project"} workspace is busy: run ${input.holder.runId} (issue ${ + input.holder.issueIdentifier ?? input.holder.issueId + }) is still running`, + ); + this.name = "WorkspaceBusyDeferral"; + this.holder = input.holder; + this.projectWorkspaceId = input.projectWorkspaceId ?? null; + this.executionWorkspaceId = input.executionWorkspaceId ?? null; + this.deferralAttempt = input.deferralAttempt; + this.wasIssueAssignee = input.wasIssueAssignee; + } +} + +export function isWorkspaceBusyDeferral( + error: unknown, +): error is WorkspaceBusyDeferral { + return error instanceof WorkspaceBusyDeferral; +} + +export function computeWorkspaceBusyRetryDelayMs( + random: () => number = Math.random, +) { + const jitter = Math.min(Math.max(random(), 0), 1); + return ( + WORKSPACE_BUSY_RETRY_BASE_DELAY_MS + + Math.floor(jitter * WORKSPACE_BUSY_RETRY_JITTER_MS) + ); +} + +function resolveCodexTransientFallbackMode( + attempt: number, +): CodexTransientFallbackMode { + if (attempt <= 1) return "same_session"; + if (attempt === 2) return "safer_invocation"; + if (attempt === 3) return "fresh_session"; + return "fresh_session_safer_invocation"; +} + +export function readHeartbeatRunErrorFamily( + run: Pick, +) { + const resultJson = parseObject(run.resultJson); + const persistedFamily = readNonEmptyString(resultJson.errorFamily); + if (persistedFamily) return persistedFamily; + + if (run.errorCode === "provider_quota") { + return "provider_quota"; + } + if ( + run.errorCode === "codex_transient_upstream" || + run.errorCode === "claude_transient_upstream" || + run.errorCode === "codex_harness_crash" + ) { + return "transient_upstream"; + } + return null; +} + +function readTransientRetryNotBeforeFromRun( + run: Pick, +) { + const resultJson = parseObject(run.resultJson); + const value = resultJson.retryNotBefore ?? resultJson.transientRetryNotBefore; + if (!( + typeof value === "string" || + typeof value === "number" || + value instanceof Date + )) { + return null; + } + const parsed = new Date(value); + return Number.isNaN(parsed.getTime()) ? null : parsed; +} + +export function readTransientRecoveryContractFromRun( + run: Pick, +) { + const errorFamily = readHeartbeatRunErrorFamily(run); + return errorFamily === "transient_upstream" || + errorFamily === "provider_quota" + ? { + errorFamily, + retryNotBefore: readTransientRetryNotBeforeFromRun(run), + } + : null; +} + +function isSpawnLikeFailureMessage(value: unknown) { + if (typeof value !== "string") return false; + return /failed to start command|spawn\b|\bENOENT\b/i.test(value); +} + +// A sandbox provider plugin's worker can be briefly down during its own +// restart window (e.g. a rolling deploy of the plugin worker process). Lease +// acquisition fails immediately in that window, but the condition is +// transient and self-healing, so it must be treated as retryable +// infrastructure rather than a terminal setup failure. See +// resolveSandboxProviderPlugin's "worker_unavailable" message in +// environment-runtime.ts (":808"), e.g. 'Sandbox provider "kubernetes" is +// installed via plugin "acme.kubernetes-sandbox-provider", but its worker is +// not running.' +// +// This is anchored on both "is installed via plugin" and "but its worker is +// not running" so it does not also match plugin-environment-driver.ts's +// unrelated, permanent "provider not installed" message ('Sandbox provider +// "X" is not installed or its plugin worker is not running.'), which +// coincidentally contains the same "worker is not running" substring but +// describes a terminal condition that must not be retried. +function isSandboxProviderWorkerUnavailableFailureMessage(value: unknown) { + if (typeof value !== "string") return false; + return /sandbox provider .* is installed via plugin .* but its worker is not running/i.test( + value, + ); +} + +function isRetryableInteractionContinuationInfrastructureFailure( + run: Pick< + typeof heartbeatRuns.$inferSelect, + "error" | "errorCode" | "resultJson" + >, +) { + // `controller_lease_lost` is only recorded before provider dispatch, so the + // retry cannot repeat started work. + if ( + run.errorCode === WORKSPACE_VALIDATION_FAILURE_CODE || + run.errorCode === "process_lost" || + run.errorCode === CONTROLLER_LEASE_LOST_FAILURE_CODE + ) { + return true; + } + + if (run.errorCode !== "adapter_failed" && run.errorCode !== "setup_failed") + return false; + + const resultJson = parseObject(run.resultJson); + return ( + isSpawnLikeFailureMessage(run.error) || + isSpawnLikeFailureMessage(resultJson.errorMessage) || + isSpawnLikeFailureMessage(resultJson.message) || + isSandboxProviderWorkerUnavailableFailureMessage(run.error) || + isSandboxProviderWorkerUnavailableFailureMessage(resultJson.errorMessage) || + isSandboxProviderWorkerUnavailableFailureMessage(resultJson.message) + ); +} + +export function computeBoundedTransientHeartbeatRetrySchedule( + attempt: number, + now = new Date(), + random: () => number = Math.random, +) { + if (!Number.isInteger(attempt) || attempt <= 0) return null; + const baseDelayMs = BOUNDED_TRANSIENT_HEARTBEAT_RETRY_DELAYS_MS[attempt - 1]; + if (typeof baseDelayMs !== "number") return null; + const delayMs = infrastructureRetryDelayMs(attempt, random()); + return { + attempt, + baseDelayMs, + delayMs, + dueAt: new Date(now.getTime() + delayMs), + maxAttempts: BOUNDED_TRANSIENT_HEARTBEAT_RETRY_MAX_ATTEMPTS, + }; +} + +export function normalizeAgentNameKey(value: string | null | undefined) { + if (typeof value !== "string") return null; + const normalized = value.trim().toLowerCase(); + return normalized.length > 0 ? normalized : null; +} + +function readNonEmptyString(value: unknown): string | null { + return typeof value === "string" && value.trim().length > 0 ? value : null; +} + +/** Retry persistence uses the service database and explicit lifecycle callbacks. */ +export function createHeartbeatRetries(db: Db, dependencies: HeartbeatRetryDependencies) { + const { + hasCaseHandoffForRun, + appendRunEvent, + escalatePlanApprovalResumeFailureNeedsAttention, + getAgentInvokability, + runDispatch, + resolveSessionBeforeForWakeup, + resolveResponsibleUserIdForRunContext, + recordPlanApprovalResumeFailureRetry, + setRunStatusIfRunning, + setWakeupStatus, + getRun, + getAgent, + releaseIssueExecutionAndPromote, + finalizeAgentStatus, + getWorktreeExecutionCutoff, + applyRunDispatchPostCommitEffects, + } = dependencies; + + async function scheduleBoundedRetryForRun( + run: typeof heartbeatRuns.$inferSelect, + agent: typeof agents.$inferSelect, + opts?: { + now?: Date; + random?: () => number; + retryReason?: string; + wakeReason?: string; + maxAttempts?: number; + delayMs?: number; + }, + ) { + if (await hasCaseHandoffForRun(run)) return { outcome: "not_scheduled" as const, + reason: "The durable case handoff owns successor work.", issueId: readNonEmptyString(run.contextSnapshot?.issueId) }; + if (["openai_dot", "muse"].includes(String(parseObject(agent.adapterConfig).provider)) + || ["openai_dot", "muse"].includes(String(parseObject(parseObject(parseObject(run.runnerProfileJson).nativeExecutionInput).provider).kind))) { + return { outcome: "not_scheduled" as const, reason: "Dot external execution must be reconciled before a new assignment; Paperclip cannot confirm its external stop.", issueId: readNonEmptyString(run.contextSnapshot?.issueId) }; + } + if (run.errorCode === "provider_tool_definition_invalid") { + return { outcome: "not_scheduled" as const, + reason: "Repair the invalid tool definitions before starting a new attempt.", + issueId: readNonEmptyString(run.contextSnapshot?.issueId) }; + } + if (Array.isArray(run.contextSnapshot?.chatCompletionDeliveryIds) && + run.contextSnapshot.chatCompletionDeliveryIds.some(id => typeof id === "string")) { + return { outcome: "not_scheduled" as const, reason: "The completion outbox owns this reply's retry budget and publication identity.", + errorCode: "chat_completion_outbox_owns_retry" as const, issueId: readNonEmptyString(run.contextSnapshot.issueId) }; + } + const now = opts?.now ?? new Date(); + const retryReason = + opts?.retryReason ?? BOUNDED_TRANSIENT_HEARTBEAT_RETRY_REASON; + const wakeReason = + opts?.wakeReason ?? BOUNDED_TRANSIENT_HEARTBEAT_RETRY_WAKE_REASON; + const maxAttempts = Math.max( + 0, + Math.floor( + opts?.maxAttempts ?? BOUNDED_TRANSIENT_HEARTBEAT_RETRY_MAX_ATTEMPTS, + ), + ); + const consumedAttempts = executionRetryAttemptCount(run, retryReason); + const nextAttempt = consumedAttempts + 1; + const computedBaseSchedule = + opts?.delayMs != null + ? nextAttempt <= maxAttempts + ? { + attempt: nextAttempt, + baseDelayMs: Math.max(0, Math.floor(opts.delayMs)), + delayMs: Math.max(0, Math.floor(opts.delayMs)), + dueAt: new Date( + now.getTime() + Math.max(0, Math.floor(opts.delayMs)), + ), + maxAttempts, + } + : null + : nextAttempt <= maxAttempts + ? computeBoundedTransientHeartbeatRetrySchedule( + nextAttempt, + now, + opts?.random, + ) + : null; + const baseSchedule = computedBaseSchedule + ? { ...computedBaseSchedule, maxAttempts } + : null; + const transientRecovery = + retryReason === BOUNDED_TRANSIENT_HEARTBEAT_RETRY_REASON + ? readTransientRecoveryContractFromRun(run) + : null; + const codexTransientFallbackMode = + agent.adapterType === "codex_local" && + transientRecovery?.errorFamily === "transient_upstream" + ? resolveCodexTransientFallbackMode(nextAttempt) + : null; + const transientRetryNotBefore = transientRecovery?.retryNotBefore ?? null; + const contextSnapshot = parseObject(run.contextSnapshot); + // A retry inherits the durable authorization scope of its source run. Do + // not promote an untrusted or legacy contextSnapshot.issueId into a new + // issue binding: that could either violate the FK or misclassify history. + const issueId = run.scopeKind === "issue" ? run.issueId : null; + + if (!baseSchedule) { + const exhaustion = { + retryReason, + scheduledRetryAttempt: consumedAttempts, + maxAttempts, + }; + await appendRunEvent(run, { + eventType: "lifecycle", + stream: "system", + level: "warn", + message: `Bounded retry exhausted after ${consumedAttempts} scheduled attempts; no further automatic retry will be queued`, + payload: exhaustion, + retryExhaustion: exhaustion, + }); + if (retryReason === INTERACTION_CONTINUATION_INFRA_RETRY_REASON) { + await escalatePlanApprovalResumeFailureNeedsAttention({ + run, + issueId, + attempt: Math.min( + consumedAttempts, + maxAttempts, + ), + maxAttempts, + }).catch((error) => { + logger.warn( + { err: error, runId: run.id, issueId }, + "failed to escalate exhausted plan-approval resume failure", + ); + }); + } + return { + outcome: "retry_exhausted" as const, + attempt: nextAttempt, + maxAttempts, + }; + } + + if (await legacyExecutionNeedsReconciliationWithEvidence(db, run)) { + return { + outcome: "not_scheduled" as const, + reason: + "Reconcile the previous execution before retrying; safe provider recovery is unavailable.", + errorCode: "legacy_execution_requires_reconciliation" as const, + issueId: readNonEmptyString(run.contextSnapshot?.issueId), + }; + } + if (retryReason !== MAX_TURN_CONTINUATION_RETRY_REASON) { + const invokability = await getAgentInvokability(agent); + if (!invokability.invokable) { + await appendRunEvent(run, { + eventType: "lifecycle", + stream: "system", + level: "warn", + message: + "Scheduled retry suppressed because the agent is not invokable", + payload: { + retryReason, + scheduledRetryAttempt: nextAttempt, + maxAttempts, + reason: invokability.reason, + invalidOrgChain: invokability.invalidOrgChain, + ...invokability.details, + }, + }); + return { + outcome: "not_scheduled" as const, + reason: + "Scheduled retry suppressed because the agent is not invokable", + errorCode: "agent_not_invokable" as const, + issueId, + }; + } + } + + const schedule = + transientRetryNotBefore && + transientRetryNotBefore.getTime() > baseSchedule.dueAt.getTime() + ? { + ...baseSchedule, + dueAt: transientRetryNotBefore, + delayMs: Math.max( + 0, + transientRetryNotBefore.getTime() - now.getTime(), + ), + } + : baseSchedule; + + const requiresIssueGate = + isTransientWorkspaceGitScanCode(run.errorCode) || + hasConversationContinuationPolicy(run.resultJson) || + (retryReason === AI_CONNECTION_BUSY_RETRY_REASON || retryReason === AI_CONNECTION_POOL_WAIT_RETRY_REASON) || + retryReason === MAX_TURN_CONTINUATION_RETRY_REASON || + retryReason === INTERACTION_CONTINUATION_INFRA_RETRY_REASON; + if (requiresIssueGate) { + const gate = await runDispatch.evaluateScheduledRetryGate({ + runId: run.id, + companyId: run.companyId, + retryReasonOverride: retryReason, + now, + }); + if (!gate.allowed) { + await appendRunEvent(run, { + eventType: "lifecycle", + stream: "system", + level: "warn", + message: gate.reason, + payload: { + retryReason, + scheduledRetryAttempt: nextAttempt, + maxAttempts, + ...gate.details, + }, + }); + return { + outcome: "not_scheduled" as const, + reason: gate.reason, + errorCode: gate.errorCode, + issueId: gate.issueId, + }; + } + } + const taskKey = deriveTaskKeyWithHeartbeatFallback(contextSnapshot, null); + const sessionBefore = await resolveSessionBeforeForWakeup(agent, taskKey); + const interactionContinuationPayload = + retryReason === INTERACTION_CONTINUATION_INFRA_RETRY_REASON + ? { + mutation: "interaction", + interactionId: readNonEmptyString(contextSnapshot.interactionId), + interactionKind: readNonEmptyString( + contextSnapshot.interactionKind, + ), + interactionStatus: readNonEmptyString( + contextSnapshot.interactionStatus, + ), + continuationPolicy: readNonEmptyString( + contextSnapshot.continuationPolicy, + ), + } + : {}; + const workspaceValidationRetryPayload = + retryReason === INTERACTION_CONTINUATION_INFRA_RETRY_REASON && + isWorkspaceValidationFailedRun(run) + ? readWorkspaceValidationPayloadFromRun(run) + : null; + const shouldQuarantineWorkspaceForRetry = + workspaceValidationRetryPayload !== null && + Object.keys(workspaceValidationRetryPayload).length > 0; + const retryContextSnapshot: Record = withRecoveryContext( + { + ...contextSnapshot, + executionRetryAccounting: accountingForScheduledRetry(run, retryReason, schedule.attempt), + retryOfRunId: run.id, + wakeReason, + retryReason, + ...(retryReason === WORKSPACE_BUSY_RETRY_REASON + ? { + failureRetriesBeforeWorkspaceWait: + executionFailureRetryCount(run), + } + : {}), + ...((retryReason === AI_CONNECTION_BUSY_RETRY_REASON || retryReason === AI_CONNECTION_POOL_WAIT_RETRY_REASON) + ? { failureRetriesBeforeAiConnectionWait: executionFailureRetryCount(run) } + : {}), + ...(shouldQuarantineWorkspaceForRetry + ? { + workspaceValidationRecovery: { + strategy: "quarantine_failed_workspace_and_retry_clean", + sourceRunId: run.id, + reason: + readNonEmptyString(workspaceValidationRetryPayload?.reason) ?? + WORKSPACE_VALIDATION_FAILURE_CODE, + fingerprint: readNonEmptyString( + workspaceValidationRetryPayload?.fingerprint, + ), + failedExecutionWorkspaceId: readNonEmptyString( + workspaceValidationRetryPayload?.executionWorkspaceId, + ), + }, + } + : {}), + ...(transientRecovery + ? { errorFamily: transientRecovery.errorFamily } + : {}), + scheduledRetryAttempt: schedule.attempt, + scheduledRetryAt: schedule.dueAt.toISOString(), + ...(transientRetryNotBefore + ? { transientRetryNotBefore: transientRetryNotBefore.toISOString() } + : {}), + ...(transientRecovery?.errorFamily === "provider_quota" && + transientRetryNotBefore + ? { + providerQuotaRetryNotBefore: + transientRetryNotBefore.toISOString(), + } + : {}), + ...(codexTransientFallbackMode ? { codexTransientFallbackMode } : {}), + }, + "normal_model", + ); + // A successor resolves current assignments; the failed run retains its snapshot. + delete retryContextSnapshot.skillVersions; + delete retryContextSnapshot.skillSelectionSnapshot; + delete retryContextSnapshot.shadowedSkills; + const responsibleUserId = await resolveResponsibleUserIdForRunContext( + run, + retryContextSnapshot, + ); + const continuationRetryIdempotencyKey = + retryReason === MAX_TURN_CONTINUATION_RETRY_REASON + ? `max-turn-continuation:${run.companyId}:${issueId ?? "no-issue"}:${run.id}:${schedule.attempt}` + : retryReason === INTERACTION_CONTINUATION_INFRA_RETRY_REASON + ? `interaction-continuation:${run.companyId}:${issueId ?? "no-issue"}:${run.id}:${schedule.attempt}` + : null; + + type ScheduledRetryTransactionResult = + | { + outcome: "scheduled"; + run: typeof heartbeatRuns.$inferSelect; + reusedExisting: boolean; + } + | { + outcome: "not_scheduled"; + reason: string; + errorCode: + | "issue_not_found" + | "issue_reassigned" + | "issue_cancelled" + | "issue_terminal_status" + | "issue_not_in_progress" + | "continuation_user_authorization_missing" + | "issue_execution_lock_changed"; + issueId: string | null; + details: Record; + }; + + const scheduleResult = await db.transaction( + async (tx): Promise => { + // All automatic failure paths share the same predecessor claim. A + // duplicate monitor, restart sweep or wake must reuse its successor. + if ( + retryReason !== MAX_TURN_CONTINUATION_RETRY_REASON && + retryReason !== INTERACTION_CONTINUATION_INFRA_RETRY_REASON + ) { + if (issueId) + await tx.execute( + sql`select id from issues where company_id = ${run.companyId} and id = ${issueId} for update`, + ); + await tx.execute( + sql`select id from heartbeat_runs where company_id = ${run.companyId} and id = ${run.id} for update`, + ); + const [existing] = await tx + .select() + .from(heartbeatRuns) + .where( + and( + eq(heartbeatRuns.companyId, run.companyId), + eq(heartbeatRuns.retryOfRunId, run.id), + ), + ) + .limit(1); + if (existing) + return { + outcome: "scheduled", + run: existing, + reusedExisting: true, + }; + } + if (retryReason === INTERACTION_CONTINUATION_INFRA_RETRY_REASON) { + if (issueId) { + await tx.execute( + sql`select id from issues where company_id = ${run.companyId} and id = ${issueId} for update`, + ); + } else { + await tx.execute( + sql`select id from heartbeat_runs where company_id = ${run.companyId} and id = ${run.id} for update`, + ); + } + + const existingContinuation = await tx + .select() + .from(heartbeatRuns) + .where( + and( + eq(heartbeatRuns.companyId, run.companyId), + eq(heartbeatRuns.retryOfRunId, run.id), + eq(heartbeatRuns.scheduledRetryReason, retryReason), + eq(heartbeatRuns.scheduledRetryAttempt, schedule.attempt), + inArray(heartbeatRuns.status, [ + ...MAX_TURN_CONTINUATION_LIVE_RUN_STATUSES, + ]), + issueId + ? sql`${heartbeatRuns.contextSnapshot} ->> 'issueId' = ${issueId}` + : sql`${heartbeatRuns.contextSnapshot} ->> 'issueId' is null`, + ), + ) + .orderBy(asc(heartbeatRuns.createdAt), asc(heartbeatRuns.id)) + .limit(1) + .then((rows) => rows[0] ?? null); + + if (existingContinuation) { + if (existingContinuation.wakeupRequestId) { + const existingWakeup = await tx + .select({ coalescedCount: agentWakeupRequests.coalescedCount }) + .from(agentWakeupRequests) + .where( + eq( + agentWakeupRequests.id, + existingContinuation.wakeupRequestId, + ), + ) + .then((rows) => rows[0] ?? null); + + await tx + .update(agentWakeupRequests) + .set({ + coalescedCount: (existingWakeup?.coalescedCount ?? 0) + 1, + updatedAt: now, + }) + .where( + eq( + agentWakeupRequests.id, + existingContinuation.wakeupRequestId, + ), + ); + } + + return { + outcome: "scheduled", + run: existingContinuation, + reusedExisting: true, + }; + } + } + + if (retryReason === MAX_TURN_CONTINUATION_RETRY_REASON) { + if (issueId) { + await tx.execute( + sql`select id from issues where company_id = ${run.companyId} and id = ${issueId} for update`, + ); + } else { + await tx.execute( + sql`select id from heartbeat_runs where company_id = ${run.companyId} and id = ${run.id} for update`, + ); + } + + const existingContinuation = await tx + .select() + .from(heartbeatRuns) + .where( + and( + eq(heartbeatRuns.companyId, run.companyId), + eq(heartbeatRuns.retryOfRunId, run.id), + eq(heartbeatRuns.scheduledRetryReason, retryReason), + eq(heartbeatRuns.scheduledRetryAttempt, schedule.attempt), + inArray(heartbeatRuns.status, [ + ...MAX_TURN_CONTINUATION_LIVE_RUN_STATUSES, + ]), + issueId + ? sql`${heartbeatRuns.contextSnapshot} ->> 'issueId' = ${issueId}` + : sql`${heartbeatRuns.contextSnapshot} ->> 'issueId' is null`, + ), + ) + .orderBy(asc(heartbeatRuns.createdAt), asc(heartbeatRuns.id)) + .limit(1) + .then((rows) => rows[0] ?? null); + + if (existingContinuation) { + if (existingContinuation.wakeupRequestId) { + const existingWakeup = await tx + .select({ coalescedCount: agentWakeupRequests.coalescedCount }) + .from(agentWakeupRequests) + .where( + eq( + agentWakeupRequests.id, + existingContinuation.wakeupRequestId, + ), + ) + .then((rows) => rows[0] ?? null); + + await tx + .update(agentWakeupRequests) + .set({ + coalescedCount: (existingWakeup?.coalescedCount ?? 0) + 1, + updatedAt: now, + }) + .where( + eq( + agentWakeupRequests.id, + existingContinuation.wakeupRequestId, + ), + ); + } + + return { + outcome: "scheduled", + run: existingContinuation, + reusedExisting: true, + }; + } + + if (issueId) { + const lockedIssue = await tx + .select({ + id: issues.id, + status: issues.status, + assigneeAgentId: issues.assigneeAgentId, + executionRunId: issues.executionRunId, + }) + .from(issues) + .where( + and( + eq(issues.id, issueId), + eq(issues.companyId, run.companyId), + ), + ) + .then((rows) => rows[0] ?? null); + + if (!lockedIssue) { + return { + outcome: "not_scheduled", + reason: + "Scheduled max-turn continuation suppressed because the target issue no longer exists", + errorCode: "issue_not_found", + issueId, + details: { issueId }, + }; + } + + if (lockedIssue.assigneeAgentId !== run.agentId) { + return { + outcome: "not_scheduled", + reason: + "Scheduled max-turn continuation suppressed because issue ownership changed", + errorCode: "issue_reassigned", + issueId, + details: { + issueId, + previousAssigneeAgentId: run.agentId, + currentAssigneeAgentId: lockedIssue.assigneeAgentId, + }, + }; + } + + if ( + lockedIssue.status === "cancelled" || + lockedIssue.status === "done" + ) { + return { + outcome: "not_scheduled", + reason: `Scheduled max-turn continuation suppressed because issue reached terminal status (${lockedIssue.status})`, + errorCode: + lockedIssue.status === "cancelled" + ? "issue_cancelled" + : "issue_terminal_status", + issueId, + details: { issueId, currentStatus: lockedIssue.status }, + }; + } + + if (lockedIssue.status !== "in_progress") { + return { + outcome: "not_scheduled", + reason: `Scheduled max-turn continuation suppressed because issue is no longer in_progress (current status: ${lockedIssue.status})`, + errorCode: "issue_not_in_progress", + issueId, + details: { + issueId, + currentStatus: lockedIssue.status, + requiredStatus: "in_progress", + }, + }; + } + + if (lockedIssue.executionRunId !== run.id) { + return { + outcome: "not_scheduled", + reason: + "Scheduled max-turn continuation suppressed because the issue execution lock belongs to a different run", + errorCode: "issue_execution_lock_changed", + issueId, + details: { + issueId, + expectedExecutionRunId: run.id, + currentExecutionRunId: lockedIssue.executionRunId, + }, + }; + } + } + } + + if ( + (retryReason === AI_CONNECTION_BUSY_RETRY_REASON || retryReason === AI_CONNECTION_POOL_WAIT_RETRY_REASON) && issueId && + !isNonAssigneeWorkspaceBusyRetry(retryReason, contextSnapshot) + ) { + // The issue row is locked above. Recheck after the preflight gate so + // cancellation or recovery cannot leave a successor without its lock. + const [lockedIssue] = await tx.select({ executionRunId: issues.executionRunId }) + .from(issues).where(and(eq(issues.id, issueId), eq(issues.companyId, run.companyId))); + if (lockedIssue?.executionRunId !== run.id) { + return { + outcome: "not_scheduled", issueId, errorCode: "issue_execution_lock_changed", + reason: "Subscription retry suppressed because the task execution lock changed", + details: { issueId, expectedExecutionRunId: run.id, currentExecutionRunId: lockedIssue?.executionRunId ?? null }, + }; + } + } + + const scheduledRunId = randomUUID(); + if (contextSnapshot.explicitUserContinuation) { + const continuation = issueId && retryReason === "transient_failure" ? await admitExplicitContinuationRetry({ + db: tx as unknown as Db, companyId: run.companyId, issueId, agentId: run.agentId, + parentRunId: run.id, successorRunId: scheduledRunId, now, + }) : null; + if (!continuation) return { + outcome: "not_scheduled", issueId, + errorCode: "continuation_user_authorization_missing", + reason: "The automatic retry could not revalidate the original user continuation.", + details: {}, + }; + retryContextSnapshot.explicitUserContinuation = continuation; + retryContextSnapshot.previousRunId = continuation.previousRunId; + } + + await notifyRunDispatch(tx, { companyId: run.companyId }); + const wakeupRequest = await tx + .insert(agentWakeupRequests) + .values({ + companyId: run.companyId, + agentId: run.agentId, + source: "automation", + triggerDetail: "system", + reason: wakeReason, + payload: withRecoveryContext( + { + ...(issueId ? { issueId } : {}), + retryOfRunId: run.id, + ...interactionContinuationPayload, + retryReason, + ...(transientRecovery + ? { errorFamily: transientRecovery.errorFamily } + : {}), + scheduledRetryAttempt: schedule.attempt, + scheduledRetryAt: schedule.dueAt.toISOString(), + ...(transientRetryNotBefore + ? { + transientRetryNotBefore: + transientRetryNotBefore.toISOString(), + } + : {}), + ...(transientRecovery?.errorFamily === "provider_quota" && + transientRetryNotBefore + ? { + providerQuotaRetryNotBefore: + transientRetryNotBefore.toISOString(), + } + : {}), + ...(codexTransientFallbackMode + ? { codexTransientFallbackMode } + : {}), + }, + "normal_model", + ), + status: "queued", + requestedByActorType: "system", + requestedByActorId: null, + idempotencyKey: continuationRetryIdempotencyKey, + updatedAt: now, + }) + .returning() + .then((rows) => rows[0]); + + const scheduledRun = await tx + .insert(heartbeatRuns) + .values({ + id: scheduledRunId, + companyId: run.companyId, + agentId: run.agentId, + scopeKind: run.scopeKind, + issueId, + invocationSource: "automation", + triggerDetail: "system", + status: "scheduled_retry", + wakeupRequestId: wakeupRequest.id, + contextSnapshot: retryContextSnapshot, + ...(hasConversationContinuationPolicy(run.resultJson) + ? { resultJson: { conversationContinuation: CONVERSATION_CONTINUATION_POLICY } } : {}), + responsibleUserId, + sessionIdBefore: sessionBefore, + retryOfRunId: run.id, + scheduledRetryAt: schedule.dueAt, + scheduledRetryAttempt: schedule.attempt, + scheduledRetryReason: retryReason, + continuationAttempt: readContinuationAttempt( + retryContextSnapshot.livenessContinuationAttempt, + ), + updatedAt: now, + }) + .returning() + .then((rows) => rows[0]); + + await tx + .update(agentWakeupRequests) + .set({ + runId: scheduledRun.id, + updatedAt: now, + }) + .where(eq(agentWakeupRequests.id, wakeupRequest.id)); + + let detachWorkspaceFromIssue = false; + if (issueId && shouldQuarantineWorkspaceForRetry) { + const issueWorkspace = await tx + .select({ + id: issues.id, + companyId: issues.companyId, + executionWorkspaceId: issues.executionWorkspaceId, + }) + .from(issues) + .where( + and(eq(issues.id, issueId), eq(issues.companyId, run.companyId)), + ) + .for("update") + .then((rows) => rows[0] ?? null); + const failedExecutionWorkspaceId = + readNonEmptyString( + workspaceValidationRetryPayload?.executionWorkspaceId, + ) ?? readNonEmptyString(issueWorkspace?.executionWorkspaceId); + + if (issueWorkspace && failedExecutionWorkspaceId) { + const failedWorkspace = await tx + .select({ + id: executionWorkspaces.id, + companyId: executionWorkspaces.companyId, + sourceIssueId: executionWorkspaces.sourceIssueId, + status: executionWorkspaces.status, + metadata: executionWorkspaces.metadata, + }) + .from(executionWorkspaces) + .where( + and( + eq(executionWorkspaces.id, failedExecutionWorkspaceId), + eq(executionWorkspaces.companyId, run.companyId), + ), + ) + .for("update") + .then((rows) => rows[0] ?? null); + + const workspaceBelongsToIssue = failedWorkspace + ? failedWorkspace.sourceIssueId === issueId + : false; + + if ( + failedWorkspace && + workspaceBelongsToIssue && + issueWorkspace.executionWorkspaceId === failedExecutionWorkspaceId + ) { + const existingMetadata = parseObject(failedWorkspace.metadata); + const quarantine = { + reason: WORKSPACE_VALIDATION_FAILURE_CODE, + retryReason, + sourceRunId: run.id, + retryRunId: scheduledRun.id, + issueId, + sourceIssueId: failedWorkspace.sourceIssueId ?? null, + quarantinedAt: now.toISOString(), + workspaceValidation: workspaceValidationRetryPayload ?? {}, + }; + await tx + .update(executionWorkspaces) + .set({ + status: "archived", + closedAt: now, + cleanupEligibleAt: null, + cleanupReason: WORKSPACE_VALIDATION_FAILURE_CODE, + metadata: { + ...existingMetadata, + workspaceValidationQuarantine: quarantine, + }, + updatedAt: now, + }) + .where( + and( + eq(executionWorkspaces.id, failedWorkspace.id), + eq(executionWorkspaces.companyId, run.companyId), + ), + ); + + await logActivity(tx as unknown as Db, { + companyId: run.companyId, + actorType: "system", + actorId: "heartbeat", + agentId: run.agentId, + runId: run.id, + action: "execution_workspace.workspace_validation_quarantined", + entityType: "execution_workspace", + entityId: failedWorkspace.id, + details: quarantine, + }); + detachWorkspaceFromIssue = + issueWorkspace.executionWorkspaceId === + failedExecutionWorkspaceId; + } + } + } + + if (issueId) { + await notifyStatusCardIssueChange(tx, run.companyId); + await tx + .update(issues) + .set({ + executionRunId: scheduledRun.id, + checkoutRunId: sql`case when ${issues.checkoutRunId} = ${run.id} then null else ${issues.checkoutRunId} end`, + executionAgentNameKey: normalizeAgentNameKey(agent.name), + executionLockedAt: now, + ...(detachWorkspaceFromIssue + ? { + executionWorkspaceId: null, + executionWorkspacePreference: null, + } + : {}), + updatedAt: now, + }) + .where( + and( + eq(issues.id, issueId), + eq(issues.companyId, run.companyId), + eq(issues.executionRunId, run.id), + ), + ); + } + + return { + outcome: "scheduled", + run: scheduledRun, + reusedExisting: false, + }; + }, + ); + + if (scheduleResult.outcome === "not_scheduled") { + await appendRunEvent(run, { + eventType: "lifecycle", + stream: "system", + level: "warn", + message: scheduleResult.reason, + payload: { + retryReason, + scheduledRetryAttempt: nextAttempt, + maxAttempts, + ...scheduleResult.details, + }, + }); + return { + outcome: "not_scheduled" as const, + reason: scheduleResult.reason, + errorCode: scheduleResult.errorCode, + issueId: scheduleResult.issueId, + }; + } + + const retryRun = scheduleResult.run; + const dueAt = retryRun.scheduledRetryAt + ? new Date(retryRun.scheduledRetryAt) + : schedule.dueAt; + + if (scheduleResult.reusedExisting) { + await appendRunEvent(run, { + eventType: "lifecycle", + stream: "system", + level: "info", + message: `Reused existing continuation retry ${retryRun.scheduledRetryAttempt}/${schedule.maxAttempts}`, + payload: { + retryRunId: retryRun.id, + retryReason, + idempotencyKey: continuationRetryIdempotencyKey, + scheduledRetryAttempt: retryRun.scheduledRetryAttempt, + scheduledRetryAt: dueAt.toISOString(), + }, + }); + + return { + outcome: "scheduled" as const, + run: retryRun, + dueAt, + attempt: retryRun.scheduledRetryAttempt, + maxAttempts: schedule.maxAttempts, + reusedExisting: true, + }; + } + + await appendRunEvent(run, { + eventType: "lifecycle", + stream: "system", + level: "warn", + message: `Scheduled bounded retry ${schedule.attempt}/${schedule.maxAttempts} for ${schedule.dueAt.toISOString()}`, + payload: { + retryRunId: retryRun.id, + retryReason, + ...(transientRecovery + ? { errorFamily: transientRecovery.errorFamily } + : {}), + scheduledRetryAttempt: schedule.attempt, + scheduledRetryAt: schedule.dueAt.toISOString(), + baseDelayMs: schedule.baseDelayMs, + delayMs: schedule.delayMs, + ...(transientRetryNotBefore + ? { transientRetryNotBefore: transientRetryNotBefore.toISOString() } + : {}), + ...(transientRecovery?.errorFamily === "provider_quota" && + transientRetryNotBefore + ? { + providerQuotaRetryNotBefore: + transientRetryNotBefore.toISOString(), + } + : {}), + ...(codexTransientFallbackMode ? { codexTransientFallbackMode } : {}), + }, + }); + + if (retryReason === INTERACTION_CONTINUATION_INFRA_RETRY_REASON) { + await recordPlanApprovalResumeFailureRetry({ + run, + issueId, + retryRunId: retryRun.id, + attempt: schedule.attempt, + maxAttempts: schedule.maxAttempts, + }).catch((error) => { + logger.warn( + { err: error, runId: run.id, issueId, retryRunId: retryRun.id }, + "failed to record plan-approval resume retry failure", + ); + }); + } + + return { + outcome: "scheduled" as const, + run: retryRun, + dueAt, + attempt: schedule.attempt, + maxAttempts: schedule.maxAttempts, + }; + } + + // Finds a running heartbeat run (other than the caller's) whose context + // issue shares the same project workspace, i.e. the run that currently + // "holds" the shared working tree. Runs that have been silent past + // WORKSPACE_BUSY_HOLDER_STALE_AFTER_MS do not count — a zombie holder must + // not park other work forever. This cutoff is independent of informational + // silence warnings. When isolated workspaces are enabled, holders whose + // issue explicitly opted into an isolated workspace never touch the shared + // tree, so they are excluded; a NULL/agent_default mode may resolve to the + // shared tree and counts as a holder (over-serializing is the safe + // direction). When the isolated-workspaces experiment is off, every run + // resolves to the shared tree, so no holder is excluded. + async function findSharedWorkspaceHolder(input: { + companyId: string; + projectWorkspaceId: string; + excludeIssueId: string; + excludeRunId: string; + honorIsolatedWorkspaceModes: boolean; + now?: Date; + }): Promise { + const staleCutoff = new Date( + (input.now ?? new Date()).getTime() - + WORKSPACE_BUSY_HOLDER_STALE_AFTER_MS, + ); + return await db + .select({ + runId: heartbeatRuns.id, + agentId: heartbeatRuns.agentId, + issueId: sql`${issues.id}::text`, + issueIdentifier: issues.identifier, + }) + .from(heartbeatRuns) + .innerJoin( + issues, + and( + eq(issues.companyId, heartbeatRuns.companyId), + sql`${issues.id}::text = ${heartbeatRuns.contextSnapshot} ->> 'issueId'`, + ), + ) + .where( + and( + eq(heartbeatRuns.companyId, input.companyId), + eq(heartbeatRuns.status, "running"), + ne(heartbeatRuns.id, input.excludeRunId), + // Last observed activity: output beats start beats creation. A run + // that started recently but has not written output yet is live. + sql`coalesce(${heartbeatRuns.lastOutputAt}, ${heartbeatRuns.startedAt}, ${heartbeatRuns.createdAt}) >= ${staleCutoff.toISOString()}::timestamptz`, + eq(issues.projectWorkspaceId, input.projectWorkspaceId), + ne(sql`${issues.id}::text`, input.excludeIssueId), + ...(input.honorIsolatedWorkspaceModes + ? [ + or( + // Covers both a NULL settings blob and a blob without a mode + // key; either may still resolve to the shared workspace. + sql`${issues.executionWorkspaceSettings} ->> 'mode' is null`, + notInArray( + sql`${issues.executionWorkspaceSettings} ->> 'mode'`, + [...ISOLATED_EXECUTION_WORKSPACE_MODES], + ), + ), + ] + : []), + ), + ) + .orderBy(asc(heartbeatRuns.createdAt), asc(heartbeatRuns.id)) + .limit(1) + .then((rows) => rows[0] ?? null); + } + + // Finds a running run (other than the caller's) on an issue bound to the + // same reused execution workspace, typically a parent run that is still + // active after delegating a child that inherited its workspace. Unlike + // findSharedWorkspaceHolder, a holder silent past + // WORKSPACE_BUSY_HOLDER_STALE_AFTER_MS is still returned (flagged stale): + // silence does not prove the holder stopped, so the caller must not treat + // the folder as free. A live holder is preferred over a stale one. + async function findExecutionWorkspaceHolder(input: { + companyId: string; + executionWorkspaceId: string; + excludeRunId: string; + now?: Date; + }): Promise { + const staleCutoff = new Date( + (input.now ?? new Date()).getTime() - + WORKSPACE_BUSY_HOLDER_STALE_AFTER_MS, + ); + const lastActivityBefore = sql`coalesce(${heartbeatRuns.lastOutputAt}, ${heartbeatRuns.startedAt}, ${heartbeatRuns.createdAt}) < ${staleCutoff.toISOString()}::timestamptz`; + return await db + .select({ + runId: heartbeatRuns.id, + agentId: heartbeatRuns.agentId, + issueId: sql`${issues.id}::text`, + issueIdentifier: issues.identifier, + stale: lastActivityBefore, + }) + .from(heartbeatRuns) + .innerJoin( + issues, + and( + eq(issues.companyId, heartbeatRuns.companyId), + eq(issues.executionWorkspaceId, input.executionWorkspaceId), + or( + eq(issues.executionRunId, heartbeatRuns.id), + sql`${issues.id}::text = coalesce(${heartbeatRuns.contextSnapshot} ->> 'issueId', ${heartbeatRuns.contextSnapshot} ->> 'taskId')`, + ), + ), + ) + .where( + and( + eq(heartbeatRuns.companyId, input.companyId), + eq(heartbeatRuns.status, "running"), + ne(heartbeatRuns.id, input.excludeRunId), + ), + ) + .orderBy(asc(lastActivityBefore), asc(heartbeatRuns.createdAt), asc(heartbeatRuns.id)) + .limit(1) + .then((rows) => rows[0] ?? null); + } + + // Credential rotation can briefly contend with a fresh runtime read. Keep + // the task on its automatic pre-provider retry path while the lock clears. + async function finalizeAiConnectionBusyDeferral( + run: typeof heartbeatRuns.$inferSelect, + error: HttpError, + wasIssueAssignee: boolean, + ) { + const now = new Date(); + const cancelled = await setRunStatusIfRunning(run.id, "cancelled", { + error: error.message, errorCode: AI_CONNECTION_BUSY_RETRY_REASON, finishedAt: now, + resultJson: { + executionRecovery: { kind: "ai_connection_wait", providerWorkStarted: false }, + cancellation: { + source: "control_plane", + expected: true, + initiator: { type: "system" }, + reason: "Waiting for shared AI credentials", + recordedAt: now.toISOString(), + }, + }, + contextSnapshot: { + ...parseObject(run.contextSnapshot), + aiConnectionBusyDeferredWhileAssignee: wasIssueAssignee, + }, + }); + if (!cancelled.updated) return; + await setWakeupStatus(run.wakeupRequestId, "cancelled", { finishedAt: now, error: error.message }).catch(() => undefined); + const cancelledRun = cancelled.run ?? await getRun(run.id); + const agent = await getAgent(run.agentId); + let scheduled = false; + try { + if (cancelledRun && agent) { + const retry = await scheduleBoundedRetryForRun(cancelledRun, agent, { + now, retryReason: AI_CONNECTION_BUSY_RETRY_REASON, wakeReason: "ai_connection_busy_retry", + maxAttempts: (cancelledRun.scheduledRetryAttempt ?? 0) + 1, + delayMs: computeWorkspaceBusyRetryDelayMs(), + }); + scheduled = retry.outcome === "scheduled"; + await appendRunEvent(cancelledRun, { + eventType: "lifecycle", stream: "system", level: "info", + message: scheduled ? "Waiting for the shared AI subscription. This task will retry automatically." : "The AI subscription is busy; this task can no longer retry automatically.", + payload: { retryScheduled: scheduled }, + }); + } + } finally { + try { + if (cancelledRun && !scheduled) await releaseIssueExecutionAndPromote(cancelledRun); + } finally { + await finalizeAgentStatus(run.agentId, "cancelled", null, { wasFirstHeartbeat: timerClaimWasFirstHeartbeat(run) }); + } + } + } + + // Terminal handling for a WorkspaceBusyDeferral thrown by the pre-dispatch + // gate: cancel the run (contention is not a failure), schedule a + // workspace_busy retry, and leave the agent idle. The issue execution lock + // transfers to the scheduled retry run inside scheduleBoundedRetryForRun, so + // the issue keeps an active execution path and recovery leaves it alone. + // Deferral has no attempt ceiling — the retry keeps rescheduling while a + // live holder exists, and holder staleness (not a counter) is what prevents + // waiting on a zombie. If no retry could be scheduled (agent no longer + // invokable), the lock is released so the issue does not strand on a + // cancelled run. + async function finalizeWorkspaceBusyDeferral( + run: typeof heartbeatRuns.$inferSelect, + deferral: WorkspaceBusyDeferral, + ) { + const now = new Date(); + const cancelWrite = await setRunStatusIfRunning(run.id, "cancelled", { + error: deferral.message, + errorCode: WORKSPACE_BUSY_ERROR_CODE, + finishedAt: now, + resultJson: { + executionRecovery: { + kind: "workspace_wait", + providerWorkStarted: false, + }, + cancellation: { + source: "control_plane", + expected: true, + initiator: { type: "system" }, + reason: `Waiting for the shared ${deferral.executionWorkspaceId ? "execution" : "project"} workspace`, + recordedAt: now.toISOString(), + }, + workspaceBusy: { + projectWorkspaceId: deferral.projectWorkspaceId, + executionWorkspaceId: deferral.executionWorkspaceId, + holderRunId: deferral.holder.runId, + holderIssueId: deferral.holder.issueId, + deferralAttempt: deferral.deferralAttempt, + }, + }, + // Recorded on the run (and inherited by the scheduled retry's context) + // so the retry promotion gate can tell a non-assignee wake — where an + // assignee mismatch is the expected state — from a reassignment race. + contextSnapshot: { + ...parseObject(run.contextSnapshot), + workspaceBusyDeferredWhileAssignee: deferral.wasIssueAssignee, + }, + }); + if (!cancelWrite.updated) { + logger.info( + { runId: run.id, currentStatus: cancelWrite.run?.status ?? null }, + "skipping workspace-busy deferral finalization because the run already left running state", + ); + return; + } + await setWakeupStatus(run.wakeupRequestId, "cancelled", { + finishedAt: now, + error: deferral.message, + }).catch(() => undefined); + + const cancelledRun = + cancelWrite.run ?? (await getRun(run.id).catch(() => null)); + const agentRow = await getAgent(run.agentId).catch(() => null); + let scheduleOutcome: string | null = null; + if (cancelledRun && agentRow) { + const scheduleResult = await scheduleBoundedRetryForRun( + cancelledRun, + agentRow, + { + now, + retryReason: WORKSPACE_BUSY_RETRY_REASON, + wakeReason: WORKSPACE_BUSY_RETRY_WAKE_REASON, + // Always admit the next attempt: workspace-busy deferral is bounded by + // holder liveness, not by an attempt counter. + maxAttempts: (cancelledRun.scheduledRetryAttempt ?? 0) + 1, + delayMs: computeWorkspaceBusyRetryDelayMs(), + }, + ).catch((scheduleErr) => { + logger.error( + { err: scheduleErr, runId: run.id }, + "failed to schedule workspace-busy retry after deferral", + ); + return null; + }); + scheduleOutcome = scheduleResult?.outcome ?? null; + } + + if (cancelledRun) { + await appendRunEvent(cancelledRun, { + eventType: "lifecycle", + stream: "system", + level: "info", + message: + scheduleOutcome === "scheduled" + ? `Deferred: ${deferral.message}. Retry ${deferral.deferralAttempt + 1} scheduled; the run waits for the workspace to free.` + : `Deferred: ${deferral.message}. No retry could be scheduled; releasing the issue for other runs.`, + payload: { + projectWorkspaceId: deferral.projectWorkspaceId, + executionWorkspaceId: deferral.executionWorkspaceId, + holderRunId: deferral.holder.runId, + holderIssueId: deferral.holder.issueId, + deferralAttempt: deferral.deferralAttempt, + retryScheduled: scheduleOutcome === "scheduled", + }, + }).catch(() => undefined); + } + + if (cancelledRun && scheduleOutcome !== "scheduled") { + await releaseIssueExecutionAndPromote(cancelledRun).catch( + (releaseErr) => { + logger.error( + { err: releaseErr, runId: run.id }, + "failed to release issue execution after workspace-busy deferral", + ); + }, + ); + } + + await finalizeAgentStatus(run.agentId, "cancelled", null, { + wasFirstHeartbeat: timerClaimWasFirstHeartbeat(run), + }).catch(() => undefined); + } + + async function scheduleInteractionContinuationInfrastructureRetryIfEligible( + run: typeof heartbeatRuns.$inferSelect, + agent: typeof agents.$inferSelect, + ) { + if (await hasCaseHandoffForRun(run)) return null; + if (!run.wakeupRequestId) return null; + if (!isResolvedInteractionContinuationWakeContext(run.contextSnapshot)) + return null; + if (!isRetryableInteractionContinuationInfrastructureFailure(run)) { + const context = parseObject(run.contextSnapshot); + const issueId = readNonEmptyString(context.issueId); + await escalatePlanApprovalResumeFailureNeedsAttention({ + run, + issueId, + attempt: Math.min( + run.scheduledRetryAttempt ?? + INTERACTION_CONTINUATION_INFRA_MAX_ATTEMPTS, + INTERACTION_CONTINUATION_INFRA_MAX_ATTEMPTS, + ), + maxAttempts: INTERACTION_CONTINUATION_INFRA_MAX_ATTEMPTS, + }).catch((error) => { + logger.warn( + { err: error, runId: run.id, issueId }, + "failed to escalate non-retryable plan-approval resume failure", + ); + }); + return null; + } + + return scheduleBoundedRetryForRun(run, agent, { + retryReason: INTERACTION_CONTINUATION_INFRA_RETRY_REASON, + wakeReason: INTERACTION_CONTINUATION_INFRA_WAKE_REASON, + maxAttempts: INTERACTION_CONTINUATION_INFRA_MAX_ATTEMPTS, + }); + } + + async function nextScheduledRetryAt() { + return runDispatch.nextScheduledRetryAt({ cutoff: await getWorktreeExecutionCutoff() }); + } + + async function promoteDueScheduledRetries(now = new Date()) { + const cutoff = await getWorktreeExecutionCutoff(); + const result = await runDispatch.promoteDueScheduledRetries({ + now, + cutoff, + }); + applyRunDispatchPostCommitEffects(result.postCommitEffects); + return { promoted: result.promoted, runIds: result.runIds }; + } + + async function getIssueRetryRun( + companyId: string, + issueId: string, + statuses: Array<"scheduled_retry" | "queued" | "running" | "cancelled">, + ) { + if (statuses.length === 0) return null; + return db + .select({ + run: heartbeatRuns, + agentName: agents.name, + }) + .from(heartbeatRuns) + .innerJoin(agents, eq(heartbeatRuns.agentId, agents.id)) + .where( + and( + eq(heartbeatRuns.companyId, companyId), + inArray(heartbeatRuns.status, statuses), + sql`${heartbeatRuns.contextSnapshot} ->> 'issueId' = ${issueId}`, + sql`${heartbeatRuns.retryOfRunId} is not null`, + ), + ) + .orderBy( + desc(heartbeatRuns.updatedAt), + desc(heartbeatRuns.createdAt), + desc(heartbeatRuns.id), + ) + .limit(1) + .then((rows) => rows[0] ?? null); + } + + function summarizeIssueScheduledRetryRun(row: { + run: typeof heartbeatRuns.$inferSelect; + agentName: string | null; + }) { + return { + runId: row.run.id, + status: row.run.status as + "scheduled_retry" | "queued" | "running" | "cancelled", + agentId: row.run.agentId, + agentName: row.agentName, + retryOfRunId: row.run.retryOfRunId, + scheduledRetryAt: row.run.scheduledRetryAt, + scheduledRetryAttempt: row.run.scheduledRetryAttempt, + scheduledRetryReason: row.run.scheduledRetryReason, + error: row.run.error, + errorCode: row.run.errorCode, + }; + } + + async function retryScheduledRetryNow(input: { + issueId: string; + actor?: { + actorType?: "user" | "agent" | "system"; + actorId?: string | null; + }; + now?: Date; + }) { + const now = input.now ?? new Date(); + const issue = await db + .select({ id: issues.id, companyId: issues.companyId }) + .from(issues) + .where(eq(issues.id, input.issueId)) + .then((rows) => rows[0] ?? null); + if (!issue) throw notFound("Issue not found"); + + const scheduled = await getIssueRetryRun(issue.companyId, issue.id, [ + "scheduled_retry", + ]); + if (!scheduled) { + const alreadyPromoted = await getIssueRetryRun( + issue.companyId, + issue.id, + ["queued", "running"], + ); + if (alreadyPromoted) { + return { + outcome: "already_promoted" as const, + message: "Scheduled retry was already promoted", + scheduledRetry: summarizeIssueScheduledRetryRun(alreadyPromoted), + }; + } + return { + outcome: "no_scheduled_retry" as const, + message: "No live scheduled retry exists for this issue", + scheduledRetry: null, + }; + } + + const contextSnapshot = { + ...parseObject(scheduled.run.contextSnapshot), + scheduledRetryAt: now.toISOString(), + retryNowRequestedAt: now.toISOString(), + retryNowRequestedByActorType: input.actor?.actorType ?? null, + retryNowRequestedByActorId: input.actor?.actorId ?? null, + }; + + const updated = await db.transaction(async (tx) => { + await notifyRunDispatch(tx, { companyId: scheduled.run.companyId }); + const row = await tx + .update(heartbeatRuns) + .set({ + scheduledRetryAt: now, + contextSnapshot, + updatedAt: now, + }) + .where( + and( + eq(heartbeatRuns.id, scheduled.run.id), + eq(heartbeatRuns.status, "scheduled_retry"), + ), + ) + .returning() + .then((rows) => rows[0] ?? null); + if (!row) return null; + + if (row.wakeupRequestId) { + const wakeupPayload = { + ...parseObject( + await tx + .select({ payload: agentWakeupRequests.payload }) + .from(agentWakeupRequests) + .where(eq(agentWakeupRequests.id, row.wakeupRequestId)) + .then((rows) => rows[0]?.payload ?? null), + ), + scheduledRetryAt: now.toISOString(), + retryNowRequestedAt: now.toISOString(), + }; + await tx + .update(agentWakeupRequests) + .set({ + payload: wakeupPayload, + updatedAt: now, + }) + .where(eq(agentWakeupRequests.id, row.wakeupRequestId)); + } + + return row; + }); + + if (!updated) { + const alreadyPromoted = await getIssueRetryRun( + issue.companyId, + issue.id, + ["queued", "running"], + ); + if (alreadyPromoted) { + return { + outcome: "already_promoted" as const, + message: "Scheduled retry was already promoted", + scheduledRetry: summarizeIssueScheduledRetryRun(alreadyPromoted), + }; + } + return { + outcome: "no_scheduled_retry" as const, + message: "No live scheduled retry exists for this issue", + scheduledRetry: null, + }; + } + + await appendRunEvent(updated, { + eventType: "lifecycle", + stream: "system", + level: "info", + message: "Scheduled retry was requested to run now", + payload: { + issueId: issue.id, + scheduledRetryAttempt: updated.scheduledRetryAttempt, + scheduledRetryAt: updated.scheduledRetryAt + ? new Date(updated.scheduledRetryAt).toISOString() + : null, + scheduledRetryReason: updated.scheduledRetryReason, + requestedByActorType: input.actor?.actorType ?? null, + requestedByActorId: input.actor?.actorId ?? null, + }, + }); + + const promotion = await runDispatch.promoteScheduledRetry({ + runId: updated.id, + companyId: updated.companyId, + now, + }); + if (promotion.outcome === "promoted") { + applyRunDispatchPostCommitEffects(promotion.postCommitEffects); + } + // Promotion can preserve this row as a cleanup wait. Read that exact run, + // not an older cancelled retry or the pre-promotion schedule. + const currentRun = await getRun(updated.id); + const scheduledRetry = currentRun + ? summarizeIssueScheduledRetryRun({ + run: currentRun, + agentName: scheduled.agentName, + }) + : null; + + if (currentRun?.status === "scheduled_retry") { + return { + outcome: "waiting" as const, + message: parseObject(currentRun.resultJson?.executionWait).cause === "execution_owner_active" + ? "Waiting for execution cleanup. Paperclip will retry automatically once cleanup finishes." + : "The retry remains scheduled. Paperclip will check again at the scheduled time.", + scheduledRetry, + }; + } + + if (promotion.outcome === "promoted") { + return { + outcome: "promoted" as const, + message: "Scheduled retry was promoted to the queued run pool", + scheduledRetry, + }; + } + if (promotion.outcome === "gate_suppressed") { + return { + outcome: "gate_suppressed" as const, + message: promotion.reason, + scheduledRetry, + }; + } + if (currentRun && ["queued", "running"].includes(currentRun.status)) { + return { + outcome: "already_promoted" as const, + message: "Scheduled retry was already promoted", + scheduledRetry, + }; + } + return { + outcome: "no_scheduled_retry" as const, + message: "No live scheduled retry exists for this issue", + scheduledRetry: null, + }; + } + + function timerClaimWasFirstHeartbeat( + run: Pick, + ): true | undefined { + return parseObject(run.contextSnapshot).timerClaimWasFirstHeartbeat === true + ? true + : undefined; + } + + return { + scheduleBoundedRetryForRun, + timerClaimWasFirstHeartbeat, + scheduleInteractionContinuationInfrastructureRetryIfEligible, + findSharedWorkspaceHolder, + findExecutionWorkspaceHolder, + finalizeAiConnectionBusyDeferral, + finalizeWorkspaceBusyDeferral, + promoteDueScheduledRetries, + nextScheduledRetryAt, + retryScheduledRetryNow, + + scheduleBoundedRetry: async ( + runId: string, + opts?: { + now?: Date; + random?: () => number; + retryReason?: string; + wakeReason?: string; + maxAttempts?: number; + delayMs?: number; + }, + ) => { + const run = await getRun(runId, { unsafeFullResultJson: true }); + if (!run) return { outcome: "missing_run" as const }; + const agent = await getAgent(run.agentId); + if (!agent) return { outcome: "missing_agent" as const }; + return scheduleBoundedRetryForRun(run, agent, opts); + }, + }; +} diff --git a/server/src/services/instance-settings.ts b/server/src/services/instance-settings.ts index bc795417faa..8a1f850e02c 100644 --- a/server/src/services/instance-settings.ts +++ b/server/src/services/instance-settings.ts @@ -1 +1,671 @@ +import { notifyRunDispatch } from "./run-dispatch-notifications.js"; import { notifyStatusCardEvaluation } from "./status-card-notifications.js"; +import { notifySkillVersionRetentionReconstruction } from "./skill-version-retention-notifications.js"; +import { notifyNativeEnvironmentSettlement } from "./native-environment-settlement-notifications.js"; +import { notifyExternalAgentSettlement } from "../modules/external-agents/index.js"; +import { notifyExternalConversationSettlement } from "./external-conversation-settlement-notifications.js"; +import { notifyNativeRecoveryNotices } from "./native-recovery-notice-notifications.js"; +import { notifyNativeRunProjection } from "./native-run-projection-notifications.js"; +import { notifyNativeResultCommit } from "./native-result-commit-notifications.js"; +import { notifyNativeWorkspaceFinalization } from "./native-workspace-finalization-notifications.js"; +import { notifyCancelledWorkspacePreservation } from "./cancelled-workspace-preservation-notifications.js"; +import { notifyInstructionCollection } from "./instruction-collection-notifications.js"; +import { notifyDirectoryCleanup } from "./directory-cleanup-notifications.js"; +import { notifyInstructionCommit } from "./instruction-commit-notifications.js"; +import { notifyScheduledRetryCancellation } from "./scheduled-retry-cancellation-notifications.js"; +import { notifyVoiceChange } from "./voice/voice-recovery-notifications.js"; +import { notifyToolConnectionHealth } from "./tool-connection-health-notifications.js"; +import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js"; +import type { Db } from "@paperclipai/db"; +import { companies, instanceSettings } from "@paperclipai/db"; + +/** + * A `Db` or an open transaction handle — the subset of query builders the + * settings writes use. Lets `update` run inside a caller's transaction so + * it commits atomically with a sibling write. + */ +type InstanceSettingsTransaction = Parameters[0]>[0]; +export type InstanceSettingsWriteDb = Pick< + Db | InstanceSettingsTransaction, + "select" | "insert" | "update" +>; +import { + DEFAULT_FEEDBACK_DATA_SHARING_PREFERENCE, + DEFAULT_BACKUP_RETENTION, + PAPERCLIP_CLOUD_MANAGED_BY, + instanceGeneralSettingsSchema, + type InstanceGeneralSettings, + instanceExperimentalSettingsSchema, + type InstanceExperimentalSettings, + type InstanceExperimentalSettingsWithManaged, + type ManagedExperimentalFeatureKey, + type ManagedSettingMetadata, + type PatchInstanceGeneralSettings, + type InstanceSettings, + type PatchInstanceSettings, + type PatchInstanceExperimentalSettings, +} from "@paperclipai/shared"; +import { + INSTANCE_FEATURE_CATALOG, + applyOperatorGeneralDefaults, + stripOperatorGeneralEchoes, +} from "@paperclipai/shared"; +import { eq } from "drizzle-orm"; +import { getManagedInstanceConfig, type ManagedInstanceConfig } from "./managed-config.js"; +import { getOperatorSettingDefaults } from "./setting-defaults.js"; +import { isCloudStagingExperimentalSetting, isCloudStagingInstance } from "./cloud-instance.js"; +import { forbidden } from "../errors.js"; + +const DEFAULT_SINGLETON_KEY = "default"; +const instanceGeneralSettingsStorageSchema = instanceGeneralSettingsSchema.strip(); +const instanceExperimentalSettingsStorageSchema = instanceExperimentalSettingsSchema.strip(); +const TRUTHY_RUNTIME_ENV_VALUES = new Set(["1", "true", "yes", "on"]); + +interface InstanceSettingsServiceOptions { + runtimeEnv?: Record; + now?: () => Date; +} + +type WorktreeRunExecutionSuppressedReason = + | "not_worktree_runtime" + | "flag_disabled" + | "missing_cutoff" + | "missing_instance_id" + | "instance_id_mismatch" + | "settings_read_error"; + +export type WorktreeRunExecutionActivationState = + | { + armed: true; + cutoff: string; + activationInstanceId: string; + reason: null; + } + | { + armed: false; + cutoff: null; + activationInstanceId: string | null; + reason: WorktreeRunExecutionSuppressedReason; + }; + +export function isTruthyRuntimeEnvValue(value: string | undefined) { + return typeof value === "string" && TRUTHY_RUNTIME_ENV_VALUES.has(value.trim().toLowerCase()); +} + +function getRuntimeInstanceId(env: Record) { + const instanceId = env.PAPERCLIP_INSTANCE_ID?.trim(); + return instanceId ? instanceId : null; +} + +function stripServerManagedExperimentalPatchFields( + patch: PatchInstanceExperimentalSettings | Record, +): PatchInstanceExperimentalSettings { + const { + worktreeRunExecutionActivatedAt: _ignoredActivatedAt, + worktreeRunExecutionActivationInstanceId: _ignoredActivationInstanceId, + ...patchable + } = patch as Record; + return patchable as PatchInstanceExperimentalSettings; +} + +export function applyExperimentalSettingsPatch( + current: unknown, + patch: PatchInstanceExperimentalSettings | Record, + options: InstanceSettingsServiceOptions = {}, +): InstanceExperimentalSettings { + const previousExperimental = normalizeExperimentalSettings(current); + const patchable = stripServerManagedExperimentalPatchFields(patch); + const nextExperimental = normalizeExperimentalSettings({ + ...previousExperimental, + ...patchable, + }); + const hasWorktreeRunExecutionPatch = Object.prototype.hasOwnProperty.call( + patchable, + "enableWorktreeRunExecution", + ); + + if (!hasWorktreeRunExecutionPatch) { + return nextExperimental; + } + + if (nextExperimental.enableWorktreeRunExecution !== true) { + return { + ...nextExperimental, + worktreeRunExecutionActivatedAt: null, + worktreeRunExecutionActivationInstanceId: null, + }; + } + + if (previousExperimental.enableWorktreeRunExecution === true) { + return nextExperimental; + } + + const runtimeEnv = options.runtimeEnv ?? process.env; + if (!isTruthyRuntimeEnvValue(runtimeEnv.PAPERCLIP_IN_WORKTREE)) { + return nextExperimental; + } + + return { + ...nextExperimental, + worktreeRunExecutionActivatedAt: (options.now ?? (() => new Date()))().toISOString(), + worktreeRunExecutionActivationInstanceId: getRuntimeInstanceId(runtimeEnv), + }; +} + +function suppressWorktreeRunExecution( + reason: WorktreeRunExecutionSuppressedReason, + activationInstanceId: string | null = null, +): WorktreeRunExecutionActivationState { + return { + armed: false, + cutoff: null, + activationInstanceId, + reason, + }; +} + +export function resolveWorktreeRunExecutionActivation( + experimental: InstanceExperimentalSettings, + currentInstanceId: string | null | undefined, +): WorktreeRunExecutionActivationState { + if (experimental.enableWorktreeRunExecution !== true) { + return suppressWorktreeRunExecution( + "flag_disabled", + experimental.worktreeRunExecutionActivationInstanceId, + ); + } + if (!experimental.worktreeRunExecutionActivatedAt) { + return suppressWorktreeRunExecution( + "missing_cutoff", + experimental.worktreeRunExecutionActivationInstanceId, + ); + } + if (!currentInstanceId) { + return suppressWorktreeRunExecution( + "missing_instance_id", + experimental.worktreeRunExecutionActivationInstanceId, + ); + } + if (experimental.worktreeRunExecutionActivationInstanceId !== currentInstanceId) { + return suppressWorktreeRunExecution( + "instance_id_mismatch", + experimental.worktreeRunExecutionActivationInstanceId, + ); + } + return { + armed: true, + cutoff: experimental.worktreeRunExecutionActivatedAt, + activationInstanceId: currentInstanceId, + reason: null, + }; +} + +export async function resolveWorktreeRunExecutionActivationState(options: { + getExperimental: () => Promise; + runtimeEnv?: Record; +}): Promise { + const runtimeEnv = options.runtimeEnv ?? process.env; + if (!isTruthyRuntimeEnvValue(runtimeEnv.PAPERCLIP_IN_WORKTREE)) { + return suppressWorktreeRunExecution("not_worktree_runtime"); + } + try { + return resolveWorktreeRunExecutionActivation( + await options.getExperimental(), + getRuntimeInstanceId(runtimeEnv), + ); + } catch { + return suppressWorktreeRunExecution("settings_read_error"); + } +} + +function normalizeGeneralSettings(raw: unknown): InstanceGeneralSettings { + const parsed = instanceGeneralSettingsStorageSchema.safeParse(raw ?? {}); + if (parsed.success) { + return { + censorUsernameInLogs: parsed.data.censorUsernameInLogs ?? false, + feedbackDataSharingPreference: + parsed.data.feedbackDataSharingPreference ?? DEFAULT_FEEDBACK_DATA_SHARING_PREFERENCE, + backupRetention: parsed.data.backupRetention ?? DEFAULT_BACKUP_RETENTION, + // Absent => unrestricted; only carry through an explicit policy. + ...(parsed.data.executionMode ? { executionMode: parsed.data.executionMode } : {}), + }; + } + return { + censorUsernameInLogs: false, + feedbackDataSharingPreference: DEFAULT_FEEDBACK_DATA_SHARING_PREFERENCE, + backupRetention: DEFAULT_BACKUP_RETENTION, + }; +} + +export function normalizeExperimentalSettings(raw: unknown): InstanceExperimentalSettings { + const parsed = instanceExperimentalSettingsStorageSchema.safeParse(raw ?? {}); + if (parsed.success) { + return { + enableEnvironments: parsed.data.enableEnvironments ?? false, + enableNativeRunner: parsed.data.enableNativeRunner ?? true, + enableAiConnectionRouters: parsed.data.enableAiConnectionRouters ?? false, + enableManagedSandboxOnly: parsed.data.enableManagedSandboxOnly ?? false, + enableIsolatedWorkspaces: parsed.data.enableIsolatedWorkspaces ?? false, + enableIsolatedWorkspacesByDefault: parsed.data.enableIsolatedWorkspacesByDefault ?? false, + enableStreamlinedLeftNavigation: parsed.data.enableStreamlinedLeftNavigation ?? true, + enableStreamlinedUi: parsed.data.enableStreamlinedUi ?? true, + // Apps graduated from Experimental. Ignore historical off values while + // continuing to accept the compatibility key in stored settings. + enableApps: true, + enableMcpAggregators: true, + enablePublicMcp: parsed.data.enablePublicMcp ?? false, + enableOpenAiDot: parsed.data.enableOpenAiDot ?? false, + enableMuse: parsed.data.enableMuse ?? false, + enableGrokBot: parsed.data.enableGrokBot ?? false, + enableChatConnectors: parsed.data.enableChatConnectors ?? false, + enableGitHubReviewBots: parsed.data.enableGitHubReviewBots ?? false, + enableMemoryConnectors: parsed.data.enableMemoryConnectors ?? false, + enablePaperclipNotebook: parsed.data.enablePaperclipNotebook ?? false, + enableCases: parsed.data.enableCases ?? false, + enableAgentChat: parsed.data.enableAgentChat ?? false, + enableCombinedInboxTasks: parsed.data.enableCombinedInboxTasks ?? false, + enableArtifactsV1: parsed.data.enableArtifactsV1 ?? false, + enableConferenceRoomChat: parsed.data.enableConferenceRoomChat ?? false, + enableClassicTaskInterface: parsed.data.enableClassicTaskInterface ?? false, + enableIssuePlanDecompositions: parsed.data.enableIssuePlanDecompositions ?? false, + enableExperimentalFileViewer: parsed.data.enableExperimentalFileViewer ?? false, + enableExternalObjects: parsed.data.enableExternalObjects ?? false, + enableSmokeLab: parsed.data.enableSmokeLab ?? false, + enableBuiltInAgents: parsed.data.enableBuiltInAgents ?? false, + enableBetaSkills: parsed.data.enableBetaSkills ?? false, + enableSummaries: parsed.data.enableSummaries ?? false, + enableStatusCards: parsed.data.enableStatusCards ?? false, + enableDecisions: parsed.data.enableDecisions ?? false, + enableFastResponses: parsed.data.enableFastResponses ?? false, + enableGoalsSidebarLink: parsed.data.enableGoalsSidebarLink ?? false, + enableServerInfoDebugView: parsed.data.enableServerInfoDebugView ?? false, + enablePaperclipDeveloperMode: parsed.data.enablePaperclipDeveloperMode ?? false, + enableSimplifiedEnglishInteractions: parsed.data.enableSimplifiedEnglishInteractions ?? false, + enableFirstTaskPlanProposal: parsed.data.enableFirstTaskPlanProposal ?? false, + autoRestartDevServerWhenIdle: parsed.data.autoRestartDevServerWhenIdle ?? false, + enableWorkspaceBranchReconcileForward: parsed.data.enableWorkspaceBranchReconcileForward ?? true, + enableWorkspaceDirtyQuarantineRepair: parsed.data.enableWorkspaceDirtyQuarantineRepair ?? true, + enableOwnerInstanceAdmin: parsed.data.enableOwnerInstanceAdmin ?? false, + enableSandboxDuplexBridge: parsed.data.enableSandboxDuplexBridge ?? false, + enableRunnerPreviewIngress: parsed.data.enableRunnerPreviewIngress ?? false, + enableWorktreeRunExecution: parsed.data.enableWorktreeRunExecution ?? false, + worktreeRunExecutionActivatedAt: parsed.data.worktreeRunExecutionActivatedAt ?? null, + worktreeRunExecutionActivationInstanceId: + parsed.data.worktreeRunExecutionActivationInstanceId ?? null, + }; + } + return { + enableEnvironments: false, + enableNativeRunner: true, + enableAiConnectionRouters: false, + enableManagedSandboxOnly: false, + enableIsolatedWorkspaces: false, + enableIsolatedWorkspacesByDefault: false, + enableStreamlinedLeftNavigation: true, + enableStreamlinedUi: true, + enableApps: true, + enableMcpAggregators: true, + enablePublicMcp: false, + enableOpenAiDot: false, + enableMuse: false, + enableGrokBot: false, + enableChatConnectors: false, + enableGitHubReviewBots: false, + enableMemoryConnectors: false, + enablePaperclipNotebook: false, + enableCases: false, + enableAgentChat: false, + enableCombinedInboxTasks: false, + enableArtifactsV1: false, + enableConferenceRoomChat: false, + enableClassicTaskInterface: false, + enableIssuePlanDecompositions: false, + enableExperimentalFileViewer: false, + enableExternalObjects: false, + enableSmokeLab: false, + enableBuiltInAgents: false, + enableBetaSkills: false, + enableSummaries: false, + enableStatusCards: false, + enableDecisions: false, + enableFastResponses: false, + enableGoalsSidebarLink: false, + enableServerInfoDebugView: false, + enablePaperclipDeveloperMode: false, + enableSimplifiedEnglishInteractions: false, + enableFirstTaskPlanProposal: false, + autoRestartDevServerWhenIdle: false, + enableWorkspaceBranchReconcileForward: true, + enableWorkspaceDirtyQuarantineRepair: true, + enableOwnerInstanceAdmin: false, + enableSandboxDuplexBridge: false, + enableRunnerPreviewIngress: false, + enableWorktreeRunExecution: false, + worktreeRunExecutionActivatedAt: null, + worktreeRunExecutionActivationInstanceId: null, + }; +} + +export type ManagedExperimentalKeyMetadata = Partial< + Record +>; + +/** + * Overlay the cloud managed-config feature values over normalized settings. + * + * Read-time precedence: code floor (cloud) > managed overlay > tenant DB + * value > schema default. (No code floors are expressed as flags today — + * floors are enforced in code at the guarded routes, independent of any + * flag value.) The overlay is deliberately never persisted: it re-asserts on + * every read, so a DB restore or manual row edit cannot resurrect a + * capability the harness has disabled. + */ +export function applyManagedExperimentalOverlay( + experimental: InstanceExperimentalSettings, + managedConfig: ManagedInstanceConfig | null, +): { experimental: InstanceExperimentalSettings; managedKeys: ManagedExperimentalKeyMetadata } { + if (!managedConfig) return { experimental, managedKeys: {} }; + const next: InstanceExperimentalSettings = { ...experimental }; + const managedKeys: ManagedExperimentalKeyMetadata = {}; + for (const [key, value] of Object.entries(managedConfig.features) as Array< + [ManagedExperimentalFeatureKey, boolean] + >) { + // Existing Cloud stack configs may still carry retired flags. Accept the + // document during rollout, but never let retired flags disable Apps or MCP aggregators. + if (key === "enableApps" || key === "enableMcpAggregators") continue; + next[key] = value; + managedKeys[key] = { managed: true, managedBy: PAPERCLIP_CLOUD_MANAGED_BY }; + } + return { experimental: next, managedKeys }; +} + +/** + * Keep self-hosted-only defaults out of Cloud. + * + * The experimental schema carries one default per flag, and the feature + * catalog pins it to `selfHostedDefault`. A flag that is on by default for + * self-hosted but off by default for Cloud (`selfHostedDefault: true`, + * `cloudDefault: false`) would therefore normalize to "on" for a managed + * instance whose tenant row and managed overlay both leave it unset. Re-assert + * the declared Cloud default for exactly those flags. An explicit tenant value + * or a managed feature value still wins (the overlay is applied afterwards). + */ +export function applyCloudCatalogDefaults( + experimental: InstanceExperimentalSettings, + rawStored: unknown, + managedConfig: ManagedInstanceConfig | null, +): InstanceExperimentalSettings { + if (!managedConfig) return experimental; + const stored = + rawStored && typeof rawStored === "object" && !Array.isArray(rawStored) + ? (rawStored as Record) + : {}; + const next: InstanceExperimentalSettings = { ...experimental }; + for (const [key, entry] of Object.entries(INSTANCE_FEATURE_CATALOG)) { + if (entry.cloudDefault !== false || entry.selfHostedDefault !== true) continue; + if (typeof stored[key] === "boolean") continue; + if (typeof managedConfig.features[key as ManagedExperimentalFeatureKey] === "boolean") continue; + (next as unknown as Record)[key] = false; + } + return next; +} + +/** + * Keep the write path from freezing a self-hosted default into a Cloud row. + * + * `updateExperimental` persists the whole normalized object, and the schema + * normalizes an omitted flag to its self-hosted default. Without this step an + * unrelated experimental write (say, turning on Cases) would store + * `enableNativeRunner: true` on a managed instance whose tenant row had never + * mentioned the flag; every later read would then treat the stored boolean as + * an explicit tenant choice and stop re-asserting the Cloud default. + * + * For each guarded flag (see `applyCloudCatalogDefaults`), the stored key is + * left absent unless the tenant already stored a boolean or this patch sets + * the flag to something other than the Cloud default. A patch value equal to + * the Cloud default is a full-GET echo of the read-time overlay, not a + * choice, and is stripped the same way `stripOperatorGeneralEchoes` treats + * operator defaults. Self-hosted rows are returned untouched. + */ +export function stripCloudCatalogDefaultEchoes( + rawStored: unknown, + patch: PatchInstanceExperimentalSettings | Record, + next: InstanceExperimentalSettings, + managedConfig: ManagedInstanceConfig | null, +): Partial { + if (!managedConfig) return next; + const stored = + rawStored && typeof rawStored === "object" && !Array.isArray(rawStored) + ? (rawStored as Record) + : {}; + const patchRecord = patch as Record; + const result: Record = { ...next }; + for (const [key, entry] of Object.entries(INSTANCE_FEATURE_CATALOG)) { + if (entry.cloudDefault !== false || entry.selfHostedDefault !== true) continue; + if (typeof stored[key] === "boolean") continue; + if ( + Object.prototype.hasOwnProperty.call(patchRecord, key) && + typeof patchRecord[key] === "boolean" && + patchRecord[key] !== entry.cloudDefault + ) { + continue; + } + delete result[key]; + } + return result as Partial; +} + +export function instanceSettingsService(db: Db, options: InstanceSettingsServiceOptions = {}) { + // Fail closed: a malformed PAPERCLIP_MANAGED_CONFIG throws here (and at + // boot in index.ts) rather than silently running without the overlay. + const managedConfig = getManagedInstanceConfig(options.runtimeEnv ?? process.env); + const staging = isCloudStagingInstance(options.runtimeEnv ?? process.env); + const enforcedManagedConfig = staging && managedConfig + ? { + ...managedConfig, + features: Object.fromEntries(Object.entries(managedConfig.features) + .filter(([key, enabled]) => enabled || !isCloudStagingExperimentalSetting(key))), + } + : managedConfig; + // Same posture for PAPERCLIP_SETTING_DEFAULTS: parsed once, applied per + // read, never persisted (see applyOperatorGeneralDefaults) — including on + // the write path, where a full-GET echo of the overlaid value is stripped + // back to the schema default (see stripOperatorGeneralEchoes). + const operatorDefaults = getOperatorSettingDefaults(options.runtimeEnv ?? process.env); + + function toGeneralView(raw: unknown): InstanceGeneralSettings { + return applyOperatorGeneralDefaults(normalizeGeneralSettings(raw), operatorDefaults); + } + + function toExperimentalView(raw: unknown): InstanceExperimentalSettingsWithManaged { + const stored = raw && typeof raw === "object" && !Array.isArray(raw) ? raw : {}; + const defaults = staging ? { ...managedConfig?.features, ...stored } : raw; + const { experimental, managedKeys } = applyManagedExperimentalOverlay( + applyCloudCatalogDefaults(normalizeExperimentalSettings(defaults), defaults, managedConfig), + enforcedManagedConfig, + ); + // Self-hosted responses stay byte-identical: no managedKeys field at all. + return managedConfig ? { ...experimental, managedKeys } : experimental; + } + + function toInstanceSettings(row: typeof instanceSettings.$inferSelect): InstanceSettings { + return { + id: row.id, + defaultEnvironmentId: row.defaultEnvironmentId ?? null, + general: toGeneralView(row.general), + experimental: toExperimentalView(row.experimental), + createdAt: row.createdAt, + updatedAt: row.updatedAt, + } as InstanceSettings; + } + async function getOrCreateRow(runner: InstanceSettingsWriteDb = db) { + const existing = await runner + .select() + .from(instanceSettings) + .where(eq(instanceSettings.singletonKey, DEFAULT_SINGLETON_KEY)) + .then((rows) => rows[0] ?? null); + if (existing) return existing; + + const now = new Date(); + const [created] = await runner + .insert(instanceSettings) + .values({ + singletonKey: DEFAULT_SINGLETON_KEY, + general: {}, + experimental: {}, + createdAt: now, + updatedAt: now, + }) + .onConflictDoUpdate({ + target: [instanceSettings.singletonKey], + set: { + updatedAt: now, + }, + }) + .returning(); + + if (created) return created; + + const raced = await runner + .select() + .from(instanceSettings) + .where(eq(instanceSettings.singletonKey, DEFAULT_SINGLETON_KEY)) + .then((rows) => rows[0] ?? null); + if (raced) return raced; + + throw new Error("Failed to initialize instance settings row"); + } + + return { + get: async (): Promise => toInstanceSettings(await getOrCreateRow()), + + update: async ( + patch: PatchInstanceSettings, + writeOptions?: { db?: InstanceSettingsWriteDb }, + ): Promise => { + // The write may run inside a caller-supplied transaction so it commits + // atomically with a sibling write (e.g. clearing the managed-default + // stamp on the environment row alongside a defaultEnvironmentId + // change). Reads use the same runner so the row is visible to the tx. + const runner = writeOptions?.db ?? db; + const current = await getOrCreateRow(runner); + const now = new Date(); + const [updated] = await runner + .update(instanceSettings) + .set({ + ...(Object.prototype.hasOwnProperty.call(patch, "defaultEnvironmentId") + ? { defaultEnvironmentId: patch.defaultEnvironmentId ?? null } + : {}), + updatedAt: now, + }) + .where(eq(instanceSettings.id, current.id)) + .returning(); + return toInstanceSettings(updated ?? current); + }, + + getGeneral: async ( + readOptions?: { db?: InstanceSettingsWriteDb }, + ): Promise => { + const row = await getOrCreateRow(readOptions?.db); + return toGeneralView(row.general); + }, + + getExperimental: async (): Promise => { + const row = await getOrCreateRow(); + return toExperimentalView(row.experimental); + }, + + updateGeneral: async (patch: PatchInstanceGeneralSettings): Promise => { + const current = await getOrCreateRow(); + const storedGeneral = normalizeGeneralSettings(current.general); + // A full-GET echo carries the overlaid operator value for a field the + // user never chose; stripping it keeps the overlay strictly read-time, + // so changing or unsetting the variable later still takes effect. + const nextGeneral = stripOperatorGeneralEchoes( + storedGeneral, + normalizeGeneralSettings({ ...storedGeneral, ...patch }), + operatorDefaults, + ); + const now = new Date(); + const [updated] = await db + .update(instanceSettings) + .set({ + general: { ...nextGeneral }, + updatedAt: now, + }) + .where(eq(instanceSettings.id, current.id)) + .returning(); + return toInstanceSettings(updated ?? current); + }, + + updateExperimental: async (patch: PatchInstanceExperimentalSettings): Promise => { + for (const [key, enabled] of Object.entries(managedConfig?.features ?? {})) { + if (enabled && patch[key as ManagedExperimentalFeatureKey] === false) { + throw forbidden(`${key} is enabled by Paperclip Cloud and cannot be disabled on this instance`, { + code: "settings_operator_managed", + }); + } + } + const current = await getOrCreateRow(); + // Guarded Cloud flags stay absent from the row unless chosen, so the + // read-time catalog default keeps applying (see stripCloudCatalogDefaultEchoes). + const nextExperimental = staging + ? applyExperimentalSettingsPatch(toExperimentalView(current.experimental), patch, options) + : stripCloudCatalogDefaultEchoes( + current.experimental, + patch, + applyExperimentalSettingsPatch(current.experimental, patch, options), + managedConfig, + ); + if (staging) { + const stored = current.experimental as Record | null; + for (const key of Object.keys(INSTANCE_FEATURE_CATALOG) as ManagedExperimentalFeatureKey[]) { + if (typeof stored?.[key] !== "boolean" && !Object.prototype.hasOwnProperty.call(patch, key)) { + delete nextExperimental[key]; + } + } + } + const now = new Date(); + const [updated] = await db.transaction(async tx => { + if (patch.enableStatusCards !== undefined) await notifyStatusCardEvaluation(tx); + await notifyDeliveryWork(tx, DELIVERY_QUEUES.mcpEvents, "*"); + await notifyDeliveryWork(tx, DELIVERY_QUEUES.dotEvents, "*"); + await notifyDeliveryWork(tx, DELIVERY_QUEUES.grokDispatch); + if (patch.enableWorktreeRunExecution !== undefined) await notifyScheduledRetryCancellation(tx); + if (patch.enableWorktreeRunExecution !== undefined) { + await notifyRunDispatch(tx); + await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers); + await notifyExternalAgentSettlement(tx); + await notifyInstructionCommit(tx); + await notifyDirectoryCleanup(tx); + await notifySkillVersionRetentionReconstruction(tx); + await notifyInstructionCollection(tx); + await notifyCancelledWorkspacePreservation(tx); + await notifyExternalConversationSettlement(tx); + await notifyNativeRecoveryNotices(tx); + await notifyNativeRunProjection(tx); + await notifyNativeEnvironmentSettlement(tx); + await notifyNativeResultCommit(tx); + await notifyNativeWorkspaceFinalization(tx); + } + if (patch.enableChatConnectors !== undefined) await notifyVoiceChange(tx); + await notifyToolConnectionHealth(tx, "*"); + return tx + .update(instanceSettings) + .set({ + experimental: { ...nextExperimental }, + updatedAt: now, + }) + .where(eq(instanceSettings.id, current.id)) + .returning(); + }); + return toInstanceSettings(updated ?? current); + }, + + listCompanyIds: async (): Promise => + db + .select({ id: companies.id }) + .from(companies) + .then((rows) => rows.map((row) => row.id)), + }; +} diff --git a/server/src/services/issue-review-state-notifications.ts b/server/src/services/issue-review-state-notifications.ts index 82d51d88c3b..b0f98f61b2a 100644 --- a/server/src/services/issue-review-state-notifications.ts +++ b/server/src/services/issue-review-state-notifications.ts @@ -1 +1,15 @@ +import { notifyRunDispatch } from "./run-dispatch-notifications.js"; +import { notifyStatusCardIssueChange } from "./status-card-notifications.js"; +import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js"; +import { notifyExternalConversationSettlement } from "./external-conversation-settlement-notifications.js"; +import { notifyNativeReviewResponse } from "./native-review-response-notifications.js"; +import { notifyNativeStatusReassessment } from "./native-status-reassessment-notifications.js"; + +export async function notifyIssueReviewState(tx: object, scope: { companyId: string; issueId?: string; runId?: string; agentId?: string }) { + await notifyRunDispatch(tx, scope); await notifyStatusCardIssueChange(tx, scope.companyId); + await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors); + await notifyExternalConversationSettlement(tx, scope); + await notifyNativeReviewResponse(tx, scope); + await notifyNativeStatusReassessment(tx, scope); +} diff --git a/server/src/services/issues.ts b/server/src/services/issues.ts index 0eff1c83613..faddf33dc93 100644 --- a/server/src/services/issues.ts +++ b/server/src/services/issues.ts @@ -1 +1,14120 @@ +import { notifyRunDispatch } from "./run-dispatch-notifications.js"; import { notifyStatusCardIssueChange } from "./status-card-notifications.js"; +import { notifyExternalConversationSettlement } from "./external-conversation-settlement-notifications.js"; +import { notifyNativeRecoveryNotices } from "./native-recovery-notice-notifications.js"; +import { notifyNativeRunProjection } from "./native-run-projection-notifications.js"; +import { notifyNativeResultCommit } from "./native-result-commit-notifications.js"; +import { notifyNativeWorkspaceFinalization } from "./native-workspace-finalization-notifications.js"; +import { notifyNativeStatusReassessment } from "./native-status-reassessment-notifications.js"; +import { notifyUnsafeWorkspaceExportRecovery } from "./unsafe-workspace-export-notifications.js"; +import { notifyChatResponseAvailability } from "./chat-response-availability.js"; +import { notifyChatInboundWakeNotices } from "./chat-inbound-wake-notice-notifications.js"; +import { notifyChatRunMilestones } from "./chat-run-milestone-notifications.js"; +import { notifyApprovalWork } from "./approval-work-notifications.js"; +import { notifyDecisionTargetWork, notifyDecisionTargetIssueRemoval } from "./decision-targets.js"; +import { assertOnboardingFirstTaskCreationAllowed, assertOnboardingFirstTaskDependenciesAllowed } from "./onboarding-first-task-admission.js"; +import { resolveConnectionIntentContinuationSource, resolveItemVerdictContinuationWake } from "./chat-interaction-continuation.js"; +import { insertIssueComment } from "./issue-comment-writes.js"; +import { measureRunTransaction } from "./execution-transaction-timing.js"; +import { readNativeMonitorSource } from "./native-runtime/native-monitor-delivery.js"; +import { readNativeContinuationSource } from "./native-runtime/native-continuation-delivery.js"; +import { assertNoCaseHandoffMutation, cancelCaseHandoffsForIssue, completeTerminalCaseHandoffsForIssue } from "./heartbeat/case-handoff-lifecycle.js"; +import { measureRunPhase } from "./execution-preparation-timing.js"; +import { automaticReadRecoveryCleanupCondition, directHumanCommentCondition, humanCreatedIssueCondition, pendingUserRequestCondition, pendingUserRequestRowCondition, touchedByUserCondition, userParticipationActivityCondition } from "./issue-participation.js"; +import { resolveProjectDefaultWorkspaceId } from "./project-default-workspace.js"; +import { type AuthorizationActor, executionWorkspaceReadSqlCondition, projectReadSqlCondition } from "./authorization.js"; +import { assertExecutionWorkspaceRecordAccess, assertTaskWorkspaceSourceProjectAccess } from "./task-workspace-access.js"; +import { type TaskWorkspaceSelection, type TaskWorkspaceIntent } from "@paperclipai/shared"; +import { executionWorkspaceService } from "./execution-workspaces.js"; +import { notifyChatPublicationWork } from "./chat-work-notifications.js"; +import { notifyTerminalWorkspaceWork } from "./terminal-workspace-work.js"; +import { requestRestorePointDeletion } from "./git-restore-point-ledger.js"; +import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js"; +import { recordChatHandoff, recordAcceptedChatHandoff, lockAcceptedChatHandoffSource, recordChatCompletion, existingChatCompletionReply, acknowledgeChatCompletionReply } from "./chat-completion-delivery.js"; +import { mirrorSlackBoardComment, slackBoardReplyBindings } from "./slack-board-messages.js"; +import { assertAgentRunWriteAllowed } from "../agent-run-cancellation.js"; +import { assertLegacyRunTaskOwnership } from "../legacy-run-task-ownership.js"; +import { retryIdempotentDatabaseOperation } from "../database-retry.js"; +import { externalConversationStateSql, nonIdleSlackIssueCondition, resumeSlackConversation } from "./slack-conversation-state.js"; +import { documentService } from "./documents.js"; +import { parseTaskSearch, taskSearchCtes, taskSearchScore } from "./task-search.js"; +import { createdFromIssueCondition } from "./issue-creation-origin.js"; +import { executionProjectionsForRuns } from "./execution-projection.js"; +import type { ExecutionProjection } from "@paperclipai/shared"; +import { Buffer } from "node:buffer"; +import { isDeepStrictEqual } from "node:util"; +import { createHash, randomUUID } from "node:crypto"; +import { isExplicitContinuationRetryClaim } from "./explicit-continuation-retry-claim.js"; +import { markdownToPlainText, parseMarkdown } from "chat"; +import { listExternalReviewWaits } from "./native-runtime/external-review-wait.js"; +import { + and, + asc, + desc, + eq, + gt, + getTableColumns, + gte, + inArray, + isNotNull, + isNull, + like, + lt, + lte, + ne, + notExists, + notInArray, + or, + sql, + type SQL, +} from "drizzle-orm"; +import { unionAll } from "drizzle-orm/pg-core"; +import type { Db } from "@paperclipai/db"; +import { + activityLog, + chatActions, + chatConversations, + chatDeliveries, + chatEndpoints, + chatMessageLinks, + chatPublications, + agentWakeupRequests, + agents, + authUsers, + approvals, + assets, + companies, + companyMemberships, + documentRevisions, + documents, + goals, + heartbeatRuns, + routineRuns, + executionWorkspaces, + issueApprovals, + issueAccessGrants, + issueAttachments, + issueCreateIdempotencyKeys, + issueInboxArchives, + issueLabels, + issueWatchdogs, + issuePlanDecompositions, + issueRecoveryActions, + issueRelations, + issueComments, + issueDocuments, + issueQuestionResponseDeliveries, + issueWorkProducts, + issueReadStates, + issueThreadInteractions, + toolActionRequests, + toolActionDeliveries, + toolInvocations, + issues, + labels, + projectWorkspaces, + projects, + toolConnections, + workspaceOperations, +} from "@paperclipai/db"; +import type { + AcceptedPlanDecomposition, + IssueComment, + IssueCommentAuthorType, + IssueCommentDerivedAuthorSource, + IssueCommentMetadata, + IssueCommentPresentation, + IssueBlockerAttention, + IssueReviewAttention, + IssueReviewAttentionPath, + IssueBlockedInboxAttention, + IssueBlockedInboxIssueRef, + IssueRelationIssueSummary, + IssueWatchdogSummary, + LowTrustBoundary, + SuccessfulRunHandoffState, +} from "@paperclipai/shared"; +import { + clampIssueRequestDepth, + extractAgentMentionIds, + extractProjectMentionIds, + issueCommentAuthorTypeSchema, + issueCommentMetadataSchema, + issueCommentPresentationSchema, + isAgentCreatedIssue, + isUuidLike, + normalizeIssueIdentifier as normalizeIssueReferenceIdentifier, +} from "@paperclipai/shared"; +import { conflict, HttpError, notFound, unprocessable } from "../errors.js"; +import { isForeignKeyViolation } from "../db-errors.js"; +import { logger } from "../middleware/logger.js"; +import { parseObject } from "../adapters/utils.js"; +import { + hydrateSuccessfulRunHandoffLiveness, + SUCCESSFUL_RUN_HANDOFF_LIVE_WAKE_STATUSES, +} from "./successful-run-handoff-state.js"; +import { + defaultIssueExecutionWorkspaceSettingsForProject, + gateProjectExecutionWorkspacePolicy, + issueExecutionWorkspaceModeForPersistedWorkspace, + isUnrunnableWorktreeCombo, + parseIssueExecutionWorkspaceSettings, + parseProjectExecutionWorkspacePolicy, + resolvePinnedIssueWorkspaceStrategyType, + WORKSPACE_WORKTREE_REQUIRES_PROJECT_CODE, + WORKSPACE_WORKTREE_REQUIRES_PROJECT_MESSAGE, + WORKSPACE_WORKTREE_REQUIRES_PROJECT_REMEDIATION, + type ParsedExecutionWorkspaceMode, +} from "./execution-workspace-policy.js"; +import { mergeExecutionWorkspaceConfig } from "./execution-workspaces.js"; +import { hasChatRunOwnedProviderInteraction } from "./chat-interaction-arbitration.js"; +import { readChatControlChronology } from "./chat-control-chronology.js"; +import { authorizeChatConversationForBoundRun } from "./native-runtime/chat-attachment-reuse.js"; +import { + assertDurableChatWakeupReceipt, + createDurableChatWakeupRequest, +} from "./durable-chat-wakeup.js"; +import { + authorizeNativeChatReviewPresentation, + retryNativeChatReviewPresentation, +} from "./native-runtime/native-chat-review-presentation.js"; +import { + applyIssueExecutionPolicyTransition, + buildInitialIssueMonitorFields, + normalizeIssueExecutionPolicy, +} from "./issue-execution-policy.js"; +import { assertQualityReviewPolicyMutation, assertQualityReviewPolicyReuse, assertRequiredQualityReviewComplete, requiredQualityReview } from "./required-quality-review.js"; +import { instanceSettingsService } from "./instance-settings.js"; +import { redactCurrentUserText } from "../log-redaction.js"; +import { redactSensitiveText } from "../redaction.js"; +import { + resolveIssueGoalId, + resolveNextIssueGoalId, +} from "./issue-goal-fallback.js"; +import { getRunLogStore, isRunLogStoreType } from "./run-log-store.js"; +import { recoverSettledNativeRunLog } from "./native-runtime/native-run-log-recovery.js"; +import { getDefaultCompanyGoal } from "./goals.js"; +import { assertAssignableAgent } from "./agent-assignability.js"; +import { + CHAT_RUN_PRESENTATION_AUTHORIZATION_REASON, + isExternalChatPresentationContext, + LEGACY_WITHHELD_RUN_COMMENT, + projectHistoricalHeartbeatRunComment, +} from "./heartbeat-run-summary.js"; +import { + DEFAULT_INSERT_CHUNK_ROWS, + insertRowsInChunks, +} from "./batch-insert.js"; +import type { + ImportIssueRow, + ImportIssueCommentRow, + ImportIssueAttachmentRow, +} from "./import-write-types.js"; +import { + summarizeIssueWatchdog, + upsertIssueWatchdogForIssue, +} from "./task-watchdogs.js"; +import { + isVerifiedIssueTreeControlInteractionWake, + issueTreeControlService, + type ActiveIssueTreePauseHoldGate, +} from "./issue-tree-control.js"; +import { + parseIssueGraphLivenessIncidentKey, + RECOVERY_ORIGIN_KINDS, +} from "./recovery/origins.js"; +import { + classifyIssueGraphLiveness, + classifyIssueReviewPaths, + type IssueGraphLivenessInput, + type IssueLivenessFinding, +} from "./recovery/issue-graph-liveness.js"; +import { visibleIssueCondition } from "./issue-visibility.js"; +import { finalizeStatusCardsForStalledGeneration } from "./status-card-finalization.js"; +import { finalizeSummarySlotsForTerminalIssue } from "./summary-slot-finalization.js"; +import { + logActivity, + persistActivity, + publishActivity, + type ActivityPublication, +} from "./activity-log.js"; +import { buildIssueChanges } from "./issue-change-receipt.js"; +import { ensurePersonalPrivateProject } from "./projects.js"; +import { projectSafeChatPublication } from "./chat-publication-projection.js"; +import { issueThreadInteractionAttentionAgentAllowed } from "./issue-thread-interaction-resolution.js"; + +const ALL_ISSUE_STATUSES = [ + "backlog", + "todo", + "in_progress", + "in_review", + "blocked", + "done", + "cancelled", +]; +const MAX_ISSUE_COMMENT_PAGE_LIMIT = 500; +const MAX_CHAT_PRESENTATION_ATTACHMENTS = 20; +export const ISSUE_LIST_DEFAULT_LIMIT = 500; +export const ISSUE_LIST_MAX_LIMIT = 1000; +export const ISSUE_BLOCKER_DIAGNOSTICS_MAX_BLOCKERS = 100; +export const ISSUE_WAKE_DIAGNOSTICS_MAX_WAKE_REQUESTS = 50; +export const ISSUE_WAKE_DIAGNOSTICS_MAX_ACTIVITY_RECORDS = 50; +export const ISSUE_WAKE_DIAGNOSTICS_LOOKBACK_DAYS = 14; +export const ISSUE_SUBTREE_DIAGNOSTICS_MAX_DEPTH = 8; +export const ISSUE_SUBTREE_DIAGNOSTICS_MAX_NODES = 100; +export const ISSUE_SUBTREE_DIAGNOSTICS_MAX_BLOCKERS_PER_NODE = 20; +export const ISSUE_SUBTREE_DIAGNOSTICS_MAX_WAKE_REQUESTS_PER_NODE = 5; +export const ISSUE_SUBTREE_DIAGNOSTICS_MAX_ACTIVITY_RECORDS_PER_NODE = 5; +const ISSUE_LIST_RELATED_QUERY_CHUNK_SIZE = 500; +export const MAX_CHILD_ISSUES_CREATED_BY_HELPER = 25; +const MAX_CHILD_COMPLETION_SUMMARIES = 20; +const CHILD_COMPLETION_SUMMARY_BODY_MAX_CHARS = 500; +// Non-human author sentinels that agents post under. These ARE eligible for +// agent-attribution derivation even though `local-board` is also materialized +// as a row in the `user` table (it is the implicit board admin). Genuine human +// users — real signups with their own ids — are never reattributed. +const NON_HUMAN_SENTINEL_AUTHOR_USER_IDS = new Set(["local-board"]); +const ISSUE_COMMENT_RUN_LOG_DERIVATION_MAX_LOG_BYTES = 2_000_000; +const ISSUE_COMMENT_RUN_LOG_DERIVATION_CHUNK_BYTES = 256_000; +const ISSUE_COMMENT_RUN_LOG_DERIVATION_END_SLACK_MS = 60_000; +const ISSUE_COMMENT_RUN_LOG_DERIVATION_MAX_PARALLEL_READS = 8; +const ISSUE_COMMENT_RUN_LOG_DERIVATION_TIMEOUT_MS = 3_000; +export const ISSUE_CREATE_IDEMPOTENCY_KEY_RETENTION_DAYS = 7; +const ISSUE_CREATE_IDEMPOTENCY_KEY_RETENTION_MS = + ISSUE_CREATE_IDEMPOTENCY_KEY_RETENTION_DAYS * 24 * 60 * 60 * 1000; +const ISSUE_CREATE_IDEMPOTENCY_KEY_CLEANUP_BATCH_SIZE = 500; +const DELETED_ISSUE_COMMENT_BODY = ""; +const ISSUE_WAKE_DIAGNOSTICS_ACTIVITY_ACTIONS = [ + "issue.tree_hold_wakeup_deferred", +] as const; + +export type IssuePostCommitAction = { + type: "cancel_native_question_run"; + runId: string; + issueId: string; + issueStatus: string; +}; + +/** Execute side effects that must never run before the issue transaction commits. */ +export async function executeIssuePostCommitActions( + db: Db, + actions: readonly IssuePostCommitAction[], +): Promise { + if (actions.length === 0) return; + const { heartbeatService } = await import("./heartbeat.js"); + const heartbeat = heartbeatService(db); + const cancelledRunIds = new Set(); + for (const action of actions) { + if (cancelledRunIds.has(action.runId)) continue; + cancelledRunIds.add(action.runId); + try { + await heartbeat.cancelRun( + action.runId, + "Task closed while waiting for operator input", + { + resultJson: { + cancelledByIssueStatus: action.issueStatus, + cancelledIssueId: action.issueId, + }, + }, + ); + } catch (err) { + // The durable marker written by the issue transaction remains available + // to startup and periodic recovery. Do not report a post-commit failure + // as though the already-committed issue transition had rolled back. + logger.warn( + { err, runId: action.runId, issueId: action.issueId }, + "native question cancellation deferred to recovery sweep", + ); + } + } +} + +function wakeRequestTargetsIssue(issueId: string) { + return sql`( + ${agentWakeupRequests.payload} ->> 'issueId' = ${issueId} + or ${agentWakeupRequests.payload} ->> 'taskId' = ${issueId} + or ${agentWakeupRequests.payload} -> '_paperclipWakeContext' ->> 'issueId' = ${issueId} + or ${agentWakeupRequests.payload} -> '_paperclipWakeContext' ->> 'taskId' = ${issueId} + )`; +} + +function wakeDiagnosticActivityTargetsIssue(issueId: string) { + return sql`( + (${activityLog.entityType} = 'issue' and ${activityLog.entityId} = ${issueId}) + or ${activityLog.details} ->> 'issueId' = ${issueId} + or ${activityLog.details} ->> 'rootIssueId' = ${issueId} + )`; +} + +function assertTransition(from: string, to: string) { + if (from === to) return; + if (!ALL_ISSUE_STATUSES.includes(to)) { + throw conflict(`Unknown issue status: ${to}`); + } +} + +function applyStatusSideEffects( + status: string | undefined, + patch: Partial, +): Partial { + if (!status) return patch; + + if (status === "in_progress" && !patch.startedAt) { + patch.startedAt = new Date(); + } + if (status === "done") { + patch.completedAt = new Date(); + } + if (status === "cancelled") { + patch.cancelledAt = new Date(); + } + return patch; +} + +function workspaceWorktreeRequiresProjectDetails() { + return { + code: WORKSPACE_WORKTREE_REQUIRES_PROJECT_CODE, + remediation: WORKSPACE_WORKTREE_REQUIRES_PROJECT_REMEDIATION, + }; +} + +function assertExplicitPinnedWorktreeIssueRunnable(input: { + projectId: string | null | undefined; + projectWorkspaceId: string | null | undefined; + executionWorkspaceId: string | null | undefined; + executionWorkspacePreference: string | null | undefined; + executionWorkspaceSettings: unknown; +}) { + const settings = parseIssueExecutionWorkspaceSettings( + input.executionWorkspaceSettings, + ); + const mode = settings?.mode; + if (mode !== "isolated_workspace" && mode !== "operator_branch") return; + + const resolvedMode = mode as ParsedExecutionWorkspaceMode; + if ( + isUnrunnableWorktreeCombo({ + issue: { + projectId: input.projectId ?? null, + projectWorkspaceId: input.projectWorkspaceId ?? null, + executionWorkspaceId: input.executionWorkspaceId ?? null, + executionWorkspacePreference: + input.executionWorkspacePreference ?? null, + }, + resolvedMode, + resolvedStrategy: resolvePinnedIssueWorkspaceStrategyType({ + mode: resolvedMode, + issueSettings: settings, + }), + hasResolvablePriorSessionWorkspace: false, + }) + ) { + throw unprocessable( + WORKSPACE_WORKTREE_REQUIRES_PROJECT_MESSAGE, + workspaceWorktreeRequiresProjectDetails(), + ); + } +} + +function readStringFromRecord(record: unknown, key: string) { + if (!record || typeof record !== "object") return null; + const value = (record as Record)[key]; + return typeof value === "string" && value.trim().length > 0 + ? value.trim() + : null; +} + +async function resolveResponsibleUserIdForIssueCreate( + reader: DbReader, + companyId: string, + input: { + explicitResponsibleUserId?: string | null; + createdByUserId?: string | null; + parentId?: string | null; + originKind?: string | null; + originRunId?: string | null; + actorRunId?: string | null; + actorResponsibleUserId?: string | null; + trustExplicitResponsibleUserId?: boolean; + }, +) { + const explicitResponsibleUserId = readStringFromRecord( + input, + "explicitResponsibleUserId", + ); + if ( + explicitResponsibleUserId && + input.trustExplicitResponsibleUserId === true + ) + return explicitResponsibleUserId; + + if (input.originKind === "routine_execution" && input.originRunId) { + const routineRun = await reader + .select({ responsibleUserId: routineRuns.responsibleUserId }) + .from(routineRuns) + .where( + and( + eq(routineRuns.companyId, companyId), + eq(routineRuns.id, input.originRunId), + ), + ) + .then((rows) => rows[0] ?? null); + if (routineRun?.responsibleUserId) return routineRun.responsibleUserId; + } + + const actorResponsibleUserId = readStringFromRecord( + input, + "actorResponsibleUserId", + ); + if (actorResponsibleUserId) return actorResponsibleUserId; + + if (input.actorRunId) { + const actorRun = await reader + .select({ responsibleUserId: heartbeatRuns.responsibleUserId }) + .from(heartbeatRuns) + .where( + and( + eq(heartbeatRuns.companyId, companyId), + eq(heartbeatRuns.id, input.actorRunId), + ), + ) + .then((rows) => rows[0] ?? null); + if (actorRun?.responsibleUserId) return actorRun.responsibleUserId; + } + + if (input.parentId) { + const parent = await reader + .select({ + responsibleUserId: issues.responsibleUserId, + createdByUserId: issues.createdByUserId, + }) + .from(issues) + .where( + and(eq(issues.companyId, companyId), eq(issues.id, input.parentId)), + ) + .then((rows) => rows[0] ?? null); + if (parent?.responsibleUserId) return parent.responsibleUserId; + if (parent?.createdByUserId) return parent.createdByUserId; + } + + return input.createdByUserId ?? null; +} + +function buildReusedExecutionWorkspaceConfigPatchFromIssueSettings( + settings: ReturnType, +) { + return { + environmentId: settings?.environmentId ?? null, + provisionCommand: settings?.workspaceStrategy?.provisionCommand ?? null, + runtimeProvisionCommand: + settings?.workspaceStrategy?.runtimeProvisionCommand ?? null, + teardownCommand: settings?.workspaceStrategy?.teardownCommand ?? null, + workspaceRuntime: settings?.workspaceRuntime ?? null, + }; +} + +// Accepted-plan children are not realized yet, so carry only unresolved +// workspace intent and let the first child run render/persist its own branch. +function buildPreRealizationExecutionWorkspaceSettings( + raw: unknown, +): Record | null { + const settings = parseIssueExecutionWorkspaceSettings(raw, { + includeEnvironmentId: true, + }); + if (!settings) return null; + const mode = + settings.mode && + settings.mode !== "inherit" && + settings.mode !== "reuse_existing" + ? settings.mode + : null; + const next: Record = {}; + if (mode) next.mode = mode; + if (settings.environmentId !== undefined) + next.environmentId = settings.environmentId; + if (settings.workspaceRuntime) + next.workspaceRuntime = settings.workspaceRuntime; + if (settings.workspaceStrategy) { + next.workspaceStrategy = { + type: settings.workspaceStrategy.type, + ...(settings.workspaceStrategy.baseRef + ? { baseRef: settings.workspaceStrategy.baseRef } + : {}), + ...(settings.workspaceStrategy.branchTemplate + ? { branchTemplate: settings.workspaceStrategy.branchTemplate } + : {}), + ...(settings.workspaceStrategy.existingBranch + ? { existingBranch: settings.workspaceStrategy.existingBranch } + : {}), + ...(settings.workspaceStrategy.worktreeParentDir + ? { worktreeParentDir: settings.workspaceStrategy.worktreeParentDir } + : {}), + ...(settings.workspaceStrategy.provisionCommand + ? { provisionCommand: settings.workspaceStrategy.provisionCommand } + : {}), + ...(settings.workspaceStrategy.runtimeProvisionCommand + ? { + runtimeProvisionCommand: + settings.workspaceStrategy.runtimeProvisionCommand, + } + : {}), + ...(settings.workspaceStrategy.teardownCommand + ? { teardownCommand: settings.workspaceStrategy.teardownCommand } + : {}), + }; + } + return Object.keys(next).length > 0 ? next : null; +} + +function toTimestampMs(value: Date | string | null | undefined) { + if (!value) return null; + const date = value instanceof Date ? value : new Date(value); + const timestamp = date.getTime(); + return Number.isFinite(timestamp) ? timestamp : null; +} + +type IssueCommentRunLogAttributionCandidate = { + id: string; + createdAt: Date | string; + authorAgentId?: string | null; + authorUserId?: string | null; + createdByRunId?: string | null; +}; + +type IssueCommentRunLogAttributionRun = { + runId: string; + agentId: string; + createdAt: Date | string; + startedAt?: Date | string | null; + finishedAt?: Date | string | null; + // Best-effort run log text. May be empty when logs were not read for a tier + // that does not need them (run-id / run-window-unique); only the + // `run_log_comment_post` tier consults this. + logContent: string; +}; + +type DerivedIssueCommentAttribution = { + derivedAuthorAgentId: string; + derivedCreatedByRunId: string; + derivedAuthorSource: IssueCommentDerivedAuthorSource; +}; + +/** + * Resolve a `created_by_run_id` safe for the heartbeat_runs FK; returns null for + * missing/invalid ids so an unknown run id never 500s a comment insert. + */ +async function resolveCommentCreatedByRun( + dbOrTx: any, + companyId: string, + runId: string | null | undefined, +): Promise<{ id: string; contextSnapshot: unknown } | null> { + const normalized = typeof runId === "string" ? runId.trim() : ""; + if (!normalized || !isUuidLike(normalized)) return null; + const existing = await dbOrTx + .select({ + id: heartbeatRuns.id, + contextSnapshot: heartbeatRuns.contextSnapshot, + }) + .from(heartbeatRuns) + .where( + and( + eq(heartbeatRuns.id, normalized), + eq(heartbeatRuns.companyId, companyId), + ), + ) + .then( + (rows: Array<{ id: string; contextSnapshot: unknown }>) => + rows[0] ?? null, + ); + return existing; +} + +export type ChatPublicationBinding = { + companyId: string; + conversationId: string; + endpointId: string; +}; + +async function resolvePublishedInteractionPromptBindings( + dbOrTx: any, + companyId: string, + issueId: string, + interactionId: string, + agentId: string, +): Promise { + const [publications, processedActions] = (await Promise.all([ + dbOrTx + .select({ + companyId: chatConversations.companyId, + conversationId: chatConversations.id, + endpointId: chatConversations.endpointId, + idempotencyKey: chatPublications.idempotencyKey, + payload: chatPublications.payload, + }) + .from(chatPublications) + .innerJoin( + chatConversations, + and( + eq(chatConversations.companyId, chatPublications.companyId), + eq(chatConversations.endpointId, chatPublications.endpointId), + eq(chatConversations.id, chatPublications.conversationId), + ), + ) + .innerJoin( + chatEndpoints, + and( + eq(chatEndpoints.companyId, chatConversations.companyId), + eq(chatEndpoints.id, chatConversations.endpointId), + ), + ) + .where( + and( + eq(chatPublications.companyId, companyId), + eq(chatPublications.issueId, issueId), + eq(chatPublications.state, "published"), + eq(chatConversations.issueId, issueId), + eq(chatEndpoints.assignedAgentId, agentId), + eq( + sql`${chatPublications.payload}->>'interactionId'`, + interactionId, + ), + notExists( + dbOrTx + .select({ id: chatPublications.id }) + .from(chatPublications) + .where( + and( + eq(chatPublications.conversationId, chatConversations.id), + eq(chatPublications.state, "published"), + or( + like(chatPublications.idempotencyKey, "control:new:%"), + like(chatPublications.idempotencyKey, "control:close:%"), + ), + ), + ), + ), + ), + ), + dbOrTx + .select({ + conversationId: chatActions.conversationId, + kind: chatActions.kind, + result: chatActions.result, + }) + .from(chatActions) + .where( + and( + eq(chatActions.companyId, companyId), + eq(chatActions.status, "processed"), + or( + inArray(chatActions.kind, [ + "question_answer", + "question_form_submit", + ]), + and( + eq(chatActions.kind, "confirmation_response"), + eq( + sql`${chatActions.result}->>'code'`, + "confirmation_resolution_committed_by_provider", + ), + ), + ), + eq( + sql`${chatActions.payload}->>'interactionId'`, + interactionId, + ), + ), + ), + ])) as [ + Array< + ChatPublicationBinding & { + idempotencyKey: string; + payload: { card?: { actions?: Array<{ type: string }> } }; + } + >, + Array<{ conversationId: string }>, + ]; + const providerWinnerConversationIds = new Set( + processedActions.map( + (action: { conversationId: string }) => action.conversationId, + ), + ); + const exactProviderWinner = providerWinnerConversationIds.size > 0; + const bindings = publications + .filter( + (publication) => + publication.idempotencyKey === + `interaction:${interactionId}:${publication.endpointId}` && + (!exactProviderWinner || + providerWinnerConversationIds.has(publication.conversationId)), + ) + .map((publication) => ({ + companyId: publication.companyId, + conversationId: publication.conversationId, + endpointId: publication.endpointId, + })); + return [ + ...new Map( + bindings.map((binding) => [binding.conversationId, binding]), + ).values(), + ]; +} + +function readChatWakeCommentIds( + contextSnapshot: Record, +): string[] { + const ids: string[] = []; + const append = (value: unknown) => { + if (typeof value !== "string") return; + const normalized = value.trim(); + if (!normalized || !isUuidLike(normalized) || ids.includes(normalized)) { + return; + } + ids.push(normalized); + }; + + const batched = contextSnapshot.wakeCommentIds; + if (Array.isArray(batched)) { + for (const value of batched) append(value); + } + append(contextSnapshot.wakeCommentId); + append(contextSnapshot.commentId); + return ids; +} + +/** A close retires its admitted sources, not every future source in a reusable + * provider thread. This exception requires a complete, independently admitted + * fresh batch; active state or copied run context alone never crosses a close. + * It grants presentation only. Provider dispatch retains its own current gate. */ +async function freshChatSourceAfterPublishedControl( + dbOrTx: any, + companyId: string, + issueId: string, + runId: string, + binding: ChatPublicationBinding, +): Promise { + const [run] = (await dbOrTx + .select() + .from(heartbeatRuns) + .where( + and(eq(heartbeatRuns.id, runId), eq(heartbeatRuns.companyId, companyId)), + ) + .limit(1)) as Array; + const context = run?.contextSnapshot ?? {}; + const commentIds = readChatWakeCommentIds(context); + if ( + !run?.wakeupRequestId || + !run.startedAt || + !commentIds.length || + commentIds.length > 50 || + (run.nativeIssueId !== null && run.nativeIssueId !== issueId) || + (context.issueId ?? context.taskId) !== issueId + ) + return false; + const [owner] = (await dbOrTx + .select() + .from(agentWakeupRequests) + .where( + and( + eq(agentWakeupRequests.id, run.wakeupRequestId), + eq(agentWakeupRequests.companyId, companyId), + eq(agentWakeupRequests.agentId, run.agentId), + eq(agentWakeupRequests.runId, run.id), + ), + ) + .limit(1)) as Array; + if (!owner || ["skipped", "cancelled"].includes(owner.status)) return false; + const admitted = (await dbOrTx + .select({ commentId: sql`${chatActions.payload}->>'commentId'` }) + .from(chatActions) + .innerJoin( + agentWakeupRequests, + and( + eq(agentWakeupRequests.id, chatActions.id), + eq(agentWakeupRequests.companyId, chatActions.companyId), + ), + ) + .where( + and( + eq(chatActions.companyId, companyId), + eq(chatActions.kind, "inbound_wakeup"), + or( + eq(agentWakeupRequests.runId, run.id), + eq(agentWakeupRequests.id, owner.id), + sql`${agentWakeupRequests.payload}->>'coalescedIntoWakeupRequestId' = ${owner.id}`, + ), + ), + ) + .limit(51)) as Array<{ commentId: string }>; + if ( + admitted.length !== commentIds.length || + admitted.some((row) => !commentIds.includes(row.commentId)) + ) + return false; + const [latestControl] = (await dbOrTx + .select({ publishedAt: chatPublications.publishedAt }) + .from(chatPublications) + .where( + and( + eq(chatPublications.companyId, companyId), + eq(chatPublications.endpointId, binding.endpointId), + eq(chatPublications.conversationId, binding.conversationId), + eq(chatPublications.state, "published"), + or( + like(chatPublications.idempotencyKey, "control:new:%"), + like(chatPublications.idempotencyKey, "control:close:%"), + ), + ), + ) + .orderBy(desc(chatPublications.publishedAt)) + .limit(1)) as Array<{ publishedAt: Date | null }>; + if (!latestControl?.publishedAt) return false; + const rows = (await dbOrTx + .select({ + action: chatActions, + receipt: agentWakeupRequests, + link: chatMessageLinks, + delivery: chatDeliveries, + comment: issueComments, + conversation: chatConversations, + endpoint: chatEndpoints, + connection: toolConnections, + }) + .from(chatActions) + .innerJoin( + agentWakeupRequests, + and( + eq(agentWakeupRequests.id, chatActions.id), + eq(agentWakeupRequests.companyId, chatActions.companyId), + ), + ) + .innerJoin( + chatMessageLinks, + and( + eq(chatMessageLinks.companyId, chatActions.companyId), + eq(chatMessageLinks.endpointId, chatActions.endpointId), + eq(chatMessageLinks.conversationId, chatActions.conversationId), + eq(chatMessageLinks.deliveryId, chatActions.deliveryId), + sql`${chatMessageLinks.commentId}::text = ${chatActions.payload}->>'commentId'`, + eq(chatMessageLinks.direction, "inbound"), + ), + ) + .innerJoin( + chatDeliveries, + and( + eq(chatDeliveries.id, chatActions.deliveryId), + eq(chatDeliveries.companyId, chatActions.companyId), + eq(chatDeliveries.endpointId, chatActions.endpointId), + eq(chatDeliveries.conversationId, chatActions.conversationId), + eq(chatDeliveries.principalId, chatActions.principalId), + ), + ) + .innerJoin( + issueComments, + and( + eq(issueComments.id, chatMessageLinks.commentId), + eq(issueComments.companyId, companyId), + eq(issueComments.issueId, issueId), + ), + ) + .innerJoin( + chatConversations, + and( + eq(chatConversations.id, binding.conversationId), + eq(chatConversations.id, chatActions.conversationId), + eq(chatConversations.companyId, companyId), + eq(chatConversations.issueId, issueId), + eq(chatConversations.endpointId, binding.endpointId), + ), + ) + .innerJoin( + chatEndpoints, + and( + eq(chatEndpoints.id, chatConversations.endpointId), + eq(chatEndpoints.companyId, companyId), + eq(chatEndpoints.assignedAgentId, run.agentId), + ), + ) + .innerJoin( + issues, + and( + eq(issues.id, issueId), + eq(issues.companyId, companyId), + eq(issues.assigneeAgentId, run.agentId), + isNull(issues.hiddenAt), + ), + ) + .innerJoin( + toolConnections, + and( + eq(toolConnections.id, chatEndpoints.connectionId), + eq(toolConnections.companyId, companyId), + ), + ) + .where( + and( + eq(chatActions.companyId, companyId), + eq(chatActions.kind, "inbound_wakeup"), + or( + eq(agentWakeupRequests.runId, run.id), + eq(agentWakeupRequests.id, owner.id), + sql`${agentWakeupRequests.payload}->>'coalescedIntoWakeupRequestId' = ${owner.id}`, + ), + isNull(issueComments.deletedAt), + sql`${issueComments.updatedAt} = ${issueComments.createdAt}`, + sql`${issueComments.createdAt} <= (select started_at from heartbeat_runs where id = ${run.id})`, + // Compare in SQL, retaining microseconds. A delayed pre-close source or + // a source preceding any later close remains internal after reopening. + notExists( + dbOrTx + .select({ id: chatPublications.id }) + .from(chatPublications) + .where( + and( + eq(chatPublications.companyId, companyId), + eq(chatPublications.endpointId, binding.endpointId), + eq(chatPublications.conversationId, binding.conversationId), + eq(chatPublications.state, "published"), + or( + like(chatPublications.idempotencyKey, "control:new:%"), + like(chatPublications.idempotencyKey, "control:close:%"), + ), + or( + isNull(chatPublications.publishedAt), + gte(chatPublications.publishedAt, chatDeliveries.receivedAt), + ), + ), + ), + ), + ), + ) + .limit(51)) as Array<{ + action: typeof chatActions.$inferSelect; + receipt: typeof agentWakeupRequests.$inferSelect; + link: typeof chatMessageLinks.$inferSelect; + delivery: typeof chatDeliveries.$inferSelect; + comment: typeof issueComments.$inferSelect; + conversation: typeof chatConversations.$inferSelect; + endpoint: typeof chatEndpoints.$inferSelect; + connection: typeof toolConnections.$inferSelect; + }>; + if ( + rows.length !== commentIds.length || + new Set(rows.map((row) => row.comment.id)).size !== commentIds.length || + rows.some((row) => !commentIds.includes(row.comment.id)) + ) + return false; + for (const row of rows) { + const { + action, + receipt, + delivery, + comment, + conversation, + endpoint, + connection, + } = row; + const payload = action.payload; + const event = parseObject(delivery.normalizedEvent); + const fence = parseObject(event.runtimeContext); + const sourceConversation = parseObject(event.conversation); + const message = parseObject(event.message); + const sentAt = + typeof message.providerSentAt === "string" && + message.providerSentAt.length <= 64 + ? Date.parse(message.providerSentAt) + : Number.NaN; + const refs = connection.credentialSecretRefs + .map((ref) => ({ + configPath: ref.configPath, + secretId: ref.secretId, + versionSelector: ref.versionSelector ?? "latest", + })) + .sort((left, right) => + `${left.configPath}:${left.secretId}:${left.versionSelector}`.localeCompare( + `${right.configPath}:${right.secretId}:${right.versionSelector}`, + ), + ); + if ( + action.status !== "processed" || + action.providerActionId !== `inbound_wakeup:${delivery.id}` || + payload.version !== 1 || + !action.principalId || + payload.issueId !== issueId || + payload.agentId !== run.agentId || + payload.commentId !== comment.id || + payload.sessionGeneration !== conversation.sessionGeneration || + delivery.state !== "processed" || + // Receipt time alone cannot make a delayed old provider message fresh. + // Missing or future provider time denies. The shared control reader below + // proves chronology against the actual command, not its later receipt. + !Number.isFinite(sentAt) || + // Historical SDK display clocks were sometimes server-local fallbacks. + // They cannot be retrospectively promoted to post-control chronology. + (endpoint.provider === "microsoft-teams" && + message.providerSentAtSource !== "teams_activity_timestamp") || + (endpoint.provider === "telegram" && + message.providerSentAtSource !== "telegram_message_date") || + sentAt > delivery.receivedAt.getTime() || + !delivery.processedAt || + !connection.enabled || + connection.status !== "active" || + fence.generation !== + Number(parseObject(endpoint.setup).runtimeGeneration ?? 0) || + fence.credentialFingerprint !== + createHash("sha256").update(JSON.stringify(refs)).digest("hex") || + sourceConversation.externalConversationId !== + conversation.externalConversationId || + sourceConversation.externalThreadId !== conversation.externalThreadId || + message.providerMessageId !== row.link.providerMessageId || + !["user", "system"].includes(String(payload.requestedByActorType)) || + typeof payload.requestedByActorId !== "string" || + receipt.payload?.issueId !== issueId || + receipt.payload?.wakeCommentId !== comment.id || + (receipt.id !== owner.id && + (receipt.status !== "coalesced" || + receipt.payload?.coalescedIntoWakeupRequestId !== owner.id)) || + (payload.requestedByActorType === "user" + ? comment.authorUserId !== payload.requestedByActorId + : comment.authorUserId !== null || + payload.requestedByActorId !== action.principalId) + ) + return false; + // The SQL above independently requires admission after every published + // control. A provider source after the command but before its confirmation + // can qualify only with complete exact proof for this published-control set. + if ( + (await readChatControlChronology( + dbOrTx, + endpoint, + { + id: conversation.externalThreadId, + channelId: conversation.externalConversationId, + }, + delivery, + { + requirePublishedControlProofForConversationId: binding.conversationId, + }, + )) !== "after_all_proven_controls" + ) + return false; + try { + assertDurableChatWakeupReceipt( + createDurableChatWakeupRequest({ + id: action.id, + companyId, + agentId: run.agentId, + issueId, + commentId: comment.id, + requestedByActorType: payload.requestedByActorType as + "user" | "system", + requestedByActorId: payload.requestedByActorId, + requestedAt: action.createdAt, + authorize: async () => {}, + }), + receipt, + ); + } catch { + return false; + } + const [lifecycle] = await dbOrTx + .select({ id: chatDeliveries.id }) + .from(chatDeliveries) + .where( + and( + eq(chatDeliveries.companyId, companyId), + eq(chatDeliveries.endpointId, binding.endpointId), + ne(chatDeliveries.state, "filtered"), + inArray(chatDeliveries.eventKind, [ + "message_updated", + "message_deleted", + "message_restored", + ]), + sql`${chatDeliveries.normalizedEvent}->'runtimeContext' = ${JSON.stringify(fence)}::jsonb`, + sql`${chatDeliveries.normalizedEvent}->'message'->>'targetProviderEventId' = ${delivery.providerEventId}`, + ), + ) + .limit(1); + if (lifecycle) return false; + } + try { + const current = await authorizeChatConversationForBoundRun( + dbOrTx, + { companyId, issueId, runId, agentId: run.agentId }, + context, + "read", + ); + return ( + current.conversationId === binding.conversationId && + current.endpointId === binding.endpointId + ); + } catch (error) { + if ( + error instanceof Error && + /^paperclip_runner_chat_attachment_(binding|destination|principal)_denied$/.test( + error.message, + ) + ) + return false; + throw error; + } +} + +export async function resolveChatOriginPublicationBindings( + dbOrTx: any, + companyId: string, + issueId: string, + runId: string | null, +): Promise { + if (!runId) return []; + const explicitEmail = await dbOrTx.select({ id: chatEndpoints.id }).from(chatEndpoints) + .innerJoin(chatConversations, eq(chatConversations.endpointId, chatEndpoints.id)) + .where(and(eq(chatConversations.companyId, companyId), eq(chatConversations.issueId, issueId), eq(chatEndpoints.publicationMode, "explicit"))).limit(1); + if (explicitEmail.length) return []; + + let originRunId = runId; + let contextSnapshot: Record | null = null; + let lineageAgentId: string | null = null; + const visitedRunIds = new Set(); + // Every hop needs its own durable authorization receipt. A task may continue + // many times; an arbitrary hop limit must not silently discard its replies. + // The visited set rejects cycles while keeping valid long lineages readable. + while (true) { + if (visitedRunIds.has(originRunId)) return []; + visitedRunIds.add(originRunId); + const run = await dbOrTx + .select({ + agentId: heartbeatRuns.agentId, + responsibleUserId: heartbeatRuns.responsibleUserId, + contextSnapshot: sql | null>` + case when jsonb_typeof(${heartbeatRuns.contextSnapshot}) = 'object' then + (select coalesce(jsonb_object_agg(entry.key, entry.value), '{}'::jsonb) + from jsonb_each(${heartbeatRuns.contextSnapshot}) as entry + where entry.key in ('issueId', 'taskId', 'wakeCommentIds', 'wakeCommentId', + 'commentId', 'source', 'sourceRunId', 'interactionId', 'endpointId')) + else ${heartbeatRuns.contextSnapshot} end`, + }) + .from(heartbeatRuns) + .where( + and( + eq(heartbeatRuns.id, originRunId), + eq(heartbeatRuns.companyId, companyId), + ), + ) + .then( + ( + rows: Array<{ + agentId: string; + responsibleUserId: string | null; + contextSnapshot: Record | null; + }>, + ) => rows[0] ?? null, + ); + if (!run?.agentId) return []; + if (lineageAgentId === null) lineageAgentId = run.agentId; + else if (run.agentId !== lineageAgentId) return []; + const snapshot = run?.contextSnapshot; + if (!snapshot) return []; + const snapshotIssueId = + readStringFromRecord(snapshot, "issueId") ?? + readStringFromRecord(snapshot, "taskId"); + if (snapshotIssueId && snapshotIssueId !== issueId) return []; + + const boardBindings = await slackBoardReplyBindings(dbOrTx, { + companyId, issueId, agentId: lineageAgentId!, + commentIds: readChatWakeCommentIds(snapshot), userId: run.responsibleUserId, + }); + if (boardBindings.length) return boardBindings; + + // A native runner can emit its continuation immediately after the durable + // `request.resolve` command is queued, before the delivery worker records + // its terminal receipt. The source run plus the already-published native + // prompt is a stronger causal edge than that tiny receipt-commit window. + const inFlightNativeInteractions: Array<{ interactionId: string }> = + await dbOrTx + .select({ + interactionId: issueQuestionResponseDeliveries.interactionId, + }) + .from(issueQuestionResponseDeliveries) + .innerJoin( + issueThreadInteractions, + and( + eq( + issueThreadInteractions.companyId, + issueQuestionResponseDeliveries.companyId, + ), + eq( + issueThreadInteractions.issueId, + issueQuestionResponseDeliveries.issueId, + ), + eq( + issueThreadInteractions.id, + issueQuestionResponseDeliveries.interactionId, + ), + ), + ) + .where( + and( + eq(issueQuestionResponseDeliveries.companyId, companyId), + eq(issueQuestionResponseDeliveries.issueId, issueId), + eq(issueQuestionResponseDeliveries.sourceRunId, originRunId), + eq(issueQuestionResponseDeliveries.status, "delivering"), + eq(issueThreadInteractions.kind, "ask_user_questions"), + eq(issueThreadInteractions.status, "answered"), + ), + ); + for (const candidate of inFlightNativeInteractions) { + const promptBindings = await resolvePublishedInteractionPromptBindings( + dbOrTx, + companyId, + issueId, + candidate.interactionId, + lineageAgentId!, + ); + if (promptBindings.length > 0) return promptBindings; + } + + // A question response can be delivered directly into an already-running + // successor without rewriting that run's pre-existing context snapshot. + // The durable delivery receipt is the authoritative causal edge and must + // win even when the target run already has a different `chat:*` source; + // otherwise a Slack answer steered into a Telegram-origin run on the same + // issue could publish the result to the wrong provider identity. + const steeredQuestionParents: Array<{ + interactionId: string; + sourceRunId: string | null; + }> = await dbOrTx + .select({ + interactionId: issueQuestionResponseDeliveries.interactionId, + sourceRunId: issueQuestionResponseDeliveries.sourceRunId, + }) + .from(issueQuestionResponseDeliveries) + .innerJoin( + issueThreadInteractions, + and( + eq( + issueThreadInteractions.companyId, + issueQuestionResponseDeliveries.companyId, + ), + eq( + issueThreadInteractions.issueId, + issueQuestionResponseDeliveries.issueId, + ), + eq( + issueThreadInteractions.id, + issueQuestionResponseDeliveries.interactionId, + ), + eq( + issueThreadInteractions.sourceRunId, + issueQuestionResponseDeliveries.sourceRunId, + ), + ), + ) + .where( + and( + eq(issueQuestionResponseDeliveries.companyId, companyId), + eq(issueQuestionResponseDeliveries.issueId, issueId), + eq(issueQuestionResponseDeliveries.targetRunId, originRunId), + inArray(issueQuestionResponseDeliveries.status, [ + "delivered", + "fallback_queued", + ]), + eq(issueThreadInteractions.kind, "ask_user_questions"), + eq(issueThreadInteractions.status, "answered"), + ), + ); + if (steeredQuestionParents.length > 0) { + const parentRunIds = [ + ...new Set( + steeredQuestionParents + .map((candidate) => candidate.sourceRunId) + .filter((candidate): candidate is string => Boolean(candidate)), + ), + ]; + // Multiple responses may coalesce into one target only when they share + // the same verified origin. Incompatible origins are intentionally + // internal rather than guessing a provider destination. + if (parentRunIds.length !== 1 || !isUuidLike(parentRunIds[0])) return []; + const promptBindings = ( + await Promise.all( + [ + ...new Set( + steeredQuestionParents.map( + (candidate) => candidate.interactionId, + ), + ), + ].map((interactionId) => + resolvePublishedInteractionPromptBindings( + dbOrTx, + companyId, + issueId, + interactionId, + lineageAgentId!, + ), + ), + ) + ).flat(); + if (promptBindings.length > 0) { + return [ + ...new Map( + promptBindings.map((binding) => [binding.conversationId, binding]), + ).values(), + ]; + } + // Native delivery resumes the source run in place and records that same + // run as its target. That is already the chat root, not a lineage hop. + if (parentRunIds[0] !== originRunId) { + originRunId = parentRunIds[0]; + continue; + } + } + + const contextSource = readStringFromRecord(snapshot, "source"); + if (contextSource === "tool_action_review") { + // Review results use their own durable wake, not the ordinary interaction + // response key. Attest the entire batch before following its origin; the + // model's context/sourceRunId alone never grants publication authority. + const [wake] = await dbOrTx.select({ payload: agentWakeupRequests.payload, + idempotencyKey: agentWakeupRequests.idempotencyKey }) + .from(agentWakeupRequests).where(and( + eq(agentWakeupRequests.companyId, companyId), + eq(agentWakeupRequests.agentId, lineageAgentId!), + eq(agentWakeupRequests.runId, originRunId), + like(agentWakeupRequests.idempotencyKey, "tool-action-response:%"), + )).limit(1); + const requestIds = wake?.payload?.toolActionRequestIds; + const sourceRunId = readStringFromRecord(snapshot, "sourceRunId"); + if (!Array.isArray(requestIds) || requestIds.length === 0 || + requestIds.some((id: unknown) => typeof id !== "string" || !isUuidLike(id)) || + !sourceRunId || !isUuidLike(sourceRunId) || sourceRunId !== wake.payload.sourceRunId || + wake.idempotencyKey !== `tool-action-response:${requestIds[0]}` || + snapshot.interactionId !== wake.payload.interactionId) return []; + const receipts = await dbOrTx.select({ id: toolActionRequests.id }) + .from(toolActionRequests) + .innerJoin(toolInvocations, and( + eq(toolInvocations.id, toolActionRequests.invocationId), + eq(toolInvocations.companyId, companyId), + eq(toolInvocations.issueId, issueId), + eq(toolInvocations.agentId, lineageAgentId!), + eq(toolInvocations.runId, sourceRunId), + )) + .innerJoin(toolActionDeliveries, and( + eq(toolActionDeliveries.actionRequestId, toolActionRequests.id), + eq(toolActionDeliveries.companyId, companyId), + eq(toolActionDeliveries.issueId, issueId), + eq(toolActionDeliveries.interactionId, toolActionRequests.interactionId), + )) + .where(and( + eq(toolActionRequests.companyId, companyId), + eq(toolActionRequests.issueId, issueId), + eq(toolActionRequests.requestedByAgentId, lineageAgentId!), + inArray(toolActionRequests.id, requestIds), + inArray(toolActionRequests.status, ["executed", "failed", "rejected", "expired", "cancelled"]), + )); + if (receipts.length !== requestIds.length) return []; + originRunId = sourceRunId; + continue; + } + if (contextSource?.startsWith("chat:")) { + contextSnapshot = snapshot; + break; + } + if (contextSource === "connection_intent.resolved") { + const interactionId = readStringFromRecord(snapshot, "interactionId"); + if (!interactionId || !isUuidLike(interactionId)) return []; + const sourceRunId = await resolveConnectionIntentContinuationSource(dbOrTx, { + companyId, issueId, interactionId, runId: originRunId, agentId: lineageAgentId!, + }); + if (!sourceRunId) return []; + // An automation can request access on an already-linked Slack task. + // Its validated receipt and published prompt preserve that conversation. + const promptBindings = await resolvePublishedInteractionPromptBindings( + dbOrTx, companyId, issueId, interactionId, lineageAgentId!, + ); + if (promptBindings.length > 0) return promptBindings; + originRunId = sourceRunId; + continue; + } + if (contextSource === "issue.monitor") { + const sourceRunId = await readNativeMonitorSource(dbOrTx, { companyId, issueId, runId: originRunId }); + if (!sourceRunId) return []; + originRunId = sourceRunId; + continue; + } + if (contextSource === "native_status_decision") { + const sourceRunId = await readNativeContinuationSource(dbOrTx, { companyId, issueId, runId: originRunId }); + if (!sourceRunId) return []; + originRunId = sourceRunId; + continue; + } + if ( + contextSource !== "issue.interaction.respond" && + contextSource !== "issue.interaction.accept" && + contextSource !== "issue.interaction.reject" && + contextSource !== "issue.interaction.verdicts" && + contextSource !== "issue.interaction.cancel" && + contextSource !== "issue.interaction.withdraw" && + contextSource !== "external_chat.interaction.resolve" + ) + return []; + + const interactionId = readStringFromRecord(snapshot, "interactionId"); + const sourceRunId = readStringFromRecord(snapshot, "sourceRunId"); + if ( + !interactionId || + !isUuidLike(interactionId) || + !sourceRunId || + !isUuidLike(sourceRunId) + ) + return []; + const interaction = await dbOrTx + .select({ + kind: issueThreadInteractions.kind, + status: issueThreadInteractions.status, + result: issueThreadInteractions.result, + sourceRunId: issueThreadInteractions.sourceRunId, + }) + .from(issueThreadInteractions) + .where( + and( + eq(issueThreadInteractions.id, interactionId), + eq(issueThreadInteractions.companyId, companyId), + eq(issueThreadInteractions.issueId, issueId), + eq(issueThreadInteractions.sourceRunId, sourceRunId), + ), + ) + .then( + ( + rows: Array<{ + kind: string; + status: string; + result: unknown; + sourceRunId: string | null; + }>, + ) => rows[0] ?? null, + ); + if (!interaction?.sourceRunId) return []; + const isAnsweredQuestion = + interaction.kind === "ask_user_questions" && + interaction.status === "answered"; + const isResolvedConfirmation = + ["request_confirmation", "request_checkbox_confirmation", "suggest_tasks"].includes(interaction.kind) && + (interaction.status === "accepted" || interaction.status === "rejected"); + const isCancelledSupportedInteraction = + interaction.status === "cancelled" && + ["ask_user_questions", "request_confirmation", "request_checkbox_confirmation", "request_item_verdicts", "suggest_tasks"].includes(interaction.kind); + const isItemVerdictResponse = interaction.kind === "request_item_verdicts" && + (interaction.status === "answered" || interaction.status === "pending") && + contextSource === "issue.interaction.verdicts"; + if ( + !isItemVerdictResponse && + !isAnsweredQuestion && + !isResolvedConfirmation && + !isCancelledSupportedInteraction + ) + return []; + const responseDelivery = isAnsweredQuestion + ? await dbOrTx + .select({ + status: issueQuestionResponseDeliveries.status, + targetRunId: issueQuestionResponseDeliveries.targetRunId, + }) + .from(issueQuestionResponseDeliveries) + .where( + and( + eq(issueQuestionResponseDeliveries.companyId, companyId), + eq(issueQuestionResponseDeliveries.issueId, issueId), + eq(issueQuestionResponseDeliveries.interactionId, interactionId), + eq(issueQuestionResponseDeliveries.sourceRunId, sourceRunId), + ), + ) + .then( + (rows: Array<{ status: string; targetRunId: string | null }>) => + rows[0] ?? null, + ) + : null; + const recordedTarget = + isAnsweredQuestion && + responseDelivery && + (responseDelivery.status === "delivered" || + responseDelivery.status === "fallback_queued") + ? responseDelivery.targetRunId + : null; + if (recordedTarget && recordedTarget !== originRunId) return []; + const continuationIdempotencyKey = isAnsweredQuestion + ? `question-response:${interactionId}` + : `interaction:${interactionId}:${interaction.status}`; + const itemVerdictWake = isItemVerdictResponse + ? await resolveItemVerdictContinuationWake(dbOrTx, { + companyId, issueId, interactionId, sourceRunId, runId: originRunId, + agentId: lineageAgentId!, result: interaction.result, + }) + : null; + const durableWakeTarget = isItemVerdictResponse ? itemVerdictWake : recordedTarget + ? null + : await dbOrTx + .select({ id: agentWakeupRequests.id }) + .from(agentWakeupRequests) + .where( + and( + eq(agentWakeupRequests.companyId, companyId), + eq( + agentWakeupRequests.idempotencyKey, + continuationIdempotencyKey, + ), + eq(agentWakeupRequests.agentId, lineageAgentId!), + eq(agentWakeupRequests.runId, originRunId), + inArray(agentWakeupRequests.status, [ + "queued", + "claimed", + "running", + "succeeded", + "completed", + "failed", + "cancelled", + "coalesced", + "deferred_issue_execution", + "retrying", + "scheduled_retry", + ]), + ), + ) + .then((rows: Array<{ id: string }>) => rows[0] ?? null); + if (recordedTarget !== originRunId && !durableWakeTarget) return []; + const promptBindings = await resolvePublishedInteractionPromptBindings( + dbOrTx, + companyId, + issueId, + interactionId, + lineageAgentId!, + ); + if (promptBindings.length > 0) return promptBindings; + originRunId = interaction.sourceRunId; + } + if (!contextSnapshot || !lineageAgentId) return []; + + const wakeCommentIds = readChatWakeCommentIds(contextSnapshot); + if (wakeCommentIds.length === 0) return []; + const originEndpointId = readStringFromRecord(contextSnapshot, "endpointId"); + + const bindings: Array = + await dbOrTx + .select({ + companyId: chatConversations.companyId, + conversationId: chatConversations.id, + endpointId: chatConversations.endpointId, + hasControl: sql`exists (select 1 from chat_publications control where control.company_id = ${chatConversations.companyId} and control.endpoint_id = ${chatConversations.endpointId} and control.conversation_id = ${chatConversations.id} and control.state = 'published' and (control.idempotency_key like 'control:new:%' or control.idempotency_key like 'control:close:%'))`, + }) + .from(chatMessageLinks) + .innerJoin( + chatConversations, + and( + eq(chatConversations.companyId, chatMessageLinks.companyId), + eq(chatConversations.endpointId, chatMessageLinks.endpointId), + eq(chatConversations.id, chatMessageLinks.conversationId), + ), + ) + .innerJoin( + chatEndpoints, + and( + eq(chatEndpoints.companyId, chatConversations.companyId), + eq(chatEndpoints.id, chatConversations.endpointId), + ), + ) + .where( + and( + eq(chatMessageLinks.companyId, companyId), + eq(chatMessageLinks.direction, "inbound"), + inArray(chatMessageLinks.commentId, wakeCommentIds), + eq(chatConversations.issueId, issueId), + eq(chatEndpoints.assignedAgentId, lineageAgentId), + ...(originEndpointId + ? [eq(chatConversations.endpointId, originEndpointId)] + : []), + ), + ); + const authorized: ChatPublicationBinding[] = []; + // A coalesced batch has one inbound link per comment. Prove each destination + // once, not once per link (the proof itself still checks the complete batch). + const candidates = new Map( + bindings.map((binding) => [binding.conversationId, binding]), + ); + for (const { hasControl, ...binding } of candidates.values()) { + if ( + !hasControl || + (await freshChatSourceAfterPublishedControl( + dbOrTx, + companyId, + issueId, + originRunId, + binding, + )) + ) + authorized.push(binding); + } + return authorized; +} + +async function resolveCommentResponsibleUserId( + dbOrTx: any, + companyId: string, + createdByRunId: string | null, + actorResponsibleUserId: string | null | undefined, +): Promise { + const actorValue = actorResponsibleUserId?.trim() || null; + if (actorValue) return actorValue; + if (!createdByRunId) return null; + return dbOrTx + .select({ responsibleUserId: heartbeatRuns.responsibleUserId }) + .from(heartbeatRuns) + .where( + and( + eq(heartbeatRuns.id, createdByRunId), + eq(heartbeatRuns.companyId, companyId), + ), + ) + .then( + (rows: Array<{ responsibleUserId: string | null }>) => + rows[0]?.responsibleUserId?.trim() || null, + ); +} + +function withAgentCommentAuthorizationMetadata( + metadata: IssueCommentMetadata | null, + authorizationReason: string | null | undefined, +): IssueCommentMetadata { + const reason = authorizationReason?.trim() || "internal_agent_write"; + return { + version: 1, + ...(metadata?.sourceRunId !== undefined + ? { sourceRunId: metadata.sourceRunId } + : {}), + authorizationReason: reason, + sections: metadata?.sections.length + ? metadata.sections + : [ + { + title: "Authorization", + rows: [{ type: "key_value", label: "Reason", value: reason }], + }, + ], + }; +} + +/** + * Chat-origin runs may create internal presentation and bookkeeping comments. + * Only comments produced through an explicit agent write surface are authored + * for the external conversation and therefore eligible for auto-publication. + */ +export function isExplicitExternalAgentComment( + metadata: IssueCommentMetadata | null | undefined, +): boolean { + const reason = metadata?.authorizationReason?.trim() ?? ""; + return reason === "paperclip_runner_protocol" || reason.startsWith("allow_"); +} + +type SelectedChatPresentationAttachment = { + id: string; + commentId: string; + commentMetadata: IssueCommentMetadata | null; + originalFilename: string | null; +}; + +async function listSelectedChatPresentationAttachments( + dbOrTx: any, + input: { + companyId: string; + issueId: string; + agentId: string; + runId: string; + }, +): Promise { + const rows = await dbOrTx + .select({ + id: issueAttachments.id, + commentId: issueComments.id, + commentMetadata: issueComments.metadata, + originalFilename: assets.originalFilename, + }) + .from(issueAttachments) + .innerJoin(assets, eq(issueAttachments.assetId, assets.id)) + .innerJoin( + issueComments, + eq(issueAttachments.issueCommentId, issueComments.id), + ) + .where( + and( + eq(issueAttachments.companyId, input.companyId), + eq(issueAttachments.issueId, input.issueId), + eq(issueAttachments.originatingRunId, input.runId), + eq(assets.companyId, input.companyId), + eq(assets.createdByAgentId, input.agentId), + eq(issueComments.companyId, input.companyId), + eq(issueComments.issueId, input.issueId), + eq(issueComments.createdByRunId, input.runId), + eq(issueComments.authorType, "agent"), + eq(issueComments.authorAgentId, input.agentId), + isNull(issueComments.deletedAt), + sql`( + coalesce(${issueComments.metadata}->>'authorizationReason', '') = 'paperclip_runner_protocol' + or left(coalesce(${issueComments.metadata}->>'authorizationReason', ''), 6) = 'allow_' + )`, + ), + ) + .orderBy( + asc(issueComments.createdAt), + asc(issueComments.id), + asc(issueAttachments.createdAt), + asc(issueAttachments.id), + ); + return (rows as SelectedChatPresentationAttachment[]).filter((row) => + isExplicitExternalAgentComment(row.commentMetadata), + ); +} + +/** + * Best-effort agent attribution for comments whose stored author is a non-human + * sentinel (e.g. `local-board`). Callers MUST pre-filter `comments` to drop any + * comment whose `authorUserId` maps to a genuine user profile so a real board / + * user comment is never reattributed. + * + * Only LOSSLESS signals are used — a comment is reattributed solely when a run + * provably authored it. Pure run-window timing overlap is intentionally NOT a + * signal: because agents post through the `local-board` subprocess, an agent + * comment and a genuine human board comment are indistinguishable rows, so any + * timing-based guess mis-attributes human board comments that merely coincided + * with an agent run (Option A). + * + * Tiers, in descending confidence (first match wins per comment): + * 1. `run_id` — the comment's own `createdByRunId` resolves to an agent run + * (lossless: that run authored the comment). + * 2. `run_log_comment_post` — an overlapping run log contains the explicit + * `comment id: {id}` post marker (lossless: the run recorded posting it). + */ +export function deriveIssueCommentRunLogAttribution( + comments: readonly IssueCommentRunLogAttributionCandidate[], + runs: readonly IssueCommentRunLogAttributionRun[], +) { + const derivedByCommentId = new Map(); + const runById = new Map(runs.map((run) => [run.runId, run] as const)); + + for (const comment of comments) { + if (comment.authorAgentId || !comment.authorUserId) continue; + + // Tier 1: the comment carries the run that authored it. Lossless even when + // the author was recorded as the `local-board` sentinel. + if (comment.createdByRunId) { + const ownRun = runById.get(comment.createdByRunId); + if (ownRun?.agentId) { + derivedByCommentId.set(comment.id, { + derivedAuthorAgentId: ownRun.agentId, + derivedCreatedByRunId: ownRun.runId, + derivedAuthorSource: "run_id", + }); + continue; + } + } + + const commentCreatedAtMs = toTimestampMs(comment.createdAt); + if (commentCreatedAtMs === null) continue; + + const overlappingRuns: Array<{ + run: IssueCommentRunLogAttributionRun; + runEndMs: number; + }> = []; + for (const run of runs) { + const runStartMs = toTimestampMs(run.startedAt ?? run.createdAt); + const runEndMs = toTimestampMs(run.finishedAt ?? run.createdAt); + if (runStartMs === null || runEndMs === null) continue; + if ( + commentCreatedAtMs < runStartMs || + commentCreatedAtMs > + runEndMs + ISSUE_COMMENT_RUN_LOG_DERIVATION_END_SLACK_MS + ) { + continue; + } + overlappingRuns.push({ run, runEndMs }); + } + + // Tier 2: an overlapping run log explicitly recorded posting this comment. + let bestLogMatch: { + runId: string; + agentId: string; + distanceMs: number; + } | null = null; + for (const { run, runEndMs } of overlappingRuns) { + if (!run.logContent.includes(`comment id: ${comment.id}`)) continue; + const distanceMs = Math.abs(runEndMs - commentCreatedAtMs); + if (!bestLogMatch || distanceMs < bestLogMatch.distanceMs) { + bestLogMatch = { runId: run.runId, agentId: run.agentId, distanceMs }; + } + } + if (bestLogMatch) { + derivedByCommentId.set(comment.id, { + derivedAuthorAgentId: bestLogMatch.agentId, + derivedCreatedByRunId: bestLogMatch.runId, + derivedAuthorSource: "run_log_comment_post", + }); + continue; + } + + // No lossless signal — leave unresolved. A pure run-window timing overlap is + // deliberately NOT enough to reattribute (it cannot tell an agent comment + // from a human board comment that happened during the run). + } + + return derivedByCommentId; +} + +// Express's default `qs` parser binds repeated query keys to a `string[]`, +// so a request like `?status=todo&status=in_progress` arrives here as an +// array. Single-key + comma-separated forms remain valid too; normalize the +// supported shapes once so the service contract matches runtime reality. +export function parseStatusFilter( + input: string | readonly string[] | undefined, +): string[] { + if (input === undefined || input === null) return []; + const entries = Array.isArray(input) + ? input + : typeof input === "string" + ? [input] + : []; + return entries + .flatMap((entry) => (typeof entry === "string" ? entry.split(",") : [])) + .map((status) => status.trim()) + .filter(Boolean); +} + +export interface IssueFilters { + attention?: "blocked"; + status?: string | readonly string[]; + /** + * Filter by assignee agent ID. + * - `string` (UUID): match issues assigned to that agent. + * - `null`: match unassigned issues (IS NULL). + * - The literal string `"null"` is also accepted as a sentinel for `null` + * so that query-string callers can pass `?assigneeAgentId=null` directly. + * The route layer normalises it before calling the service, but the service + * also normalises it for direct callers. + */ + assigneeAgentId?: string | null; + participantAgentId?: string; + issueIds?: string[]; + listProjection?: "index" | "rows"; + parentIds?: string[]; + oneChildPerParent?: boolean; + assigneeUserId?: string; + touchedByUserId?: string; + inboxArchivedByUserId?: string; + unreadForUserId?: string; + projectId?: string; + workspaceId?: string; + executionWorkspaceId?: string; + parentId?: string; + descendantOf?: string; + createdFromIssueId?: string; + labelId?: string; + originKind?: string; + originKindPrefix?: string; + originId?: string; + includeAgentCreatedTasks?: boolean; + /** Authenticated inbox discovery can retain existing participation. */ + creationVisibilityUserId?: string; + includeRoutineExecutions?: boolean; + excludeRoutineExecutions?: boolean; + includePluginOperations?: boolean; + includeBlockedBy?: boolean; + includeBlockedInboxAttention?: boolean; + includeLiveDescendantSummary?: boolean; + hasPlanDocument?: boolean; + lowTrustBoundary?: LowTrustBoundary & { companyId: string }; + readCondition?: SQL; + q?: string; + limit?: number; + offset?: number; + sortField?: "updated" | "id"; + afterId?: string; + sortDir?: "asc" | "desc"; + /** ISO 8601 timestamp — only return issues with updatedAt strictly after this value. */ + updatedSince?: string; +} + +type IssueRow = typeof issues.$inferSelect & { externalConversationState?: "active" | "waiting" | null }; +type IssueLabelRow = typeof labels.$inferSelect; +type IssuePlanDecompositionRow = typeof issuePlanDecompositions.$inferSelect; +type IssueActiveRunRow = { + execution?: ExecutionProjection; + id: string; + status: string; + agentId: string; + invocationSource: string; + triggerDetail: string | null; + startedAt: Date | null; + finishedAt: Date | null; + createdAt: Date; +}; +type IssueScheduledRetryRow = { + runId: string; + status: "scheduled_retry" | "queued" | "running" | "cancelled"; + agentId: string; + agentName: string | null; + retryOfRunId: string | null; + scheduledRetryAt: Date | null; + scheduledRetryAttempt: number; + scheduledRetryReason: string | null; + retryExhaustedReason?: string | null; + error?: string | null; + errorCode?: string | null; +}; +type IssueWithLabels = IssueRow & { + labels: IssueLabelRow[]; + labelIds: string[]; + watchdog?: IssueWatchdogSummary | null; +}; +type IssueWithLabelsAndRun = IssueWithLabels & { + activeRun: IssueActiveRunRow | null; +}; +type IssueUserCommentStats = { + issueId: string; + isUnreadForMe: boolean; + myLastCommentAt: Date | null; + lastExternalCommentAt: Date | null; +}; +type IssueReadStat = { + issueId: string; + myLastReadAt: Date | null; +}; +type IssueLastActivityStat = { + issueId: string; + latestCommentAt: Date | null; + latestLogAt: Date | null; +}; + +function serializeAcceptedPlanDecomposition( + decomposition: IssuePlanDecompositionRow, +): AcceptedPlanDecomposition { + return { + id: decomposition.id, + companyId: decomposition.companyId, + sourceIssueId: decomposition.sourceIssueId, + acceptedPlanRevisionId: decomposition.acceptedPlanRevisionId, + acceptedInteractionId: decomposition.acceptedInteractionId, + status: decomposition.status as AcceptedPlanDecomposition["status"], + requestFingerprint: decomposition.requestFingerprint, + // Intentionally omit requestedChildren here; the API only needs stable counts + // and child ids, while the durable table keeps the full child draft payload. + requestedChildCount: decomposition.requestedChildCount, + childIssueIds: normalizeIssuePlanDecompositionChildIds( + decomposition.childIssueIds, + ), + ownerAgentId: decomposition.ownerAgentId, + ownerUserId: decomposition.ownerUserId, + ownerRunId: decomposition.ownerRunId, + completedAt: decomposition.completedAt, + createdAt: decomposition.createdAt, + updatedAt: decomposition.updatedAt, + }; +} +type IssueUserContextInput = { + createdByAgentId?: string | null; + originKind?: string | null; + createdByUserId: string | null; + assigneeUserId: string | null; + createdAt: Date | string; + updatedAt: Date | string; +}; +type ProjectGoalReader = Pick; +type DbReader = Pick; +/** Conversation containers cannot acquire new child edges, even with the experiment disabled. */ +async function assertExecutionTaskParent(db: Db, companyId: string, parentId?: string | null) { + if (!parentId) return; + const [parent] = await db.select({ conversationAgentId: issues.conversationAgentId }) + .from(issues).where(and(eq(issues.id, parentId), eq(issues.companyId, companyId))); + if (parent?.conversationAgentId) throw unprocessable("Conversations cannot have new subtasks; create a task in a project instead"); +} + +type DbTransaction = Parameters[0]>[0]; +type IssueCreateInput = Omit & { + workspaceSelection?: TaskWorkspaceSelection; + workspaceSelectionSource?: "explicit" | "channel"; + workspaceSelectionActor?: AuthorizationActor; + title?: string; + initialPlan?: string | null; + /** Internal, server-derived provenance for an accepted in-app chat suggestion. */ + conversationHandoff?: { conversationId: string; sessionGeneration: number }; + labelIds?: string[]; + blockedByIssueIds?: string[]; + inheritExecutionWorkspaceFromIssueId?: string | null; + skipExecutionWorkspaceInheritance?: boolean; + watchdog?: { agentId: string; instructions?: string | null } | null; + watchdogActorRunId?: string | null; + actorRunId?: string | null; + actorResponsibleUserId?: string | null; + trustExplicitResponsibleUserId?: boolean; + idempotencyKey?: string | null; + allowDuplicate?: boolean; + assertCanReuseIssue?: (issue: typeof issues.$inferSelect) => Promise; + onDeduplicated?: (reason: "idempotency_key" | "recent_open_title") => void; +}; +type IssueChildCreateInput = IssueCreateInput & { + acceptanceCriteria?: string[]; + blockParentUntilDone?: boolean; + executionWorkspaceInheritanceMode?: "linkage" | "strategy_only"; + actorAgentId?: string | null; + actorUserId?: string | null; +}; +type AcceptedPlanDecompositionInput = { + acceptedPlanRevisionId: string; + children: IssueChildCreateInput[]; + actorAgentId?: string | null; + actorUserId?: string | null; + actorRunId?: string | null; +}; +type AcceptedPlanDocumentInteraction = { + id: string; +}; +type IssueRelationSummaryMap = { + blockedBy: IssueRelationIssueSummary[]; + blocks: IssueRelationIssueSummary[]; +}; +type IssueBlockerDiagnosticsIssueRow = { + id: string; + companyId: string; + projectId: string | null; + parentId: string | null; + identifier: string | null; + title: string; + status: (typeof ALL_ISSUE_STATUSES)[number]; + priority: string; + assigneeAgentId: string | null; + assigneeUserId: string | null; +}; +type IssueWakeDiagnosticsWakeRequestRow = { + agentId: string; + source: string; + reason: string | null; + status: string; + coalescedCount: number; + runId: string | null; + requestedAt: Date; + claimedAt: Date | null; + finishedAt: Date | null; + error: string | null; +}; +type IssueWakeDiagnosticsActivityRow = { + action: string; + entityType: string; + entityId: string; + agentId: string | null; + runId: string | null; + details: Record | null; + createdAt: Date; +}; +type IssueSubtreeDiagnosticsIssueRow = IssueBlockerDiagnosticsIssueRow & { + depth: number; + createdAt: Date; + updatedAt: Date; +}; +type IssueSubtreeDiagnosticsBlockerRow = IssueBlockerDiagnosticsIssueRow & { + blockedIssueId: string; + relationCreatedAt: Date; +}; +type IssueSubtreeDiagnosticsWakeRequestRow = + IssueWakeDiagnosticsWakeRequestRow & { + issueId: string; + }; +type IssueSubtreeDiagnosticsActivityRow = IssueWakeDiagnosticsActivityRow & { + issueId: string; +}; +type IssueSubtreeDiagnosticsBlockerResultRow = + IssueSubtreeDiagnosticsBlockerRow & { + rowNumber: number | string; + }; +type IssueSubtreeDiagnosticsWakeRequestResultRow = + IssueSubtreeDiagnosticsWakeRequestRow & { + rowNumber: number | string; + }; +type IssueSubtreeDiagnosticsActivityResultRow = + IssueSubtreeDiagnosticsActivityRow & { + rowNumber: number | string; + }; +export type IssueDependencyReadiness = { + issueId: string; + blockerIssueIds: string[]; + unresolvedBlockerIssueIds: string[]; + unresolvedBlockerCount: number; + /** Blockers whose status is `done` but whose execution workspace has not yet finalized. */ + pendingFinalizeBlockerIssueIds: string[]; + allBlockersDone: boolean; + isDependencyReady: boolean; +}; +export type ChildIssueCompletionSummary = { + id: string; + identifier: string | null; + title: string; + status: string; + priority: string; + assigneeAgentId: string | null; + assigneeUserId: string | null; + updatedAt: Date; + summary: string | null; +}; + +function sameRunLock(checkoutRunId: string | null, actorRunId: string | null) { + if (actorRunId) return checkoutRunId === actorRunId; + return checkoutRunId == null; +} + +export const TERMINAL_HEARTBEAT_RUN_STATUSES = new Set([ + "succeeded", + "interrupted", + "failed", + "cancelled", + "timed_out", +]); +const ISSUE_LIST_DESCRIPTION_MAX_CHARS = 1200; +const ISSUE_LIST_DESCRIPTION_MAX_BYTES = ISSUE_LIST_DESCRIPTION_MAX_CHARS * 4; + +function escapeLikePattern(value: string): string { + return value.replace(/[\\%_]/g, "\\$&"); +} + +export function clampIssueListLimit(limit: number): number { + return Math.min(ISSUE_LIST_MAX_LIMIT, Math.max(1, Math.floor(limit))); +} + +function chunkList(values: T[], size: number): T[][] { + const chunks: T[][] = []; + for (let index = 0; index < values.length; index += size) { + chunks.push(values.slice(index, index + size)); + } + return chunks; +} + +function truncateInlineSummary( + value: string | null | undefined, + maxChars = CHILD_COMPLETION_SUMMARY_BODY_MAX_CHARS, +) { + const normalized = value?.trim(); + if (!normalized) return null; + return normalized.length > maxChars + ? `${normalized.slice(0, Math.max(0, maxChars - 15)).trimEnd()} [truncated]` + : normalized; +} + +function truncateByCodePoint(value: string, maxChars: number): string { + if (value.length <= maxChars) return value; + return Array.from(value).slice(0, maxChars).join(""); +} + +function decodeDatabaseTextPreview( + value: string | null | undefined, + maxChars: number, +): string | null { + if (value == null) return null; + return truncateByCodePoint( + Buffer.from(value, "base64").toString("utf8"), + maxChars, + ); +} + +function appendAcceptanceCriteriaToDescription( + description: string | null | undefined, + acceptanceCriteria: string[] | undefined, +) { + const criteria = (acceptanceCriteria ?? []) + .map((item) => item.trim()) + .filter(Boolean); + if (criteria.length === 0) return description ?? null; + const base = description?.trim() ?? ""; + const criteriaMarkdown = [ + "## Acceptance Criteria", + "", + ...criteria.map((item) => `- ${item}`), + ].join("\n"); + return base ? `${base}\n\n${criteriaMarkdown}` : criteriaMarkdown; +} + +function normalizeAcceptedPlanDecompositionFingerprintValue( + value: unknown, +): unknown { + if (value === undefined) return null; + if ( + value == null || + typeof value === "string" || + typeof value === "number" || + typeof value === "boolean" + ) { + return value; + } + if (value instanceof Date) return value.toISOString(); + if (Array.isArray(value)) { + return value.map((item) => + normalizeAcceptedPlanDecompositionFingerprintValue(item), + ); + } + if (typeof value === "object") { + const record = value as Record; + return Object.fromEntries( + Object.keys(record) + .sort() + .map((key) => [ + key, + normalizeAcceptedPlanDecompositionFingerprintValue(record[key]), + ]), + ); + } + return String(value); +} + +const ACCEPTED_PLAN_DECOMPOSITION_FINGERPRINT_CHILD_METADATA_KEYS = new Set([ + "id", + "companyId", + "parentId", + "identifier", + "checkoutRunId", + "executionRunId", + "executionLockedAt", + "startedAt", + "completedAt", + "cancelledAt", + "hiddenAt", + "createdAt", + "updatedAt", + "createdByAgentId", + "createdByUserId", + "updatedByAgentId", + "updatedByUserId", + "actorAgentId", + "actorUserId", + "executionWorkspaceInheritanceMode", + "skipExecutionWorkspaceInheritance", +]); + +function normalizeAcceptedPlanDecompositionFingerprintChild( + child: IssueChildCreateInput, +) { + return Object.fromEntries( + Object.entries(child).filter( + ([key]) => + !ACCEPTED_PLAN_DECOMPOSITION_FINGERPRINT_CHILD_METADATA_KEYS.has(key), + ), + ); +} + +function createAcceptedPlanDecompositionRequestFingerprint(input: { + acceptedPlanRevisionId: string; + children: IssueChildCreateInput[]; +}) { + const canonical = JSON.stringify( + normalizeAcceptedPlanDecompositionFingerprintValue({ + acceptedPlanRevisionId: input.acceptedPlanRevisionId, + children: input.children.map( + normalizeAcceptedPlanDecompositionFingerprintChild, + ), + }), + ); + return createHash("sha256").update(canonical).digest("hex"); +} + +function normalizeIssuePlanDecompositionChildIds(value: unknown): string[] { + if (!Array.isArray(value)) return []; + return value.filter( + (item): item is string => typeof item === "string" && item.length > 0, + ); +} + +export function readAcceptedPlanConfirmationTarget( + payload: unknown, + fallbackIssueId?: string, +): { + revisionId: string; + key: string; + issueId: string; +} | null { + if (!payload || typeof payload !== "object" || Array.isArray(payload)) + return null; + const target = (payload as Record).target; + if (!target || typeof target !== "object" || Array.isArray(target)) + return null; + const record = target as Record; + if (record.type !== "issue_document") return null; + const revisionId = readStringFromRecord(record, "revisionId"); + const key = readStringFromRecord(record, "key"); + const issueId = readStringFromRecord(record, "issueId") ?? fallbackIssueId; + if (!revisionId || !key || !issueId) return null; + return { revisionId, key, issueId }; +} + +async function resolveAcceptedPlanClaimOwner(input: { + dbOrTx: Pick; + claim: Pick< + typeof issuePlanDecompositions.$inferSelect, + "ownerAgentId" | "ownerUserId" | "ownerRunId" + >; + actorAgentId?: string | null; + actorUserId?: string | null; + actorRunId?: string | null; +}) { + const nextOwner = { + ownerAgentId: input.actorAgentId ?? null, + ownerUserId: input.actorUserId ?? null, + ownerRunId: input.actorRunId ?? null, + }; + if ( + input.claim.ownerAgentId === nextOwner.ownerAgentId && + input.claim.ownerUserId === nextOwner.ownerUserId && + input.claim.ownerRunId === nextOwner.ownerRunId + ) { + return nextOwner; + } + + if (!input.claim.ownerRunId) { + return nextOwner; + } + + const existingOwnerRun = await input.dbOrTx + .select({ status: heartbeatRuns.status }) + .from(heartbeatRuns) + .where(eq(heartbeatRuns.id, input.claim.ownerRunId)) + .then((rows) => rows[0] ?? null); + if ( + existingOwnerRun && + !TERMINAL_HEARTBEAT_RUN_STATUSES.has(existingOwnerRun.status) + ) { + return { + ownerAgentId: input.claim.ownerAgentId, + ownerUserId: input.claim.ownerUserId, + ownerRunId: input.claim.ownerRunId, + }; + } + + return nextOwner; +} + +async function findAcceptedPlanDocumentInteraction( + dbOrTx: Pick, + input: { + companyId: string; + sourceIssueId: string; + acceptedPlanRevisionId: string; + }, +): Promise { + const rows = await dbOrTx + .select({ + id: issueThreadInteractions.id, + payload: issueThreadInteractions.payload, + }) + .from(issueThreadInteractions) + .where( + and( + eq(issueThreadInteractions.companyId, input.companyId), + eq(issueThreadInteractions.issueId, input.sourceIssueId), + eq(issueThreadInteractions.kind, "request_confirmation"), + eq(issueThreadInteractions.status, "accepted"), + ), + ) + .orderBy( + desc(issueThreadInteractions.resolvedAt), + desc(issueThreadInteractions.createdAt), + ); + + for (const row of rows) { + const target = readAcceptedPlanConfirmationTarget( + row.payload, + input.sourceIssueId, + ); + if ( + target?.issueId === input.sourceIssueId && + target.key === "plan" && + target.revisionId === input.acceptedPlanRevisionId + ) { + return { id: row.id }; + } + } + return null; +} + +function createIssueDependencyReadiness( + issueId: string, +): IssueDependencyReadiness { + return { + issueId, + blockerIssueIds: [], + unresolvedBlockerIssueIds: [], + unresolvedBlockerCount: 0, + pendingFinalizeBlockerIssueIds: [], + allBlockersDone: true, + isDependencyReady: true, + }; +} + +/** + * Returns the set of execution-workspace ids whose most recent workspace operation + * is NOT a successful `workspace_finalize`. These workspaces have either an in-flight + * run, a failed finalize, or never reached the finalize barrier — dependents that + * read this workspace must wait until finalize succeeds. + * + * Workspaces with no recorded operations are considered finalized (nothing has + * touched them since they were realized). + */ +export async function listUnfinalizedExecutionWorkspaceIds( + dbOrTx: Pick, + companyId: string, + executionWorkspaceIds: string[], +): Promise> { + const unfinalized = new Set(); + if (executionWorkspaceIds.length === 0) return unfinalized; + + // Pull every workspace op for the candidate workspaces and pick the latest per + // workspace in memory. Per-workspace LATERAL queries would be tighter, but the + // candidate set is tiny in practice (one workspace per blocker per readiness call). + const rows = await dbOrTx + .select({ + executionWorkspaceId: workspaceOperations.executionWorkspaceId, + phase: workspaceOperations.phase, + status: workspaceOperations.status, + startedAt: workspaceOperations.startedAt, + }) + .from(workspaceOperations) + .where( + and( + eq(workspaceOperations.companyId, companyId), + inArray( + workspaceOperations.executionWorkspaceId, + executionWorkspaceIds, + ), + ), + ); + + const latestByWorkspace = new Map< + string, + { phase: string; status: string; startedAt: Date } + >(); + for (const row of rows) { + if (!row.executionWorkspaceId) continue; + const current = latestByWorkspace.get(row.executionWorkspaceId); + if (!current || row.startedAt > current.startedAt) { + latestByWorkspace.set(row.executionWorkspaceId, { + phase: row.phase, + status: row.status, + startedAt: row.startedAt, + }); + } + } + + for (const workspaceId of executionWorkspaceIds) { + const latest = latestByWorkspace.get(workspaceId); + if (!latest) continue; // no ops recorded → treat as finalized + if (latest.phase === "workspace_finalize" && latest.status === "succeeded") + continue; + unfinalized.add(workspaceId); + } + + return unfinalized; +} + +async function listPendingFinalizeBlockerIssueIds( + dbOrTx: Pick, + companyId: string, + blockerWorkspacePairs: Array<{ + blockerIssueId: string; + executionWorkspaceId: string; + }>, + lockSources = false, +): Promise> { + const pending = new Set(); + const blockerIssueIds = [ + ...new Set(blockerWorkspacePairs.map((pair) => pair.blockerIssueId)), + ]; + const executionWorkspaceIds = [ + ...new Set(blockerWorkspacePairs.map((pair) => pair.executionWorkspaceId)), + ]; + if (blockerIssueIds.length === 0 || executionWorkspaceIds.length === 0) + return pending; + const blockerWorkspaceKeys = new Set( + blockerWorkspacePairs.map( + (pair) => `${pair.blockerIssueId}:${pair.executionWorkspaceId}`, + ), + ); + + if (lockSources) { + + + await dbOrTx.select({ id: executionWorkspaces.id }).from(executionWorkspaces) + .where(and(eq(executionWorkspaces.companyId, companyId), inArray(executionWorkspaces.id, executionWorkspaceIds))) + .for("update", { noWait: true }); + } + const query = dbOrTx + .select({ + issueId: workspaceOperations.issueId, + executionWorkspaceId: workspaceOperations.executionWorkspaceId, + phase: workspaceOperations.phase, + status: workspaceOperations.status, + startedAt: workspaceOperations.startedAt, + }) + .from(workspaceOperations) + .where( + and( + eq(workspaceOperations.companyId, companyId), + inArray( + workspaceOperations.executionWorkspaceId, + executionWorkspaceIds, + ), + ), + ); + + const rows = await (lockSources ? query.for("share", { noWait: true }) : query); + + const latestAttributedByBlockerWorkspace = new Map< + string, + { phase: string; status: string; startedAt: Date } + >(); + const latestUnattributedByWorkspace = new Map< + string, + { phase: string; status: string; startedAt: Date } + >(); + const latestSuccessfulFinalizeByWorkspace = new Map(); + for (const row of rows) { + if (!row.executionWorkspaceId) continue; + if (row.phase === "workspace_finalize" && row.status === "succeeded") { + const current = latestSuccessfulFinalizeByWorkspace.get( + row.executionWorkspaceId, + ); + if (!current || row.startedAt > current) { + latestSuccessfulFinalizeByWorkspace.set( + row.executionWorkspaceId, + row.startedAt, + ); + } + } + if (row.issueId) { + const key = `${row.issueId}:${row.executionWorkspaceId}`; + if (!blockerWorkspaceKeys.has(key)) continue; + const current = latestAttributedByBlockerWorkspace.get(key); + if (!current || row.startedAt > current.startedAt) { + latestAttributedByBlockerWorkspace.set(key, { + phase: row.phase, + status: row.status, + startedAt: row.startedAt, + }); + } + continue; + } + + const current = latestUnattributedByWorkspace.get(row.executionWorkspaceId); + if (!current || row.startedAt > current.startedAt) { + latestUnattributedByWorkspace.set(row.executionWorkspaceId, { + phase: row.phase, + status: row.status, + startedAt: row.startedAt, + }); + } + } + + for (const pair of blockerWorkspacePairs) { + const latest = + latestAttributedByBlockerWorkspace.get( + `${pair.blockerIssueId}:${pair.executionWorkspaceId}`, + ) ?? latestUnattributedByWorkspace.get(pair.executionWorkspaceId); + if (!latest) continue; // no ops recorded -> nothing to finalize for this blocker + if (latest.phase === "workspace_finalize" && latest.status === "succeeded") + continue; + const laterSuccessfulFinalize = latestSuccessfulFinalizeByWorkspace.get( + pair.executionWorkspaceId, + ); + if (laterSuccessfulFinalize && laterSuccessfulFinalize > latest.startedAt) + continue; + pending.add(pair.blockerIssueId); + } + + return pending; +} + +/** + * Whether a heartbeat run has reached a terminal state or no longer exists. + * A terminal/missing run can make no further progress on its execution + * workspace, so callers must not wait on it to advance an in-flight operation. + */ +export async function heartbeatRunIsTerminalOrMissing( + dbOrTx: Pick, + runId: string, +): Promise { + const run = await dbOrTx + .select({ status: heartbeatRuns.status }) + .from(heartbeatRuns) + .where(eq(heartbeatRuns.id, runId)) + .then((rows: Array<{ status: string }>) => rows[0] ?? null); + if (!run) return true; + return TERMINAL_HEARTBEAT_RUN_STATUSES.has(run.status); +} + +/** + * Returns whether a specific run's sync-back on a specific execution workspace + * has settled — i.e. the accept/review gates that guard against a still-in-flight + * worktree sync no longer need to block on this run. + * + * Semantics: + * - No operations recorded → settled. The run never touched the workspace state + * the gates protect. + * - Earlier phases recorded but no `workspace_finalize` yet → NOT settled. The + * sync-back hasn't been attempted; the gate should wait for it. + * - Latest `workspace_finalize` reached a terminal status (`succeeded`, `failed`, + * or `skipped`) → settled. A finalize that ran and finished is done even if it + * failed: it will not retry within this run, so continuing to block would wedge + * the gate forever — a failed sync-back must not permanently block a + * confirmation accept behind a misleading "still syncing" error. + * - Latest `workspace_finalize` is still `running` → in flight, so NOT settled — + * unless the owning run has itself ended, in which case the `running` record is + * stale (the process died mid-finalize) and we treat it as settled rather than + * wait on a run that can never make progress. + */ +export async function runWorkspaceIsFinalized( + dbOrTx: Pick, + companyId: string, + executionWorkspaceId: string, + runId: string, +): Promise { + const rows = await dbOrTx + .select({ + phase: workspaceOperations.phase, + status: workspaceOperations.status, + startedAt: workspaceOperations.startedAt, + }) + .from(workspaceOperations) + .where( + and( + eq(workspaceOperations.companyId, companyId), + eq(workspaceOperations.executionWorkspaceId, executionWorkspaceId), + eq(workspaceOperations.heartbeatRunId, runId), + ), + ); + + if (rows.length === 0) return true; + + let latestFinalize: { status: string; startedAt: Date } | null = null; + for (const row of rows) { + if (row.phase !== "workspace_finalize") continue; + if (!latestFinalize || row.startedAt > latestFinalize.startedAt) + latestFinalize = row; + } + + // The run touched the workspace but hasn't reached the sync-back phase yet. + if (!latestFinalize) return false; + + // A finalize that reached any terminal status is settled — including `failed` + // and `skipped`. It will not retry within this run, so gates must stop waiting. + if (latestFinalize.status !== "running") return true; + + // Finalize is still marked `running`. It is only genuinely in flight while the + // owning run is alive; a `running` record left behind by an ended run is stale. + return heartbeatRunIsTerminalOrMissing(dbOrTx, runId); +} + +async function listIssueDependencyReadinessMap( + dbOrTx: Pick, + companyId: string, + issueIds: string[], + lockSources = false, +) { + const uniqueIssueIds = [...new Set(issueIds.filter(Boolean))]; + const readinessMap = new Map(); + for (const issueId of uniqueIssueIds) { + readinessMap.set(issueId, createIssueDependencyReadiness(issueId)); + } + if (uniqueIssueIds.length === 0) return readinessMap; + + const blockerQuery = dbOrTx + .select({ + issueId: issueRelations.relatedIssueId, + blockerIssueId: issueRelations.issueId, + blockerStatus: issues.status, + blockerExecutionWorkspaceId: issues.executionWorkspaceId, + }) + .from(issueRelations) + .innerJoin(issues, eq(issueRelations.issueId, issues.id)) + .where( + and( + eq(issueRelations.companyId, companyId), + eq(issueRelations.type, "blocks"), + inArray(issueRelations.relatedIssueId, uniqueIssueIds), + ), + ); + + + + const blockerRows = await (lockSources ? blockerQuery.for("update", { noWait: true }) : blockerQuery); + + // Collect issue/workspace pairs of "done" blockers — these are the only ones + // subject to the workspace-finalize barrier. Blockers that aren't done already + // mark the dependent as not-ready and don't need a finalize check. + const doneBlockerWorkspacePairs: Array<{ + blockerIssueId: string; + executionWorkspaceId: string; + }> = []; + for (const row of blockerRows) { + if (row.blockerStatus === "done" && row.blockerExecutionWorkspaceId) { + doneBlockerWorkspacePairs.push({ + blockerIssueId: row.blockerIssueId, + executionWorkspaceId: row.blockerExecutionWorkspaceId, + }); + } + } + const pendingFinalizeBlockerIssueIds = + await listPendingFinalizeBlockerIssueIds( + dbOrTx, + companyId, + doneBlockerWorkspacePairs, + lockSources, + ); + + for (const row of blockerRows) { + const current = + readinessMap.get(row.issueId) ?? + createIssueDependencyReadiness(row.issueId); + current.blockerIssueIds.push(row.blockerIssueId); + // Only done blockers resolve dependents; cancelled blockers stay unresolved + // until an operator removes or replaces the blocker relationship explicitly. + if (row.blockerStatus !== "done") { + current.unresolvedBlockerIssueIds.push(row.blockerIssueId); + current.unresolvedBlockerCount += 1; + current.allBlockersDone = false; + current.isDependencyReady = false; + } else if ( + row.blockerExecutionWorkspaceId && + pendingFinalizeBlockerIssueIds.has(row.blockerIssueId) + ) { + // Workspace-finalize barrier: the blocker's most recent run on its + // execution workspace hasn't recorded a successful workspace_finalize. + // Treat the dependent as not-ready until sync-back lands (or the run + // finalizes); a subsequent finalize wake will re-evaluate readiness. + // `allBlockersDone` is cleared too so that callers using it as a + // proxy for "this dependent can proceed" still see the gate. + current.unresolvedBlockerIssueIds.push(row.blockerIssueId); + current.unresolvedBlockerCount += 1; + current.pendingFinalizeBlockerIssueIds.push(row.blockerIssueId); + current.allBlockersDone = false; + current.isDependencyReady = false; + } + readinessMap.set(row.issueId, current); + } + + return readinessMap; +} + +async function listUnresolvedBlockerDetails( + dbOrTx: Pick, + companyId: string, + unresolvedBlockerIssueIds: string[], + pendingFinalizeBlockerIssueIds: string[] = [], +) { + if (unresolvedBlockerIssueIds.length === 0) return []; + const pendingFinalizeIds = new Set(pendingFinalizeBlockerIssueIds); + const rows = await dbOrTx + .select({ + issueId: issues.id, + identifier: issues.identifier, + title: issues.title, + }) + .from(issues) + .where( + and( + eq(issues.companyId, companyId), + inArray(issues.id, unresolvedBlockerIssueIds), + ), + ); + const rowsById = new Map(rows.map((row) => [row.issueId, row])); + return unresolvedBlockerIssueIds.map((issueId) => { + const row = rowsById.get(issueId); + return { + issueId, + identifier: row?.identifier ?? null, + title: row?.title ?? null, + reason: pendingFinalizeIds.has(issueId) + ? ("pending_finalize" as const) + : ("not_done" as const), + }; + }); +} + +async function listUnresolvedBlockerIssueIds( + dbOrTx: Pick, + companyId: string, + blockerIssueIds: string[], +) { + const uniqueBlockerIssueIds = [...new Set(blockerIssueIds.filter(Boolean))]; + if (uniqueBlockerIssueIds.length === 0) return []; + return dbOrTx + .select({ id: issues.id }) + .from(issues) + .where( + and( + eq(issues.companyId, companyId), + inArray(issues.id, uniqueBlockerIssueIds), + // Cancelled blockers intentionally remain unresolved until the relation changes. + ne(issues.status, "done"), + ), + ) + .then((rows) => rows.map((row) => row.id)); +} +async function getProjectDefaultGoalId( + db: ProjectGoalReader, + companyId: string, + projectId: string | null | undefined, +) { + if (!projectId) return null; + const row = await db + .select({ goalId: projects.goalId }) + .from(projects) + .where(and(eq(projects.id, projectId), eq(projects.companyId, companyId))) + .then((rows) => rows[0] ?? null); + return row?.goalId ?? null; +} + +async function getWorkspaceInheritanceIssue( + db: DbReader, + companyId: string, + issueId: string, +) { + const issue = await db + .select({ + id: issues.id, + projectId: issues.projectId, + projectWorkspaceId: issues.projectWorkspaceId, + executionWorkspaceId: issues.executionWorkspaceId, + executionWorkspaceSettings: issues.executionWorkspaceSettings, + }) + .from(issues) + .where(and(eq(issues.id, issueId), eq(issues.companyId, companyId))) + .then((rows) => rows[0] ?? null); + if (!issue) { + throw notFound("Workspace inheritance issue not found"); + } + return issue; +} + +function participatedByAgentCondition(companyId: string, agentId: string) { + return sql` + ( + ${issues.createdByAgentId} = ${agentId} + OR ${issues.assigneeAgentId} = ${agentId} + OR EXISTS ( + SELECT 1 + FROM ${issueComments} + WHERE ${issueComments.issueId} = ${issues.id} + AND ${issueComments.companyId} = ${companyId} + AND ${issueComments.authorAgentId} = ${agentId} + ) + OR EXISTS ( + SELECT 1 + FROM ${activityLog} + WHERE ${activityLog.companyId} = ${companyId} + AND ${activityLog.entityType} = 'issue' + AND ${activityLog.entityId} = ${issues.id}::text + AND ${activityLog.agentId} = ${agentId} + ) + ) + `; +} + +function myLastCommentAtExpr(companyId: string, userId: string) { + return sql` + ( + SELECT MAX(${issueComments.createdAt}) + FROM ${issueComments} + WHERE ${issueComments.issueId} = ${issues.id} + AND ${issueComments.companyId} = ${companyId} + AND ${issueComments.authorUserId} = ${userId} + AND ${directHumanCommentCondition()} + ) + `; +} + +function myLastReadAtExpr(companyId: string, userId: string) { + return sql` + ( + SELECT MAX(${issueReadStates.lastReadAt}) + FROM ${issueReadStates} + WHERE ${issueReadStates.issueId} = ${issues.id} + AND ${issueReadStates.companyId} = ${companyId} + AND ${issueReadStates.userId} = ${userId} + ) + `; +} + +function myLastTouchAtExpr(companyId: string, userId: string) { + const myLastCommentAt = myLastCommentAtExpr(companyId, userId); + const myLastReadAt = myLastReadAtExpr(companyId, userId); + return sql` + GREATEST( + COALESCE(${myLastCommentAt}, to_timestamp(0)), + COALESCE(${myLastReadAt}, to_timestamp(0)), + COALESCE(CASE WHEN ${issues.createdByUserId} = ${userId} AND ${humanCreatedIssueCondition()} THEN ${issues.createdAt} ELSE NULL END, to_timestamp(0)), + COALESCE(CASE WHEN ${issues.assigneeUserId} = ${userId} THEN ${issues.updatedAt} ELSE NULL END, to_timestamp(0)) + ) + `; +} + +const ISSUE_LOCAL_INBOX_ACTIVITY_ACTIONS = [ + "issue.read_marked", + "issue.read_unmarked", + "issue.inbox_archived", + "issue.inbox_unarchived", +] as const; + +function issueLatestCommentAtExpr(companyId: string) { + return sql` + ( + SELECT MAX(${issueComments.createdAt}) + FROM ${issueComments} + WHERE ${issueComments.issueId} = ${issues.id} + AND ${issueComments.companyId} = ${companyId} + ) + `; +} + +function issueLatestLogAtExpr(companyId: string) { + return sql` + ( + SELECT MAX(${activityLog.createdAt}) + FROM ${activityLog} + WHERE ${activityLog.companyId} = ${companyId} + AND ${activityLog.entityType} = 'issue' + AND ${activityLog.entityId} = ${issues.id}::text + AND NOT ${automaticReadRecoveryCleanupCondition()} + AND ${activityLog.action} NOT IN (${sql.join( + ISSUE_LOCAL_INBOX_ACTIVITY_ACTIONS.map((action) => sql`${action}`), + sql`, `, + )}) + ) + `; +} + +function issueCanonicalLastActivityAtExpr(companyId: string) { + const latestCommentAt = issueLatestCommentAtExpr(companyId); + const latestLogAt = issueLatestLogAtExpr(companyId); + return sql` + GREATEST( + ${issues.updatedAt}, + COALESCE(${latestCommentAt}, to_timestamp(0)), + COALESCE(${latestLogAt}, to_timestamp(0)) + ) + `; +} + +function unreadForUserCondition(companyId: string, userId: string) { + const touchedCondition = touchedByUserCondition(companyId, userId); + const myLastTouchAt = myLastTouchAtExpr(companyId, userId); + const myLastRequestReadAt = sql`GREATEST( + COALESCE(${myLastCommentAtExpr(companyId, userId)}, to_timestamp(0)), + COALESCE(${myLastReadAtExpr(companyId, userId)}, to_timestamp(0)) + )`; + return sql` + ( + ${touchedCondition} + AND (EXISTS ( + SELECT 1 + FROM ${issueComments} + WHERE ${issueComments.issueId} = ${issues.id} + AND ${issueComments.companyId} = ${companyId} + AND ( + ${issueComments.authorUserId} IS NULL + OR ${issueComments.authorUserId} <> ${userId} + ) + AND ${issueComments.createdAt} > ${myLastTouchAt} + ) OR ${pendingUserRequestCondition(companyId, userId, myLastRequestReadAt)}) + ) + `; +} + +function inboxVisibleForUserCondition(companyId: string, userId: string) { + return sql` + NOT EXISTS ( + SELECT 1 + FROM ${issueInboxArchives} + WHERE ${issueInboxArchives.issueId} = ${issues.id} + AND ${issueInboxArchives.companyId} = ${companyId} + AND ${issueInboxArchives.userId} = ${userId} + AND NOT ( + EXISTS ( + SELECT 1 + FROM ${issueThreadInteractions} + WHERE ${issueThreadInteractions.issueId} = ${issues.id} + AND ${issueThreadInteractions.companyId} = ${companyId} + AND ${issueThreadInteractions.kind} IN ( + 'suggest_tasks', + 'ask_user_questions', + 'request_confirmation' + ) + AND ${issueThreadInteractions.createdAt} > ${issueInboxArchives.archivedAt} + ) + OR ${pendingUserRequestCondition(companyId, userId, sql`${issueInboxArchives.archivedAt}`)} + OR EXISTS ( + SELECT 1 + FROM ${activityLog} + WHERE ${activityLog.companyId} = ${companyId} + AND ${activityLog.entityType} = 'issue' + AND ${activityLog.entityId} = ${issues.id}::text + AND ${activityLog.action} = 'issue.updated' + AND ${activityLog.createdAt} > ${issueInboxArchives.archivedAt} + AND ${activityLog.details}->>'status' IN ('in_review', 'blocked', 'done') + AND ${activityLog.details}->'_previous'->>'status' + IS DISTINCT FROM ${activityLog.details}->>'status' + AND NOT ( + ${activityLog.details}->>'status' = 'done' + AND ${issues.completedAt} IS NOT NULL + AND ${issueInboxArchives.archivedAt} >= ${issues.completedAt} + ) + ) + OR EXISTS ( + SELECT 1 + FROM ${issueComments} + WHERE ${issueComments.issueId} = ${issues.id} + AND ${issueComments.companyId} = ${companyId} + AND ${issueComments.createdAt} > ${issueInboxArchives.archivedAt} + AND ${issueComments.deletedAt} IS NULL + AND ( + ( + ${issueComments.authorUserId} IS NOT NULL + AND ${issueComments.authorUserId} <> ${userId} + AND ${issueComments.authorAgentId} IS NULL + AND ${issueComments.derivedAuthorAgentId} IS NULL + ) + OR POSITION(${`](user://${userId})`} IN ${issueComments.body}) > 0 + ) + ) + ) + ) + `; +} + +const LEGACY_PLUGIN_OPERATION_ORIGIN_KINDS = [ + "plugin:paperclipai.content-machine:case", + "plugin:paperclipai.content-machine:evaluation", + "plugin:paperclipai.content-machine:source-sync", +] as const; + +function nonPluginOperationIssueCondition() { + return sql`NOT ( + ${issues.originKind} LIKE 'plugin:%:operation' + OR ${issues.originKind} LIKE 'plugin:%:operation:%' + OR ${inArray(issues.originKind, LEGACY_PLUGIN_OPERATION_ORIGIN_KINDS)} + )`; +} + +function shouldIncludePluginOperationIssues(filters: IssueFilters | undefined) { + return Boolean( + filters?.includePluginOperations || + filters?.originKind || + filters?.originKindPrefix || + filters?.originId || + filters?.projectId, + ); +} + +export function deriveIssueUserContext( + issue: IssueUserContextInput, + userId: string, + stats: + | { + isUnreadForMe?: boolean; + myLastCommentAt: Date | string | null; + myLastReadAt: Date | string | null; + lastExternalCommentAt: Date | string | null; + } + | null + | undefined, +) { + const normalizeDate = (value: Date | string | null | undefined) => { + if (!value) return null; + if (value instanceof Date) + return Number.isNaN(value.getTime()) ? null : value; + const parsed = new Date(value); + return Number.isNaN(parsed.getTime()) ? null : parsed; + }; + + const myLastCommentAt = normalizeDate(stats?.myLastCommentAt); + const myLastReadAt = normalizeDate(stats?.myLastReadAt); + const createdTouchAt = + issue.createdByUserId === userId && !isAgentCreatedIssue(issue) + ? normalizeDate(issue.createdAt) + : null; + const assignedTouchAt = + issue.assigneeUserId === userId ? normalizeDate(issue.updatedAt) : null; + const myLastTouchAt = + [myLastCommentAt, myLastReadAt, createdTouchAt, assignedTouchAt] + .filter((value): value is Date => value instanceof Date) + .sort((a, b) => b.getTime() - a.getTime())[0] ?? null; + const lastExternalCommentAt = normalizeDate(stats?.lastExternalCommentAt); + const isUnreadForMe = stats?.isUnreadForMe ?? Boolean( + myLastTouchAt && + lastExternalCommentAt && + lastExternalCommentAt.getTime() > myLastTouchAt.getTime(), + ); + + return { + myLastTouchAt, + lastExternalCommentAt, + isUnreadForMe, + }; +} + +function latestIssueActivityAt( + ...values: Array +): Date | null { + const normalized = values + .map((value) => { + if (!value) return null; + if (value instanceof Date) + return Number.isNaN(value.getTime()) ? null : value; + const parsed = new Date(value); + return Number.isNaN(parsed.getTime()) ? null : parsed; + }) + .filter((value): value is Date => value instanceof Date) + .sort((a, b) => b.getTime() - a.getTime()); + return normalized[0] ?? null; +} + +type InboxArchiveAttributionRow = { + issueId: string; + archivedAt: Date; + archivedByActorType: "user" | "agent"; + archivedByAgentId: string | null; + archivedByRunId: string | null; +}; + +async function inboxArchiveRowsForIssues( + dbOrTx: Db, + companyId: string, + userId: string, + issueIds: string[], +): Promise { + if (issueIds.length === 0) return []; + return dbOrTx + .select({ + issueId: issueInboxArchives.issueId, + archivedAt: issueInboxArchives.archivedAt, + archivedByActorType: issueInboxArchives.archivedByActorType, + archivedByAgentId: issueInboxArchives.archivedByAgentId, + archivedByRunId: issueInboxArchives.archivedByRunId, + }) + .from(issueInboxArchives) + .where( + and( + eq(issueInboxArchives.companyId, companyId), + eq(issueInboxArchives.userId, userId), + inArray(issueInboxArchives.issueId, issueIds), + ), + ); +} + +function activeInboxArchiveFields( + archive: InboxArchiveAttributionRow | undefined, + lastActivityAt: Date, +) { + if (!archive || archive.archivedAt.getTime() < lastActivityAt.getTime()) + return {}; + return { + archivedAt: archive.archivedAt, + archivedByActorType: archive.archivedByActorType, + archivedByAgentId: archive.archivedByAgentId, + archivedByRunId: archive.archivedByRunId, + }; +} + +function issueListOrderBy( + companyId: string, + { + hasSearch, + priorityOrder, + searchOrder, + sortField, + sortDir, + }: { + hasSearch: boolean; + priorityOrder: SQL; + searchOrder: SQL; + sortField?: IssueFilters["sortField"]; + sortDir?: IssueFilters["sortDir"]; + }, +) { + if (sortField === "id") return [sortDir === "desc" ? desc(issues.id) : asc(issues.id)]; + const canonicalLastActivityAt = issueCanonicalLastActivityAtExpr(companyId); + if (sortField === "updated") { + const activityOrder = + sortDir === "asc" + ? asc(canonicalLastActivityAt) + : desc(canonicalLastActivityAt); + const updatedOrder = + sortDir === "asc" ? asc(issues.updatedAt) : desc(issues.updatedAt); + const idOrder = sortDir === "asc" ? asc(issues.id) : desc(issues.id); + return hasSearch + ? [asc(searchOrder), activityOrder, updatedOrder, idOrder] + : [activityOrder, updatedOrder, idOrder]; + } + + return [ + hasSearch ? asc(searchOrder) : asc(priorityOrder), + asc(priorityOrder), + desc(canonicalLastActivityAt), + desc(issues.updatedAt), + desc(issues.id), + ]; +} + +async function labelMapForIssues( + dbOrTx: any, + issueIds: string[], +): Promise> { + const map = new Map(); + if (issueIds.length === 0) return map; + for (const issueIdChunk of chunkList( + issueIds, + ISSUE_LIST_RELATED_QUERY_CHUNK_SIZE, + )) { + const rows = await dbOrTx + .select({ + issueId: issueLabels.issueId, + label: labels, + }) + .from(issueLabels) + .innerJoin(labels, eq(issueLabels.labelId, labels.id)) + .where(inArray(issueLabels.issueId, issueIdChunk)) + .orderBy(asc(labels.name), asc(labels.id)); + + for (const row of rows) { + const existing = map.get(row.issueId); + if (existing) existing.push(row.label); + else map.set(row.issueId, [row.label]); + } + } + return map; +} + +async function withIssueLabels( + dbOrTx: any, + rows: IssueRow[], +): Promise { + if (rows.length === 0) return []; + const issueIds = rows.map((row) => row.id); + const [labelsByIssueId, watchdogByIssueId] = await Promise.all([ + labelMapForIssues(dbOrTx, issueIds), + watchdogMapForIssues(dbOrTx, rows), + ]); + return rows.map((row) => { + const issueLabels = labelsByIssueId.get(row.id) ?? []; + return { + ...row, + labels: issueLabels, + labelIds: issueLabels.map((label) => label.id), + watchdog: watchdogByIssueId.get(row.id) ?? null, + }; + }); +} + +async function watchdogMapForIssues( + dbOrTx: any, + rows: IssueRow[], +): Promise> { + const map = new Map(); + if (rows.length === 0) return map; + const byCompany = new Map(); + for (const row of rows) { + const ids = byCompany.get(row.companyId) ?? []; + ids.push(row.id); + byCompany.set(row.companyId, ids); + } + for (const [companyId, issueIds] of byCompany.entries()) { + for (const issueIdChunk of chunkList( + [...new Set(issueIds)], + ISSUE_LIST_RELATED_QUERY_CHUNK_SIZE, + )) { + const watchdogRows = await dbOrTx + .select() + .from(issueWatchdogs) + .where( + and( + eq(issueWatchdogs.companyId, companyId), + inArray(issueWatchdogs.issueId, issueIdChunk), + eq(issueWatchdogs.status, "active"), + ), + ); + for (const row of watchdogRows) { + map.set(row.issueId, summarizeIssueWatchdog(row)); + } + } + } + return map; +} + +const ACTIVE_RUN_STATUSES = ["queued", "running"]; +const BLOCKER_ATTENTION_ACTIVE_RUN_STATUSES = ["queued", "running"]; +const BLOCKER_ATTENTION_ACTIVE_WAKE_STATUSES = [ + "queued", + "deferred_issue_execution", +]; +const BLOCKER_ATTENTION_PENDING_INTERACTION_STATUSES = ["pending"]; +const BLOCKER_ATTENTION_PENDING_APPROVAL_STATUSES = [ + "pending", + "revision_requested", +]; +const BLOCKER_ATTENTION_OPEN_RECOVERY_ORIGIN_KIND = + "harness_liveness_escalation"; +const BLOCKER_ATTENTION_CHILD_TERMINAL_STATUSES = ["done", "cancelled"]; +function lowTrustBoundaryIssueCondition( + companyId: string, + boundary: (LowTrustBoundary & { companyId: string }) | null | undefined, +) { + if (!boundary || boundary.companyId !== companyId) return null; + const clauses: SQL[] = []; + const issueIds = [...new Set(boundary.issueIds ?? [])]; + const projectIds = [...new Set(boundary.projectIds ?? [])]; + if (issueIds.length > 0) clauses.push(inArray(issues.id, issueIds)); + if (projectIds.length > 0) + clauses.push(inArray(issues.projectId, projectIds)); + if (boundary.rootIssueId) { + clauses.push(sql` + ${issues.id} IN ( + WITH RECURSIVE descendants(id) AS ( + SELECT ${issues.id} + FROM ${issues} + WHERE ${issues.companyId} = ${companyId} + AND ${issues.id} = ${boundary.rootIssueId} + UNION + SELECT ${issues.id} + FROM ${issues} + JOIN descendants ON ${issues.parentId} = descendants.id + WHERE ${issues.companyId} = ${companyId} + ) + SELECT id FROM descendants + ) + `); + } + if (clauses.length === 0) return sql`false`; + return or(...clauses); +} + +const BLOCKER_ATTENTION_OPEN_RECOVERY_TERMINAL_STATUSES = ["done", "cancelled"]; +export const BLOCKER_ATTENTION_MAX_DEPTH = 8; +export const BLOCKER_ATTENTION_MAX_NODES = 2000; +const BLOCKER_ATTENTION_INVOKABLE_AGENT_STATUSES = new Set([ + "active", + "idle", + "running", + "error", +]); + +type IssueBlockerAttentionNode = { + id: string; + companyId: string; + parentId: string | null; + identifier: string | null; + title: string; + status: string; + executionRunId?: string | null; + assigneeAgentId: string | null; + assigneeUserId: string | null; +}; +type IssueBlockerAttentionInputNode = Pick< + IssueBlockerAttentionNode, + | "id" + | "companyId" + | "parentId" + | "identifier" + | "title" + | "status" + | "assigneeAgentId" + | "assigneeUserId" +> & { executionRunId?: string | null }; + +type IssueBlockerAttentionEdge = { + issueId: string; + blockerIssueId: string; +}; +type IssueBlockerAttentionQueryRow = IssueBlockerAttentionNode & { + issueId: string | null; + blockerIssueId: string; +}; +type IssueBlockerAttentionActivePathRow = { + issueId: string | null; +}; +type IssueBlockerAttentionAgentRow = { + id: string; + companyId: string; + status: string; +}; + +async function activeRunMapForIssues( + dbOrTx: any, + issueRows: IssueWithLabels[], +): Promise> { + const map = new Map(); + const runIds = issueRows + .map((row) => row.executionRunId) + .filter((id): id is string => id != null); + if (runIds.length === 0) return map; + + for (const runIdChunk of chunkList( + [...new Set(runIds)], + ISSUE_LIST_RELATED_QUERY_CHUNK_SIZE, + )) { + const rows = await dbOrTx + .select({ + id: heartbeatRuns.id, + status: heartbeatRuns.status, + agentId: heartbeatRuns.agentId, + invocationSource: heartbeatRuns.invocationSource, + triggerDetail: heartbeatRuns.triggerDetail, + startedAt: heartbeatRuns.startedAt, + finishedAt: heartbeatRuns.finishedAt, + createdAt: heartbeatRuns.createdAt, + }) + .from(heartbeatRuns) + .where( + and( + inArray(heartbeatRuns.id, runIdChunk), + inArray(heartbeatRuns.status, ACTIVE_RUN_STATUSES), + ), + ); + + for (const row of rows) { + map.set(row.id, row); + } + } + for (const companyId of new Set(issueRows.map(row => row.companyId))) { + const scopedIds = issueRows.filter(row => row.companyId === companyId).flatMap(row => row.executionRunId && map.has(row.executionRunId) ? [row.executionRunId] : []); + const projections = await executionProjectionsForRuns(dbOrTx, companyId, scopedIds); + for (const [runId, execution] of projections) { + const row = map.get(runId); + if (row) row.execution = execution; + } + } + return map; +} + +async function liveDescendantCountMapForIssues( + dbOrTx: any, + companyId: string, + issueIds: string[], +): Promise> { + const uniqueIssueIds = [...new Set(issueIds)]; + const map = new Map(); + if (uniqueIssueIds.length === 0) return map; + + for (const issueIdChunk of chunkList( + uniqueIssueIds, + ISSUE_LIST_RELATED_QUERY_CHUNK_SIZE, + )) { + const targetRows = issueIdChunk.map((issueId) => sql`(${issueId}::uuid)`); + const rows = await dbOrTx.execute(sql<{ + issueId: string; + liveDescendantCount: number; + }>` + WITH RECURSIVE + target_issues(issue_id) AS ( + VALUES ${sql.join(targetRows, sql`, `)} + ), + live_issues(live_issue_id, parent_id) AS ( + SELECT DISTINCT live_issue.id, live_issue.parent_id + FROM issues live_issue + JOIN heartbeat_runs live_run ON live_run.id = live_issue.execution_run_id + WHERE live_issue.company_id = ${companyId} + AND live_issue.hidden_at IS NULL + AND live_issue.harness_kind IS NULL + AND live_run.company_id = ${companyId} + AND live_run.status IN ('queued', 'running') + UNION + SELECT DISTINCT live_issue.id, live_issue.parent_id + FROM heartbeat_runs live_run + JOIN issues live_issue ON live_issue.id::text = (live_run.context_snapshot ->> 'issueId') + WHERE live_issue.company_id = ${companyId} + AND live_issue.hidden_at IS NULL + AND live_issue.harness_kind IS NULL + AND live_run.company_id = ${companyId} + AND live_run.status IN ('queued', 'running') + ), + live_ancestors(live_issue_id, ancestor_id, next_parent_id, visited_issue_ids) AS ( + SELECT live_issues.live_issue_id, parent.id, parent.parent_id, ARRAY[live_issues.live_issue_id, parent.id] + FROM live_issues + JOIN issues parent ON parent.id = live_issues.parent_id + WHERE parent.company_id = ${companyId} + AND parent.hidden_at IS NULL + AND parent.harness_kind IS NULL + UNION ALL + SELECT + live_ancestors.live_issue_id, + parent.id, + parent.parent_id, + live_ancestors.visited_issue_ids || parent.id + FROM live_ancestors + JOIN issues parent ON parent.id = live_ancestors.next_parent_id + WHERE parent.company_id = ${companyId} + AND parent.hidden_at IS NULL + AND parent.harness_kind IS NULL + AND NOT parent.id = ANY(live_ancestors.visited_issue_ids) + ) + SELECT + live_ancestors.ancestor_id::text AS "issueId", + count(DISTINCT live_ancestors.live_issue_id)::int AS "liveDescendantCount" + FROM live_ancestors + JOIN target_issues ON target_issues.issue_id = live_ancestors.ancestor_id + WHERE live_ancestors.ancestor_id <> live_ancestors.live_issue_id + GROUP BY live_ancestors.ancestor_id + `); + + const resultRows = Array.isArray(rows) + ? rows + : Array.from(rows as Iterable); + for (const row of resultRows) { + if (typeof row !== "object" || row === null) continue; + const issueId = (row as { issueId?: unknown }).issueId; + const liveDescendantCount = (row as { liveDescendantCount?: unknown }) + .liveDescendantCount; + if (typeof issueId !== "string") continue; + const count = + typeof liveDescendantCount === "number" + ? liveDescendantCount + : Number(liveDescendantCount); + if (Number.isFinite(count)) map.set(issueId, count); + } + } + + return map; +} + +function createIssueBlockerAttention( + input: Partial = {}, +): IssueBlockerAttention { + return { + state: input.state ?? "none", + reason: input.reason ?? null, + unresolvedBlockerCount: input.unresolvedBlockerCount ?? 0, + coveredBlockerCount: input.coveredBlockerCount ?? 0, + stalledBlockerCount: input.stalledBlockerCount ?? 0, + attentionBlockerCount: input.attentionBlockerCount ?? 0, + pendingFinalizeBlockerIssueIds: input.pendingFinalizeBlockerIssueIds ?? [], + sampleBlockerIdentifier: input.sampleBlockerIdentifier ?? null, + sampleStalledBlockerIdentifier: + input.sampleStalledBlockerIdentifier ?? null, + blockingTreeLive: input.blockingTreeLive ?? false, + directBlockerIssueId: input.directBlockerIssueId ?? null, + terminalBlockerIssueId: input.terminalBlockerIssueId ?? null, + terminalBlocker: input.terminalBlocker ?? null, + }; +} + +function blockerSampleIdentifier( + node: IssueBlockerAttentionNode | null | undefined, +) { + return node?.identifier ?? node?.id ?? null; +} + +function appendBlockerAttentionEdges( + edgesByIssueId: Map, + rows: IssueBlockerAttentionEdge[], +) { + for (const row of rows) { + const existing = edgesByIssueId.get(row.issueId) ?? []; + if (!existing.some((edge) => edge.blockerIssueId === row.blockerIssueId)) { + existing.push(row); + edgesByIssueId.set(row.issueId, existing); + } + } +} + +type IssueRelationSummaryRow = { + relatedId: string; + identifier: string | null; + title: string; + status: string; + priority: string; + assigneeAgentId: string | null; + assigneeUserId: string | null; +}; + +function summarizeIssueRelationRow( + row: IssueRelationSummaryRow, +): IssueRelationIssueSummary { + return { + id: row.relatedId, + identifier: row.identifier, + title: row.title, + status: row.status as IssueRelationIssueSummary["status"], + priority: row.priority as IssueRelationIssueSummary["priority"], + assigneeAgentId: row.assigneeAgentId, + assigneeUserId: row.assigneeUserId, + }; +} + +/** Inline draft images become task attachments in the same transaction as publication. */ +async function attachOwnedDraftImages(tx: DbTransaction, issue: typeof issues.$inferSelect, body: string | null | undefined, + actor: { userId?: string | null; agentId?: string | null }) { + const ids = [...new Set(Array.from((body ?? "").matchAll(/\/api\/assets\/([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})\/content/gi), match => match[1]!))]; + if (!ids.length || (!actor.userId && !actor.agentId)) return; + const owned = await tx.select().from(assets).where(and(eq(assets.companyId, issue.companyId), inArray(assets.id, ids), + actor.userId ? eq(assets.createdByUserId, actor.userId) : eq(assets.createdByAgentId, actor.agentId!))); + for (const asset of owned) { + if (!asset.objectKey.startsWith(`${issue.companyId}/assets/issues/drafts/`)) continue; + await tx.insert(issueAttachments).values({ companyId: issue.companyId, issueId: issue.id, assetId: asset.id }).onConflictDoNothing(); + } +} + +export async function ensureAssignmentIssueAccessGrant( + dbOrTx: any, + issue: typeof issues.$inferSelect, + previous: Pick | null, + actor: { agentId?: string | null; userId?: string | null }, +) { + if (issue.visibility !== "private") return null; + const subject = issue.assigneeAgentId + ? { subjectType: "agent" as const, subjectId: issue.assigneeAgentId } + : issue.assigneeUserId + ? { subjectType: "user" as const, subjectId: issue.assigneeUserId } + : null; + if (!subject) return null; + if ( + previous + && previous.visibility === "private" + && previous.assigneeAgentId === issue.assigneeAgentId + && previous.assigneeUserId === issue.assigneeUserId + ) return null; + + const grantIssueId = issue.id; + const lockKey = `issue-access-grant:${grantIssueId}:${subject.subjectType}:${subject.subjectId}`; + await dbOrTx.execute(sql`select pg_advisory_xact_lock(hashtextextended(${lockKey}, 0))`); + const active = await dbOrTx + .select({ id: issueAccessGrants.id }) + .from(issueAccessGrants) + .where(and( + eq(issueAccessGrants.issueId, grantIssueId), + eq(issueAccessGrants.subjectType, subject.subjectType), + eq(issueAccessGrants.subjectId, subject.subjectId), + isNull(issueAccessGrants.revokedAt), + )) + .limit(1) + .then((rows: Array<{ id: string }>) => rows[0] ?? null); + if (active) return null; + + const grant = await dbOrTx + .insert(issueAccessGrants) + .values({ + issueId: grantIssueId, + ...subject, + source: "assignment", + grantedByUserId: actor.userId ?? null, + grantedByAgentId: actor.agentId ?? null, + }) + .returning() + .then((rows: Array) => rows[0]!); + await logActivity(dbOrTx as Db, { + companyId: issue.companyId, + actorType: actor.agentId ? "agent" : actor.userId ? "user" : "system", + actorId: actor.agentId ?? actor.userId ?? "issue_service", + agentId: actor.agentId ?? null, + action: "issue_access_grant.created", + entityType: "issue_access_grant", + entityId: grant.id, + details: { + issueId: grant.issueId, + subjectType: grant.subjectType, + subjectId: grant.subjectId, + source: grant.source, + }, + }); + return grant; +} + +async function terminalExplicitBlockersByRoot( + companyId: string, + roots: IssueRelationIssueSummary[], + dbOrTx: DbReader, +): Promise> { + const rootIds = [...new Set(roots.map((root) => root.id))]; + const terminalByRoot = new Map(); + if (rootIds.length === 0) return terminalByRoot; + + const nodesById = new Map(); + const edgesByIssueId = new Map(); + for (const root of roots) nodesById.set(root.id, root); + + let frontier = rootIds; + for ( + let depth = 0; + frontier.length > 0 && depth < BLOCKER_ATTENTION_MAX_DEPTH; + depth += 1 + ) { + const nextFrontier = new Set(); + for (const chunk of chunkList( + [...new Set(frontier)], + ISSUE_LIST_RELATED_QUERY_CHUNK_SIZE, + )) { + const rows = await dbOrTx + .select({ + currentIssueId: issueRelations.relatedIssueId, + relatedId: issues.id, + identifier: issues.identifier, + title: issues.title, + status: issues.status, + priority: issues.priority, + assigneeAgentId: issues.assigneeAgentId, + assigneeUserId: issues.assigneeUserId, + }) + .from(issueRelations) + .innerJoin(issues, eq(issueRelations.issueId, issues.id)) + .where( + and( + eq(issueRelations.companyId, companyId), + eq(issueRelations.type, "blocks"), + inArray(issueRelations.relatedIssueId, chunk), + eq(issues.companyId, companyId), + ne(issues.status, "done"), + ), + ); + + for (const row of rows) { + const existingEdges = edgesByIssueId.get(row.currentIssueId) ?? []; + if (!existingEdges.includes(row.relatedId)) { + existingEdges.push(row.relatedId); + edgesByIssueId.set(row.currentIssueId, existingEdges); + } + if (!nodesById.has(row.relatedId)) { + nodesById.set(row.relatedId, summarizeIssueRelationRow(row)); + nextFrontier.add(row.relatedId); + } + } + } + + if (nodesById.size > BLOCKER_ATTENTION_MAX_NODES) break; + frontier = [...nextFrontier]; + } + + const collectTerminal = ( + issueId: string, + seen: Set, + ): IssueRelationIssueSummary[] => { + if (seen.has(issueId)) return []; + const node = nodesById.get(issueId); + if (!node || node.status === "done") return []; + const nextSeen = new Set(seen); + nextSeen.add(issueId); + const downstreamIds = edgesByIssueId.get(issueId) ?? []; + if (downstreamIds.length === 0) return [node]; + return downstreamIds.flatMap((downstreamId) => + collectTerminal(downstreamId, nextSeen), + ); + }; + + for (const rootId of rootIds) { + const deduped = new Map(); + for (const blocker of collectTerminal(rootId, new Set())) { + if (blocker.id !== rootId) deduped.set(blocker.id, blocker); + } + if (deduped.size > 0) { + terminalByRoot.set( + rootId, + [...deduped.values()].sort((a, b) => a.title.localeCompare(b.title)), + ); + } + } + + return terminalByRoot; +} + +async function listIssueBlockerAttentionMap( + dbOrTx: any, + companyId: string, + issueRows: IssueBlockerAttentionInputNode[], +): Promise> { + const roots = issueRows.filter( + (row) => row.companyId === companyId && row.status === "blocked", + ); + const attentionMap = new Map(); + for (const row of issueRows) { + if (row.status !== "blocked") { + attentionMap.set(row.id, createIssueBlockerAttention()); + } + } + if (roots.length === 0) return attentionMap; + + const nodesById = new Map(); + const edgesByIssueId = new Map(); + for (const root of roots) nodesById.set(root.id, { ...root }); + + let frontier = roots.map((root) => root.id); + let truncated = false; + const pendingFinalizeBlockerIssueIds = new Set(); + for ( + let depth = 0; + frontier.length > 0 && depth < BLOCKER_ATTENTION_MAX_DEPTH; + depth += 1 + ) { + const nextFrontier = new Set(); + + for (const chunk of chunkList( + [...new Set(frontier)], + ISSUE_LIST_RELATED_QUERY_CHUNK_SIZE, + )) { + const readinessByIssueId = await listIssueDependencyReadinessMap( + dbOrTx, + companyId, + chunk, + ); + for (const readiness of readinessByIssueId.values()) { + for (const blockerIssueId of readiness.pendingFinalizeBlockerIssueIds) { + pendingFinalizeBlockerIssueIds.add(blockerIssueId); + } + } + const explicitBlockerRowsPromise: Promise< + IssueBlockerAttentionQueryRow[] + > = dbOrTx + .select({ + issueId: issueRelations.relatedIssueId, + blockerIssueId: issues.id, + id: issues.id, + companyId: issues.companyId, + parentId: issues.parentId, + identifier: issues.identifier, + title: issues.title, + status: issues.status, + executionRunId: issues.executionRunId, + assigneeAgentId: issues.assigneeAgentId, + assigneeUserId: issues.assigneeUserId, + }) + .from(issueRelations) + .innerJoin(issues, eq(issueRelations.issueId, issues.id)) + .where( + and( + eq(issueRelations.companyId, companyId), + eq(issueRelations.type, "blocks"), + inArray(issueRelations.relatedIssueId, chunk), + eq(issues.companyId, companyId), + ), + ); + const childRowsPromise: Promise = dbOrTx + .select({ + issueId: issues.parentId, + blockerIssueId: issues.id, + id: issues.id, + companyId: issues.companyId, + parentId: issues.parentId, + identifier: issues.identifier, + title: issues.title, + status: issues.status, + executionRunId: issues.executionRunId, + assigneeAgentId: issues.assigneeAgentId, + assigneeUserId: issues.assigneeUserId, + }) + .from(issues) + .where( + and( + eq(issues.companyId, companyId), + inArray(issues.parentId, chunk), + notInArray( + issues.status, + BLOCKER_ATTENTION_CHILD_TERMINAL_STATUSES, + ), + ), + ); + const [explicitBlockerRows, childRows] = await Promise.all([ + explicitBlockerRowsPromise, + childRowsPromise, + ]); + + const unresolvedExplicitBlockerRows = explicitBlockerRows.filter( + (row) => + row.status !== "done" || + pendingFinalizeBlockerIssueIds.has(row.blockerIssueId), + ); + appendBlockerAttentionEdges(edgesByIssueId, [ + ...unresolvedExplicitBlockerRows + .filter( + (row): row is IssueBlockerAttentionQueryRow & { issueId: string } => + row.issueId !== null, + ) + .map((row) => ({ + issueId: row.issueId, + blockerIssueId: row.blockerIssueId, + })), + ...childRows + .filter( + (row): row is IssueBlockerAttentionQueryRow & { issueId: string } => + row.issueId !== null, + ) + .map((row) => ({ + issueId: row.issueId, + blockerIssueId: row.blockerIssueId, + })), + ]); + + for (const row of [...unresolvedExplicitBlockerRows, ...childRows]) { + if (!row.issueId || nodesById.has(row.blockerIssueId)) continue; + nodesById.set(row.blockerIssueId, { + id: row.blockerIssueId, + companyId: row.companyId, + parentId: row.parentId, + identifier: row.identifier, + title: row.title, + status: row.status, + executionRunId: row.executionRunId, + assigneeAgentId: row.assigneeAgentId, + assigneeUserId: row.assigneeUserId, + }); + nextFrontier.add(row.blockerIssueId); + } + } + + if (nodesById.size > BLOCKER_ATTENTION_MAX_NODES) { + truncated = true; + break; + } + frontier = [...nextFrontier]; + } + if (frontier.length > 0) truncated = true; + + const nodeIds = [...nodesById.keys()]; + const activeIssueIds = new Set(); + const agentIds = new Set(); + const issueIdByExecutionRunId = new Map(); + for (const node of nodesById.values()) { + if (node.assigneeAgentId) agentIds.add(node.assigneeAgentId); + if (node.executionRunId) + issueIdByExecutionRunId.set(node.executionRunId, node.id); + } + + for (const chunk of chunkList( + [...issueIdByExecutionRunId.keys()], + ISSUE_LIST_RELATED_QUERY_CHUNK_SIZE, + )) { + const runRows: Array<{ id: string }> = await dbOrTx + .select({ + id: heartbeatRuns.id, + }) + .from(heartbeatRuns) + .where( + and( + eq(heartbeatRuns.companyId, companyId), + inArray(heartbeatRuns.status, BLOCKER_ATTENTION_ACTIVE_RUN_STATUSES), + inArray(heartbeatRuns.id, chunk), + ), + ); + + for (const row of runRows) { + const issueId = issueIdByExecutionRunId.get(row.id); + if (issueId) activeIssueIds.add(issueId); + } + } + + for (const chunk of chunkList(nodeIds, ISSUE_LIST_RELATED_QUERY_CHUNK_SIZE)) { + const wakeRowsPromise: Promise = + dbOrTx + .select({ + issueId: sql< + string | null + >`${agentWakeupRequests.payload} ->> 'issueId'`, + }) + .from(agentWakeupRequests) + .where( + and( + eq(agentWakeupRequests.companyId, companyId), + inArray( + agentWakeupRequests.status, + BLOCKER_ATTENTION_ACTIVE_WAKE_STATUSES, + ), + sql`${agentWakeupRequests.runId} is null`, + inArray( + sql`${agentWakeupRequests.payload} ->> 'issueId'`, + chunk, + ), + ), + ); + const wakeRows = await wakeRowsPromise; + for (const row of wakeRows) { + if (row.issueId) activeIssueIds.add(row.issueId); + } + } + + const explicitWaitCandidateIds = [...nodesById.values()] + .filter((node) => node.status !== "done") + .map((node) => node.id); + const explicitWaitingIssueIds = new Set(); + if (explicitWaitCandidateIds.length > 0) { + for (const chunk of chunkList( + explicitWaitCandidateIds, + ISSUE_LIST_RELATED_QUERY_CHUNK_SIZE, + )) { + const interactionRows: Array<{ issueId: string }> = await dbOrTx + .select({ issueId: issueThreadInteractions.issueId }) + .from(issueThreadInteractions) + .where( + and( + eq(issueThreadInteractions.companyId, companyId), + inArray( + issueThreadInteractions.status, + BLOCKER_ATTENTION_PENDING_INTERACTION_STATUSES, + ), + inArray(issueThreadInteractions.issueId, chunk), + ), + ); + for (const row of interactionRows) + explicitWaitingIssueIds.add(row.issueId); + + const approvalRows: Array<{ issueId: string }> = await dbOrTx + .select({ issueId: issueApprovals.issueId }) + .from(issueApprovals) + .innerJoin(approvals, eq(issueApprovals.approvalId, approvals.id)) + .where( + and( + eq(issueApprovals.companyId, companyId), + inArray( + approvals.status, + BLOCKER_ATTENTION_PENDING_APPROVAL_STATUSES, + ), + inArray(issueApprovals.issueId, chunk), + ), + ); + for (const row of approvalRows) explicitWaitingIssueIds.add(row.issueId); + } + + // Recovery rows are intentionally company-wide: a liveness escalation for + // the same leaf blocker represents an active waiting path even when that + // blocker is reached through another blocked graph. + const recoveryRows: Array<{ id: string; originId: string | null }> = + await dbOrTx + .select({ id: issues.id, originId: issues.originId }) + .from(issues) + .where( + and( + eq(issues.companyId, companyId), + eq(issues.originKind, BLOCKER_ATTENTION_OPEN_RECOVERY_ORIGIN_KIND), + visibleIssueCondition(), + notInArray( + issues.status, + BLOCKER_ATTENTION_OPEN_RECOVERY_TERMINAL_STATUSES, + ), + ), + ); + for (const row of recoveryRows) { + const parsed = parseIssueGraphLivenessIncidentKey(row.originId); + if (!parsed || parsed.companyId !== companyId) continue; + explicitWaitingIssueIds.add(row.id); + explicitWaitingIssueIds.add(parsed.issueId); + explicitWaitingIssueIds.add(parsed.leafIssueId); + } + + const recoveryActionRows: Array<{ + id: string; + sourceIssueId: string; + status: string; + ownerType: string; + ownerAgentId: string | null; + ownerUserId: string | null; + }> = await dbOrTx + .select({ + id: issueRecoveryActions.id, + sourceIssueId: issueRecoveryActions.sourceIssueId, + status: issueRecoveryActions.status, + ownerType: issueRecoveryActions.ownerType, + ownerAgentId: issueRecoveryActions.ownerAgentId, + ownerUserId: issueRecoveryActions.ownerUserId, + }) + .from(issueRecoveryActions) + .where( + and( + eq(issueRecoveryActions.companyId, companyId), + inArray(issueRecoveryActions.status, ["active", "escalated"]), + inArray(issueRecoveryActions.sourceIssueId, explicitWaitCandidateIds), + ), + ); + const recoveryActionIds = recoveryActionRows.map((row) => row.id); + const liveRecoveryActionIds = new Set(); + for (const chunk of chunkList( + recoveryActionIds, + ISSUE_LIST_RELATED_QUERY_CHUNK_SIZE, + )) { + const [runRows, wakeRows] = await Promise.all([ + dbOrTx + .select({ + recoveryActionId: sql< + string | null + >`${heartbeatRuns.contextSnapshot} ->> 'recoveryActionId'`, + }) + .from(heartbeatRuns) + .where( + and( + eq(heartbeatRuns.companyId, companyId), + inArray( + heartbeatRuns.status, + BLOCKER_ATTENTION_ACTIVE_RUN_STATUSES, + ), + inArray( + sql`${heartbeatRuns.contextSnapshot} ->> 'recoveryActionId'`, + chunk, + ), + ), + ), + dbOrTx + .select({ + recoveryActionId: sql< + string | null + >`${agentWakeupRequests.payload} ->> 'recoveryActionId'`, + }) + .from(agentWakeupRequests) + .where( + and( + eq(agentWakeupRequests.companyId, companyId), + inArray( + agentWakeupRequests.status, + BLOCKER_ATTENTION_ACTIVE_WAKE_STATUSES, + ), + inArray( + sql`${agentWakeupRequests.payload} ->> 'recoveryActionId'`, + chunk, + ), + ), + ), + ]); + for (const row of [...runRows, ...wakeRows]) { + if (row.recoveryActionId) + liveRecoveryActionIds.add(row.recoveryActionId); + } + } + for (const row of recoveryActionRows) { + const healthy = + (row.status === "escalated" && row.ownerType === "board") || + Boolean(row.ownerUserId) || + (Boolean(row.ownerAgentId) && liveRecoveryActionIds.has(row.id)); + if (healthy) explicitWaitingIssueIds.add(row.sourceIssueId); + } + } + + const agentRows: IssueBlockerAttentionAgentRow[] = + agentIds.size > 0 + ? await dbOrTx + .select({ + id: agents.id, + companyId: agents.companyId, + status: agents.status, + }) + .from(agents) + .where( + and( + eq(agents.companyId, companyId), + inArray(agents.id, [...agentIds]), + ), + ) + : []; + const agentsById = new Map(agentRows.map((agent) => [agent.id, agent])); + + type PathClassification = { + covered: boolean; + stalled: boolean; + sampleBlockerIdentifier: string | null; + sampleStalledBlockerIdentifier: string | null; + terminalBlockerIssueId?: string | null; + }; + const classifyPath = ( + nodeId: string, + seen: Set, + ): PathClassification => { + const sample = blockerSampleIdentifier(nodesById.get(nodeId)); + if (truncated || seen.has(nodeId)) { + return { + covered: false, + stalled: false, + sampleBlockerIdentifier: sample, + sampleStalledBlockerIdentifier: null, + }; + } + const node = nodesById.get(nodeId); + if (!node || node.companyId !== companyId) { + return { + covered: false, + stalled: false, + sampleBlockerIdentifier: nodeId, + sampleStalledBlockerIdentifier: null, + }; + } + const nodeSample = blockerSampleIdentifier(node); + if ( + node.status === "done" && + !pendingFinalizeBlockerIssueIds.has(node.id) + ) { + return { + covered: true, + stalled: false, + sampleBlockerIdentifier: nodeSample, + sampleStalledBlockerIdentifier: null, + }; + } + if (explicitWaitingIssueIds.has(node.id)) { + return { + covered: true, + stalled: false, + sampleBlockerIdentifier: nodeSample, + sampleStalledBlockerIdentifier: null, + }; + } + if (node.assigneeUserId && node.status !== "cancelled") { + return { + covered: true, + stalled: false, + sampleBlockerIdentifier: nodeSample, + sampleStalledBlockerIdentifier: null, + }; + } + if (node.status === "in_review") { + const hasWaitingPath = + activeIssueIds.has(node.id) || Boolean(node.assigneeUserId); + if (hasWaitingPath) { + return { + covered: true, + stalled: false, + sampleBlockerIdentifier: nodeSample, + sampleStalledBlockerIdentifier: null, + }; + } + return { + covered: false, + stalled: true, + sampleBlockerIdentifier: nodeSample, + sampleStalledBlockerIdentifier: nodeSample, + terminalBlockerIssueId: node.id, + }; + } + if (activeIssueIds.has(node.id)) { + return { + covered: true, + stalled: false, + sampleBlockerIdentifier: nodeSample, + sampleStalledBlockerIdentifier: null, + }; + } + if (node.status === "cancelled") { + return { + covered: false, + stalled: false, + sampleBlockerIdentifier: nodeSample, + sampleStalledBlockerIdentifier: null, + terminalBlockerIssueId: node.id, + }; + } + if (node.status === "backlog" && node.assigneeAgentId) { + return { + covered: false, + stalled: false, + sampleBlockerIdentifier: nodeSample, + sampleStalledBlockerIdentifier: null, + terminalBlockerIssueId: node.id, + }; + } + + const downstream = (edgesByIssueId.get(node.id) ?? []).filter((edge) => { + const blocker = nodesById.get(edge.blockerIssueId); + return ( + blocker?.status !== "done" || + pendingFinalizeBlockerIssueIds.has(edge.blockerIssueId) + ); + }); + if (downstream.length > 0) { + const nextSeen = new Set(seen); + nextSeen.add(nodeId); + const classified = downstream.map((edge) => + classifyPath(edge.blockerIssueId, nextSeen), + ); + const stalledChild = classified.find( + (result) => result.stalled || result.sampleStalledBlockerIdentifier, + ); + const sampleStalled = + stalledChild?.sampleStalledBlockerIdentifier ?? null; + const hardAttention = + classified.find( + (result) => + !result.covered && !result.stalled && result.terminalBlockerIssueId, + ) ?? classified.find((result) => !result.covered && !result.stalled); + if (hardAttention) { + return { + covered: false, + stalled: false, + sampleBlockerIdentifier: hardAttention.sampleBlockerIdentifier, + sampleStalledBlockerIdentifier: sampleStalled, + terminalBlockerIssueId: hardAttention.terminalBlockerIssueId ?? null, + }; + } + const stalledEntry = classified.find((result) => result.stalled); + if (stalledEntry) { + return { + covered: false, + stalled: true, + sampleBlockerIdentifier: stalledEntry.sampleBlockerIdentifier, + sampleStalledBlockerIdentifier: sampleStalled, + terminalBlockerIssueId: stalledEntry.terminalBlockerIssueId ?? null, + }; + } + return { + covered: true, + stalled: false, + sampleBlockerIdentifier: + classified[0]?.sampleBlockerIdentifier ?? nodeSample, + sampleStalledBlockerIdentifier: null, + }; + } + + if (node.assigneeAgentId) { + const assignee = agentsById.get(node.assigneeAgentId); + if ( + !assignee || + assignee.companyId !== companyId || + !BLOCKER_ATTENTION_INVOKABLE_AGENT_STATUSES.has(assignee.status) + ) { + return { + covered: false, + stalled: false, + sampleBlockerIdentifier: nodeSample, + sampleStalledBlockerIdentifier: null, + terminalBlockerIssueId: node.id, + }; + } + } + + return { + covered: false, + stalled: false, + sampleBlockerIdentifier: nodeSample, + sampleStalledBlockerIdentifier: null, + terminalBlockerIssueId: node.id, + }; + }; + + const pathHasLiveWork = (nodeId: string, seen: Set): boolean => { + if (seen.has(nodeId)) return false; + const node = nodesById.get(nodeId); + if (!node || node.companyId !== companyId) return false; + if ( + node.status === "in_progress" || + activeIssueIds.has(node.id) || + explicitWaitingIssueIds.has(node.id) || + Boolean(node.assigneeUserId) + ) + return true; + + const nextSeen = new Set(seen); + nextSeen.add(nodeId); + return (edgesByIssueId.get(node.id) ?? []).some((edge) => { + const blocker = nodesById.get(edge.blockerIssueId); + if ( + blocker?.status === "done" && + !pendingFinalizeBlockerIssueIds.has(edge.blockerIssueId) + ) + return false; + return pathHasLiveWork(edge.blockerIssueId, nextSeen); + }); + }; + + const issueIdForSample = (sample: string | null | undefined) => { + if (!sample) return null; + for (const node of nodesById.values()) { + if (node.id === sample || node.identifier === sample) return node.id; + } + return null; + }; + + for (const root of roots) { + const topLevelEdges = (edgesByIssueId.get(root.id) ?? []).filter((edge) => { + const blocker = nodesById.get(edge.blockerIssueId); + return ( + blocker?.status !== "done" || + pendingFinalizeBlockerIssueIds.has(edge.blockerIssueId) + ); + }); + if (topLevelEdges.length === 0) { + attentionMap.set( + root.id, + createIssueBlockerAttention({ + state: "needs_attention", + reason: "attention_required", + terminalBlockerIssueId: root.id, + }), + ); + continue; + } + + const classified = topLevelEdges.map((edge) => ({ + edge, + result: classifyPath(edge.blockerIssueId, new Set([root.id])), + })); + const coveredBlockerCount = classified.filter( + (entry) => entry.result.covered, + ).length; + const stalledBlockerCount = classified.filter( + (entry) => entry.result.stalled, + ).length; + const attentionBlockerCount = + classified.length - coveredBlockerCount - stalledBlockerCount; + const hardAttentionEntry = + classified.find( + (entry) => + !entry.result.covered && + !entry.result.stalled && + entry.result.terminalBlockerIssueId, + ) ?? + classified.find( + (entry) => !entry.result.covered && !entry.result.stalled, + ); + const stalledEntry = + classified.find( + (entry) => entry.result.stalled && entry.result.terminalBlockerIssueId, + ) ?? classified.find((entry) => entry.result.stalled); + const sampleEntry = + hardAttentionEntry ?? stalledEntry ?? classified[0] ?? null; + const sampleNode = sampleEntry + ? nodesById.get(sampleEntry.edge.blockerIssueId) + : null; + const sampleStalledFromChain = classified + .map((entry) => entry.result.sampleStalledBlockerIdentifier) + .find((value) => value); + const sampledTerminalIdentifier = sampleEntry?.result.stalled + ? (sampleEntry.result.sampleStalledBlockerIdentifier ?? + sampleEntry.result.sampleBlockerIdentifier) + : (sampleEntry?.result.sampleBlockerIdentifier ?? + blockerSampleIdentifier(sampleNode)); + const terminalBlockerIssueId = + sampleEntry?.result.terminalBlockerIssueId ?? + issueIdForSample(sampledTerminalIdentifier); + const terminalBlockerNode = terminalBlockerIssueId + ? (nodesById.get(terminalBlockerIssueId) ?? null) + : null; + + let state: IssueBlockerAttention["state"]; + let reason: IssueBlockerAttention["reason"]; + if (attentionBlockerCount > 0) { + state = "needs_attention"; + reason = "attention_required"; + } else if (stalledBlockerCount > 0) { + state = "stalled"; + reason = "stalled_review"; + } else { + state = "covered"; + reason = topLevelEdges.every( + (edge) => nodesById.get(edge.blockerIssueId)?.parentId === root.id, + ) + ? "active_child" + : "active_dependency"; + } + + attentionMap.set( + root.id, + createIssueBlockerAttention({ + state, + reason, + unresolvedBlockerCount: topLevelEdges.length, + coveredBlockerCount, + stalledBlockerCount, + attentionBlockerCount, + pendingFinalizeBlockerIssueIds: topLevelEdges + .map((edge) => edge.blockerIssueId) + .filter((blockerIssueId) => + pendingFinalizeBlockerIssueIds.has(blockerIssueId), + ), + sampleBlockerIdentifier: + sampleEntry?.result.sampleBlockerIdentifier ?? + blockerSampleIdentifier(sampleNode), + sampleStalledBlockerIdentifier: + stalledEntry?.result.sampleStalledBlockerIdentifier ?? + sampleStalledFromChain ?? + null, + blockingTreeLive: topLevelEdges.some((edge) => + pathHasLiveWork(edge.blockerIssueId, new Set([root.id])), + ), + directBlockerIssueId: sampleEntry?.edge.blockerIssueId ?? null, + terminalBlockerIssueId, + terminalBlocker: terminalBlockerNode + ? { + id: terminalBlockerNode.id, + identifier: terminalBlockerNode.identifier, + title: terminalBlockerNode.title, + } + : null, + }), + ); + } + + return attentionMap; +} + +type IssueReviewAttentionInput = Pick; + +function reviewPathLabel( + kind: IssueReviewAttentionPath["kind"], + detail?: string | null, +) { + switch (kind) { + case "execution_participant": + return "Execution review participant"; + case "interaction": + return detail + ? `Pending ${detail.replaceAll("_", " ")}` + : "Pending issue interaction"; + case "approval": + return "Linked approval"; + case "monitor": + return "Scheduled review monitor"; + case "external_response": + return "Waiting for an external update"; + case "human_reviewer": + return "Human reviewer"; + case "active_run": + return "Active review run"; + case "queued_wake": + return detail + ? `Queued ${detail.replaceAll("_", " ")} wake` + : "Queued review wake"; + case "recovery": + return "Open review recovery"; + } +} + +function reviewAttentionNone(): IssueReviewAttention { + return { state: "none", paths: [], reason: null }; +} + +async function listIssueReviewAttentionMap( + dbOrTx: any, + companyId: string, + issueRows: IssueReviewAttentionInput[], +): Promise> { + const result = new Map(); + for (const row of issueRows) result.set(row.id, reviewAttentionNone()); + + const reviewIds = issueRows + .filter((row) => row.companyId === companyId && row.status === "in_review") + .map((row) => row.id); + if (reviewIds.length === 0) return result; + + const reviewIssues: IssueRow[] = []; + for (const chunk of chunkList( + reviewIds, + ISSUE_LIST_RELATED_QUERY_CHUNK_SIZE, + )) { + reviewIssues.push( + ...(await dbOrTx + .select() + .from(issues) + .where( + and(eq(issues.companyId, companyId), inArray(issues.id, chunk), nonIdleSlackIssueCondition()), + )), + ); + } + if (reviewIssues.length === 0) return result; + + const [ + agentRows, + activeRunRows, + wakeRows, + interactionRows, + approvalRows, + recoveryActionRows, + recoveryIssueRows, + ] = await Promise.all([ + dbOrTx + .select({ + id: agents.id, + companyId: agents.companyId, + name: agents.name, + role: agents.role, + title: agents.title, + status: agents.status, + reportsTo: agents.reportsTo, + }) + .from(agents) + .where(eq(agents.companyId, companyId)), + dbOrTx + .select({ + id: heartbeatRuns.id, + companyId: heartbeatRuns.companyId, + issueId: sql< + string | null + >`coalesce(${heartbeatRuns.contextSnapshot} ->> 'issueId', ${heartbeatRuns.contextSnapshot} ->> 'taskId')`, + agentId: heartbeatRuns.agentId, + status: heartbeatRuns.status, + createdAt: heartbeatRuns.createdAt, + }) + .from(heartbeatRuns) + .where( + and( + eq(heartbeatRuns.companyId, companyId), + inArray(heartbeatRuns.status, ["queued", "running"]), + inArray( + sql`coalesce(${heartbeatRuns.contextSnapshot} ->> 'issueId', ${heartbeatRuns.contextSnapshot} ->> 'taskId')`, + reviewIds, + ), + ), + ), + dbOrTx + .select({ + id: agentWakeupRequests.id, + companyId: agentWakeupRequests.companyId, + issueId: sql`coalesce( + ${agentWakeupRequests.payload} ->> 'issueId', + ${agentWakeupRequests.payload} ->> 'taskId', + ${agentWakeupRequests.payload} -> '_paperclipWakeContext' ->> 'issueId', + ${agentWakeupRequests.payload} -> '_paperclipWakeContext' ->> 'taskId' + )`, + agentId: agentWakeupRequests.agentId, + status: agentWakeupRequests.status, + reason: agentWakeupRequests.reason, + createdAt: agentWakeupRequests.requestedAt, + }) + .from(agentWakeupRequests) + .where( + and( + eq(agentWakeupRequests.companyId, companyId), + inArray(agentWakeupRequests.status, [ + "queued", + "deferred_issue_execution", + "claimed", + ]), + inArray( + sql`coalesce( + ${agentWakeupRequests.payload} ->> 'issueId', + ${agentWakeupRequests.payload} ->> 'taskId', + ${agentWakeupRequests.payload} -> '_paperclipWakeContext' ->> 'issueId', + ${agentWakeupRequests.payload} -> '_paperclipWakeContext' ->> 'taskId' + )`, + reviewIds, + ), + ), + ), + dbOrTx + .select({ + id: issueThreadInteractions.id, + companyId: issueThreadInteractions.companyId, + issueId: issueThreadInteractions.issueId, + status: issueThreadInteractions.status, + kind: issueThreadInteractions.kind, + createdByAgentId: issueThreadInteractions.createdByAgentId, + sourceRunId: issueThreadInteractions.sourceRunId, + addresseeAgentId: issueThreadInteractions.addresseeAgentId, + effectiveResolverPolicy: + issueThreadInteractions.effectiveResolverPolicy, + resolverPolicyProvenance: + issueThreadInteractions.resolverPolicyProvenance, + createdAt: issueThreadInteractions.createdAt, + }) + .from(issueThreadInteractions) + .where( + and( + eq(issueThreadInteractions.companyId, companyId), + eq(issueThreadInteractions.status, "pending"), + inArray(issueThreadInteractions.issueId, reviewIds), + ), + ), + dbOrTx + .select({ + id: approvals.id, + companyId: issueApprovals.companyId, + issueId: issueApprovals.issueId, + status: approvals.status, + createdAt: approvals.createdAt, + }) + .from(issueApprovals) + .innerJoin(approvals, eq(issueApprovals.approvalId, approvals.id)) + .where( + and( + eq(issueApprovals.companyId, companyId), + eq(approvals.companyId, companyId), + inArray(approvals.status, ["pending", "revision_requested"]), + inArray(issueApprovals.issueId, reviewIds), + ), + ), + dbOrTx + .select({ + id: issueRecoveryActions.id, + companyId: issueRecoveryActions.companyId, + issueId: issueRecoveryActions.sourceIssueId, + status: issueRecoveryActions.status, + createdAt: issueRecoveryActions.createdAt, + }) + .from(issueRecoveryActions) + .where( + and( + eq(issueRecoveryActions.companyId, companyId), + inArray(issueRecoveryActions.status, ["active", "escalated"]), + inArray(issueRecoveryActions.sourceIssueId, reviewIds), + ), + ), + dbOrTx + .select({ + id: issues.id, + companyId: issues.companyId, + originKind: issues.originKind, + originId: issues.originId, + status: issues.status, + createdAt: issues.createdAt, + }) + .from(issues) + .where( + and( + eq(issues.companyId, companyId), + inArray(issues.originKind, [ + RECOVERY_ORIGIN_KINDS.strandedIssueRecovery, + RECOVERY_ORIGIN_KINDS.issueGraphLivenessEscalation, + ]), + visibleIssueCondition(), + notInArray(issues.status, ["done", "cancelled"]), + ), + ), + ]); + + const recoveryPaths = [ + ...(recoveryActionRows as Array<{ + id: string; + companyId: string; + issueId: string; + status: string; + createdAt: Date; + }>), + ]; + for (const recovery of recoveryIssueRows as Array<{ + id: string; + companyId: string; + originKind: string; + originId: string | null; + status: string; + createdAt: Date; + }>) { + if ( + recovery.originKind === RECOVERY_ORIGIN_KINDS.strandedIssueRecovery && + recovery.originId && + reviewIds.includes(recovery.originId) + ) { + recoveryPaths.push({ ...recovery, issueId: recovery.originId }); + continue; + } + const parsed = parseIssueGraphLivenessIncidentKey(recovery.originId); + if (parsed?.companyId === companyId && reviewIds.includes(parsed.issueId)) { + recoveryPaths.push({ ...recovery, issueId: parsed.issueId }); + } + } + + const livenessInput: IssueGraphLivenessInput = { + issues: reviewIssues.map((issue) => ({ + id: issue.id, + companyId: issue.companyId, + identifier: issue.identifier, + title: issue.title, + status: issue.status, + projectId: issue.projectId, + goalId: issue.goalId, + parentId: issue.parentId, + assigneeAgentId: issue.assigneeAgentId, + assigneeUserId: issue.assigneeUserId, + createdByAgentId: issue.createdByAgentId, + createdByUserId: issue.createdByUserId, + conversationAgentId: issue.conversationAgentId, + conversationUserId: issue.conversationUserId, + conversationState: issue.conversationState, + executionPolicy: issue.executionPolicy, + executionState: issue.executionState, + monitorNextCheckAt: issue.monitorNextCheckAt, + monitorAttemptCount: issue.monitorAttemptCount, + })), + relations: [], + agents: agentRows, + activeRuns: activeRunRows, + queuedWakeRequests: wakeRows, + pendingInteractions: interactionRows, + pendingApprovals: approvalRows, + externalResponseWaits: await listExternalReviewWaits(dbOrTx, companyId, reviewIds), + openRecoveryIssues: recoveryPaths, + now: new Date(), + }; + const findingsByIssueId = new Map( + classifyIssueGraphLiveness(livenessInput).map((finding) => [ + finding.issueId, + finding, + ]), + ); + const agentNameById = new Map( + (agentRows as Array<{ id: string; name: string }>).map((agent) => [ + agent.id, + agent.name, + ]), + ); + const userIds = new Set(); + for (const issue of reviewIssues) { + if (issue.assigneeUserId) userIds.add(issue.assigneeUserId); + const participant = parseObject(issue.executionState).currentParticipant; + if ( + participant && + typeof participant === "object" && + !Array.isArray(participant) + ) { + const userId = (participant as Record).userId; + if (typeof userId === "string") userIds.add(userId); + } + } + const userRows = + userIds.size > 0 + ? await dbOrTx + .select({ id: authUsers.id, name: authUsers.name }) + .from(authUsers) + .where(inArray(authUsers.id, [...userIds])) + : []; + const userNameById = new Map( + (userRows as Array<{ id: string; name: string }>).map((user) => [ + user.id, + user.name, + ]), + ); + const interactionKindById = new Map( + (interactionRows as Array<{ id: string; kind: string }>).map((row) => [ + row.id, + row.kind, + ]), + ); + const interactionAudienceById = new Map( + ( + interactionRows as Array<{ + id: string; + createdByAgentId: string | null; + sourceRunId: string | null; + addresseeAgentId: string | null; + effectiveResolverPolicy: string; + resolverPolicyProvenance: string | null; + }> + ).map((row) => [row.id, row]), + ); + const wakeReasonById = new Map( + (wakeRows as Array<{ id: string; reason: string | null }>).map((row) => [ + row.id, + row.reason, + ]), + ); + + for (const issue of reviewIssues) { + const pathFacts = classifyIssueReviewPaths( + livenessInput, + livenessInput.issues.find((entry) => entry.id === issue.id)!, + ); + const paths: IssueReviewAttentionPath[] = pathFacts.map((path) => { + const interactionAudience = + path.kind === "interaction" && path.ref + ? (interactionAudienceById.get(path.ref) ?? null) + : null; + const candidateAgentId = + interactionAudience?.addresseeAgentId ?? issue.assigneeAgentId; + const interactionResponderAgentId = + interactionAudience && + candidateAgentId && + issueThreadInteractionAttentionAgentAllowed({ + agentId: candidateAgentId, + interaction: interactionAudience, + }) + ? candidateAgentId + : null; + return { + kind: path.kind, + label: reviewPathLabel( + path.kind, + path.kind === "interaction" && path.ref + ? (interactionKindById.get(path.ref) ?? null) + : path.kind === "queued_wake" && path.ref + ? (wakeReasonById.get(path.ref) ?? null) + : null, + ), + responder: path.agentId + ? (agentNameById.get(path.agentId) ?? path.agentId) + : path.userId + ? (userNameById.get(path.userId) ?? path.userId) + : path.kind === "interaction" && interactionResponderAgentId + ? (agentNameById.get(interactionResponderAgentId) ?? + interactionResponderAgentId) + : path.kind === "interaction" || path.kind === "approval" + ? "Board" + : null, + since: path.since + ? (path.since instanceof Date + ? path.since + : new Date(path.since) + ).toISOString() + : issue.updatedAt.toISOString(), + ref: path.ref, + }; + }); + + if (paths.length > 0) { + result.set(issue.id, { + state: "covered", + paths, + reason: + paths.length === 1 + ? "Review has a maintained action path." + : `Review has ${paths.length} maintained action paths.`, + }); + continue; + } + + const finding = findingsByIssueId.get(issue.id); + result.set(issue.id, { + state: "stalled", + paths: [], + reason: + finding?.reason ?? + "Issue is in review without a maintained action path.", + }); + } + + return result; +} + +const issueListSelect = { + externalConversationState: externalConversationStateSql(), + conversationAgentId: issues.conversationAgentId, + conversationUserId: issues.conversationUserId, + conversationState: issues.conversationState, + conversationSessionGeneration: issues.conversationSessionGeneration, + conversationBoundaryCommentId: issues.conversationBoundaryCommentId, + id: issues.id, + companyId: issues.companyId, + projectId: issues.projectId, + projectWorkspaceId: issues.projectWorkspaceId, + goalId: issues.goalId, + parentId: issues.parentId, + visibility: issues.visibility, + privacyRootIssueId: issues.privacyRootIssueId, + privacyParentIssueId: issues.privacyParentIssueId, + title: issues.title, + titleNeedsGeneration: issues.titleNeedsGeneration, + description: sql` + CASE + WHEN ${issues.description} IS NULL THEN NULL + ELSE encode( + substring( + convert_to(${issues.description}, current_setting('server_encoding')) + FROM 1 FOR ${ISSUE_LIST_DESCRIPTION_MAX_BYTES} + ), + 'base64' + ) + END + `, + descriptionTruncated: sql` + CASE + WHEN ${issues.description} IS NULL THEN false + ELSE length(${issues.description}) > ${ISSUE_LIST_DESCRIPTION_MAX_CHARS} + END + `, + status: issues.status, + statusVersion: issues.statusVersion, + lastStatusDecisionId: issues.lastStatusDecisionId, + workMode: issues.workMode, + harnessKind: issues.harnessKind, + priority: issues.priority, + reviewPolicy: issues.reviewPolicy, + assigneeAgentId: issues.assigneeAgentId, + assigneeUserId: issues.assigneeUserId, + checkoutRunId: issues.checkoutRunId, + executionRunId: issues.executionRunId, + executionAgentNameKey: issues.executionAgentNameKey, + executionLockedAt: issues.executionLockedAt, + createdByAgentId: issues.createdByAgentId, + createdByUserId: issues.createdByUserId, + responsibleUserId: issues.responsibleUserId, + issueNumber: issues.issueNumber, + identifier: issues.identifier, + originKind: issues.originKind, + originId: issues.originId, + originRunId: issues.originRunId, + originIdentityContextId: issues.originIdentityContextId, + continuationIdentityContextId: issues.continuationIdentityContextId, + originFingerprint: issues.originFingerprint, + requestDepth: issues.requestDepth, + billingCode: issues.billingCode, + assigneeAdapterOverrides: issues.assigneeAdapterOverrides, + executionPolicy: sql`null`, + executionState: sql`null`, + monitorNextCheckAt: issues.monitorNextCheckAt, + monitorWakeRequestedAt: issues.monitorWakeRequestedAt, + monitorLastTriggeredAt: issues.monitorLastTriggeredAt, + monitorAttemptCount: issues.monitorAttemptCount, + monitorNotes: issues.monitorNotes, + monitorScheduledBy: issues.monitorScheduledBy, + executionWorkspaceId: issues.executionWorkspaceId, + executionWorkspacePreference: issues.executionWorkspacePreference, + executionWorkspaceSettings: sql`null`, + workspaceSelection: sql`null`, + workspacePendingSelection: sql`null`, + workspaceBindingRevision: issues.workspaceBindingRevision, + sourceTrust: issues.sourceTrust, + unblockDescriptor: issues.unblockDescriptor, + blockedTransitionAt: issues.blockedTransitionAt, + blockedOwnerNotifiedAt: issues.blockedOwnerNotifiedAt, + startedAt: issues.startedAt, + completedAt: issues.completedAt, + cancelledAt: issues.cancelledAt, + hiddenAt: issues.hiddenAt, + createdAt: issues.createdAt, + updatedAt: issues.updatedAt, +}; + +function withActiveRuns( + issueRows: IssueWithLabels[], + runMap: Map, +): IssueWithLabelsAndRun[] { + return issueRows.map((row) => ({ + ...row, + activeRun: row.executionRunId + ? (runMap.get(row.executionRunId) ?? null) + : null, + })); +} + +async function userCommentStatsForIssues( + dbOrTx: any, + companyId: string, + userId: string, + issueIds: string[], +): Promise { + const stats: IssueUserCommentStats[] = []; + for (const issueIdChunk of chunkList( + issueIds, + ISSUE_LIST_RELATED_QUERY_CHUNK_SIZE, + )) { + const rows = await dbOrTx + .select({ + issueId: issueComments.issueId, + myLastCommentAt: sql` + MAX(CASE WHEN ${issueComments.authorUserId} = ${userId} AND ${directHumanCommentCondition()} + THEN ${issueComments.createdAt} END) + `, + lastExternalCommentAt: sql` + MAX( + CASE + WHEN ${issueComments.authorUserId} IS NULL OR ${issueComments.authorUserId} <> ${userId} + THEN ${issueComments.createdAt} + END + ) + `, + }) + .from(issueComments) + .where( + and( + eq(issueComments.companyId, companyId), + inArray(issueComments.issueId, issueIdChunk), + ), + ) + .groupBy(issueComments.issueId); + const unreadRows = await dbOrTx.select({ id: issues.id }).from(issues).where(and( + eq(issues.companyId, companyId), + inArray(issues.id, issueIdChunk), + unreadForUserCondition(companyId, userId), + )); + const unreadIds = new Set(unreadRows.map((row: { id: string }) => row.id)); + const commentsByIssueId = new Map( + rows.map((row: { issueId: string; myLastCommentAt: Date | null; lastExternalCommentAt: Date | null }) => [row.issueId, row]), + ); + for (const issueId of issueIdChunk) stats.push({ + issueId, + myLastCommentAt: commentsByIssueId.get(issueId)?.myLastCommentAt ?? null, + lastExternalCommentAt: commentsByIssueId.get(issueId)?.lastExternalCommentAt ?? null, + isUnreadForMe: unreadIds.has(issueId), + }); + } + return stats; +} + +async function userReadStatsForIssues( + dbOrTx: any, + companyId: string, + userId: string, + issueIds: string[], +): Promise { + const stats: IssueReadStat[] = []; + for (const issueIdChunk of chunkList( + issueIds, + ISSUE_LIST_RELATED_QUERY_CHUNK_SIZE, + )) { + const rows = await dbOrTx + .select({ + issueId: issueReadStates.issueId, + myLastReadAt: issueReadStates.lastReadAt, + }) + .from(issueReadStates) + .where( + and( + eq(issueReadStates.companyId, companyId), + eq(issueReadStates.userId, userId), + inArray(issueReadStates.issueId, issueIdChunk), + ), + ); + stats.push(...rows); + } + return stats; +} + +async function lastActivityStatsForIssues( + dbOrTx: any, + companyId: string, + issueIds: string[], +): Promise { + const byIssueId = new Map(); + for (const issueIdChunk of chunkList( + issueIds, + ISSUE_LIST_RELATED_QUERY_CHUNK_SIZE, + )) { + const [commentRows, logRows] = await Promise.all([ + dbOrTx + .select({ + issueId: issueComments.issueId, + latestCommentAt: sql`MAX(${issueComments.createdAt})`, + }) + .from(issueComments) + .where( + and( + eq(issueComments.companyId, companyId), + inArray(issueComments.issueId, issueIdChunk), + ), + ) + .groupBy(issueComments.issueId), + dbOrTx + .select({ + issueId: activityLog.entityId, + latestLogAt: sql`MAX(${activityLog.createdAt})`, + }) + .from(activityLog) + .where( + and( + eq(activityLog.companyId, companyId), + eq(activityLog.entityType, "issue"), + inArray(activityLog.entityId, issueIdChunk), + sql`NOT ${automaticReadRecoveryCleanupCondition()}`, + sql`${activityLog.action} NOT IN (${sql.join( + ISSUE_LOCAL_INBOX_ACTIVITY_ACTIONS.map( + (action) => sql`${action}`, + ), + sql`, `, + )})`, + ), + ) + .groupBy(activityLog.entityId), + ]); + + for (const row of commentRows) { + byIssueId.set(row.issueId, { + issueId: row.issueId, + latestCommentAt: row.latestCommentAt, + latestLogAt: null, + }); + } + for (const row of logRows) { + const existing = byIssueId.get(row.issueId); + if (existing) existing.latestLogAt = row.latestLogAt; + else { + byIssueId.set(row.issueId, { + issueId: row.issueId, + latestCommentAt: null, + latestLogAt: row.latestLogAt, + }); + } + } + } + return [...byIssueId.values()]; +} + +async function blockedByMapForIssues( + dbOrTx: any, + companyId: string, + issueIds: string[], +): Promise> { + const map = new Map(); + const uniqueIssueIds = [...new Set(issueIds)]; + if (uniqueIssueIds.length === 0) return map; + + for (const issueId of uniqueIssueIds) { + map.set(issueId, []); + } + + for (const issueIdChunk of chunkList( + uniqueIssueIds, + ISSUE_LIST_RELATED_QUERY_CHUNK_SIZE, + )) { + const rows = await dbOrTx + .select({ + currentIssueId: issueRelations.relatedIssueId, + relatedId: issues.id, + identifier: issues.identifier, + title: issues.title, + status: issues.status, + priority: issues.priority, + assigneeAgentId: issues.assigneeAgentId, + assigneeUserId: issues.assigneeUserId, + }) + .from(issueRelations) + .innerJoin(issues, eq(issueRelations.issueId, issues.id)) + .where( + and( + eq(issueRelations.companyId, companyId), + eq(issueRelations.type, "blocks"), + inArray(issueRelations.relatedIssueId, issueIdChunk), + ), + ); + + for (const row of rows) { + const blockedBy = map.get(row.currentIssueId); + if (!blockedBy) continue; + blockedBy.push({ + id: row.relatedId, + identifier: row.identifier, + title: row.title, + status: row.status as IssueRelationIssueSummary["status"], + priority: row.priority as IssueRelationIssueSummary["priority"], + assigneeAgentId: row.assigneeAgentId, + assigneeUserId: row.assigneeUserId, + }); + } + } + + for (const blockedBy of map.values()) { + blockedBy.sort((a, b) => a.title.localeCompare(b.title)); + } + + return map; +} + +const BLOCKED_INBOX_TERMINAL_STATUSES = ["done", "cancelled"] as const; +const BLOCKED_INBOX_ACTIVE_RUN_STATUSES = ["queued", "running"] as const; +const BLOCKED_INBOX_ACTIVE_WAKE_STATUSES = + SUCCESSFUL_RUN_HANDOFF_LIVE_WAKE_STATUSES; +const BLOCKED_INBOX_PENDING_INTERACTION_STATUSES = ["pending"] as const; +const BLOCKED_INBOX_PENDING_APPROVAL_STATUSES = [ + "pending", + "revision_requested", +] as const; +const BLOCKED_INBOX_RECOVERY_ORIGIN_KINDS = [ + "harness_liveness_escalation", + "stranded_issue_recovery", +] as const; +const BLOCKED_INBOX_SUCCESSFUL_RUN_HANDOFF_ACTIONS = [ + "issue.successful_run_handoff_required", + "issue.successful_run_handoff_resolved", + "issue.successful_run_handoff_escalated", +] as const; + +type BlockedInboxIssueRow = IssueRow & { + labels?: IssueLabelRow[]; + labelIds?: string[]; +}; +type BlockedInboxInteractionRow = { + id: string; + issueId: string; + kind: string; + createdAt: Date; +}; +type BlockedInboxApprovalRow = { + approvalId: string; + issueId: string; + createdAt: Date; +}; + +function issueRef( + row: + | Pick< + IssueRow, + | "id" + | "identifier" + | "title" + | "status" + | "priority" + | "assigneeAgentId" + | "assigneeUserId" + > + | null + | undefined, +): IssueBlockedInboxIssueRef | null { + if (!row) return null; + return { + id: row.id, + identifier: row.identifier, + title: row.title, + status: row.status as IssueBlockedInboxIssueRef["status"], + priority: row.priority as IssueBlockedInboxIssueRef["priority"], + assigneeAgentId: row.assigneeAgentId, + assigneeUserId: row.assigneeUserId, + }; +} + +function hasPlanDocumentCondition( + companyId: string, + hasPlanDocument: boolean, +): SQL { + const existsPlanDocument = sql` + EXISTS ( + SELECT 1 + FROM ${issueDocuments} + WHERE ${issueDocuments.companyId} = ${companyId} + AND ${issueDocuments.issueId} = ${issues.id} + AND ${issueDocuments.key} = 'plan' + ) + `; + return hasPlanDocument + ? existsPlanDocument + : sql`NOT ${existsPlanDocument}`; +} + +function isoDate(value: Date | string | null | undefined): string | null { + if (!value) return null; + const date = value instanceof Date ? value : new Date(value); + return Number.isNaN(date.getTime()) ? null : date.toISOString(); +} + +function attentionBase(input: { + state: IssueBlockedInboxAttention["state"]; + reason: IssueBlockedInboxAttention["reason"]; + severity: IssueBlockedInboxAttention["severity"]; + stoppedSinceAt: Date | string | null | undefined; + owner: IssueBlockedInboxAttention["owner"]; + action: IssueBlockedInboxAttention["action"]; + sourceIssue: IssueBlockedInboxIssueRef | null; + leafIssue?: IssueBlockedInboxIssueRef | null; + recoveryIssue?: IssueBlockedInboxIssueRef | null; + approvalId?: string | null; + interactionId?: string | null; + sampleIssueIdentifier?: string | null; + externalDetailsRedacted?: boolean; +}): IssueBlockedInboxAttention { + return { + kind: "blocked", + state: input.state, + reason: input.reason, + severity: input.severity, + stoppedSinceAt: isoDate(input.stoppedSinceAt), + owner: input.owner, + action: input.action, + sourceIssue: input.sourceIssue, + leafIssue: input.leafIssue ?? null, + recoveryIssue: input.recoveryIssue ?? null, + approvalId: input.approvalId ?? null, + interactionId: input.interactionId ?? null, + sampleIssueIdentifier: + input.sampleIssueIdentifier ?? + input.leafIssue?.identifier ?? + input.recoveryIssue?.identifier ?? + input.sourceIssue?.identifier ?? + null, + redaction: { + externalDetailsRedacted: input.externalDetailsRedacted ?? false, + secretFieldsOmitted: true, + }, + }; +} + +function readSuccessfulRunHandoffFromActivity(row: { + action: string; + agentId: string | null; + runId: string | null; + details: Record | null; + createdAt: Date; +}): SuccessfulRunHandoffState | null { + const details = row.details ?? {}; + const state = + row.action === "issue.successful_run_handoff_required" + ? "required" + : row.action === "issue.successful_run_handoff_resolved" + ? "resolved" + : row.action === "issue.successful_run_handoff_escalated" + ? "escalated" + : null; + if (!state) return null; + + const detectedProgressSummary = + readStringFromRecord(details, "detectedProgressSummary") ?? + readStringFromRecord(details, "detected_progress_summary") ?? + null; + + return { + state, + required: state === "required", + hasLiveContinuation: false, + sourceRunId: + readStringFromRecord(details, "sourceRunId") ?? + readStringFromRecord(details, "source_run_id") ?? + readStringFromRecord(details, "resumeFromRunId") ?? + row.runId ?? + null, + correctiveRunId: + readStringFromRecord(details, "correctiveRunId") ?? + readStringFromRecord(details, "corrective_run_id") ?? + (state !== "required" ? row.runId : null), + assigneeAgentId: + readStringFromRecord(details, "assigneeAgentId") ?? + readStringFromRecord(details, "agentId") ?? + row.agentId ?? + null, + detectedProgressSummary: detectedProgressSummary + ? redactSensitiveText(detectedProgressSummary) + : null, + createdAt: row.createdAt, + }; +} + +async function listSuccessfulRunHandoffMapForIssues( + dbOrTx: any, + companyId: string, + issueIds: string[], + options?: { hydrateLiveness?: boolean }, +): Promise> { + const uniqueIssueIds = [...new Set(issueIds)]; + const states = new Map(); + if (uniqueIssueIds.length === 0) return states; + + for (const issueIdChunk of chunkList( + uniqueIssueIds, + ISSUE_LIST_RELATED_QUERY_CHUNK_SIZE, + )) { + const rows = await dbOrTx + .select({ + entityId: activityLog.entityId, + action: activityLog.action, + agentId: activityLog.agentId, + runId: activityLog.runId, + details: activityLog.details, + createdAt: activityLog.createdAt, + }) + .from(activityLog) + .where( + and( + eq(activityLog.companyId, companyId), + eq(activityLog.entityType, "issue"), + inArray(activityLog.entityId, issueIdChunk), + inArray(activityLog.action, [ + ...BLOCKED_INBOX_SUCCESSFUL_RUN_HANDOFF_ACTIONS, + ]), + ), + ) + .orderBy( + activityLog.entityId, + desc(activityLog.createdAt), + desc(activityLog.id), + ); + + for (const row of rows as Array<{ + entityId: string; + action: string; + agentId: string | null; + runId: string | null; + details: Record | null; + createdAt: Date; + }>) { + if (states.has(row.entityId)) continue; + const state = readSuccessfulRunHandoffFromActivity(row); + if (state) states.set(row.entityId, state); + } + } + + return options?.hydrateLiveness === false + ? states + : hydrateSuccessfulRunHandoffLiveness(dbOrTx, companyId, states); +} + +function externalWaitFromDescription( + description: string | null, +): { owner: string; action: string } | null { + if (!description) return null; + const owner = description + .match(/^\s*external owner\s*:\s*(.+)$/im)?.[1] + ?.trim(); + const action = description + .match(/^\s*external action\s*:\s*(.+)$/im)?.[1] + ?.trim(); + if (!owner || !action) return null; + return { + owner: owner.slice(0, 120), + action: action.slice(0, 240), + }; +} + +function escapeRegExp(value: string) { + return value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); +} + +function redactExternalWaitDescription( + description: string | null | undefined, + external: { owner: string; action: string } | null, +) { + if (!description) return null; + let redacted = description + .split(/\r?\n/) + .filter((line) => !/^\s*external\s+(?:owner|action)\s*:/i.test(line)) + .join("\n"); + + for (const value of [external?.owner, external?.action]) { + if (!value) continue; + redacted = redacted.replace( + new RegExp(escapeRegExp(value), "gi"), + "[redacted external wait detail]", + ); + } + + redacted = redacted.replace(/\n{3,}/g, "\n\n").trim(); + return redacted.length > 0 ? redacted : null; +} + +function blockedInboxResponseDescription( + attention: IssueBlockedInboxAttention, + row: BlockedInboxIssueRow, +) { + if (!attention.redaction.externalDetailsRedacted) return row.description; + return redactExternalWaitDescription( + row.description, + externalWaitFromDescription(row.description), + ); +} + +function blockedInboxSearchText( + attention: IssueBlockedInboxAttention, + row: BlockedInboxIssueRow, +) { + return [ + row.identifier, + row.title, + blockedInboxResponseDescription(attention, row), + attention.sourceIssue?.identifier, + attention.sourceIssue?.title, + attention.leafIssue?.identifier, + attention.leafIssue?.title, + attention.recoveryIssue?.identifier, + attention.recoveryIssue?.title, + attention.action.label, + attention.action.detail, + ] + .filter( + (value): value is string => typeof value === "string" && value.length > 0, + ) + .join(" ") + .toLowerCase(); +} + +function blockedInboxSeverityRank( + severity: IssueBlockedInboxAttention["severity"], +) { + switch (severity) { + case "critical": + return 0; + case "high": + return 1; + case "medium": + return 2; + case "low": + return 3; + } +} + +function issuePriorityRank(priority: string) { + switch (priority) { + case "critical": + return 0; + case "high": + return 1; + case "medium": + return 2; + case "low": + return 3; + default: + return 4; + } +} + +function compareBlockedInboxRows( + left: BlockedInboxIssueRow & { + blockedInboxAttention: IssueBlockedInboxAttention; + lastActivityAt?: Date | null; + }, + right: BlockedInboxIssueRow & { + blockedInboxAttention: IssueBlockedInboxAttention; + lastActivityAt?: Date | null; + }, +) { + const leftAttention = left.blockedInboxAttention; + const rightAttention = right.blockedInboxAttention; + const severity = + blockedInboxSeverityRank(leftAttention.severity) - + blockedInboxSeverityRank(rightAttention.severity); + if (severity !== 0) return severity; + + const leftStopped = leftAttention.stoppedSinceAt + ? new Date(leftAttention.stoppedSinceAt).getTime() + : Number.POSITIVE_INFINITY; + const rightStopped = rightAttention.stoppedSinceAt + ? new Date(rightAttention.stoppedSinceAt).getTime() + : Number.POSITIVE_INFINITY; + if (leftStopped !== rightStopped) return leftStopped - rightStopped; + + const priority = + issuePriorityRank(left.priority) - issuePriorityRank(right.priority); + if (priority !== 0) return priority; + + const leftActivity = left.lastActivityAt + ? new Date(left.lastActivityAt).getTime() + : new Date(left.updatedAt).getTime(); + const rightActivity = right.lastActivityAt + ? new Date(right.lastActivityAt).getTime() + : new Date(right.updatedAt).getTime(); + if (leftActivity !== rightActivity) return rightActivity - leftActivity; + + return right.id.localeCompare(left.id); +} + +async function listIssueBlockedInboxAttentionMap( + dbOrTx: any, + companyId: string, + issueRows: BlockedInboxIssueRow[], +): Promise> { + const rowIssueIds = [...new Set(issueRows.map((row) => row.id))]; + const result = new Map(); + if (rowIssueIds.length === 0) return result; + + const [graphIssueRows, graphRelationRows, companyAgentRows] = + await Promise.all([ + dbOrTx + .select() + .from(issues) + .where( + and( + eq(issues.companyId, companyId), + visibleIssueCondition(), + ne(issues.status, "done"), + ), + ), + dbOrTx + .select({ + companyId: issueRelations.companyId, + blockerIssueId: issueRelations.issueId, + blockedIssueId: issueRelations.relatedIssueId, + }) + .from(issueRelations) + .where( + and( + eq(issueRelations.companyId, companyId), + eq(issueRelations.type, "blocks"), + ), + ), + dbOrTx + .select({ + id: agents.id, + companyId: agents.companyId, + name: agents.name, + role: agents.role, + title: agents.title, + status: agents.status, + reportsTo: agents.reportsTo, + }) + .from(agents) + .where(eq(agents.companyId, companyId)), + ]); + + const graphIssues = graphIssueRows as IssueRow[]; + const graphRelations = graphRelationRows as Array<{ + companyId: string; + blockerIssueId: string; + blockedIssueId: string; + }>; + const companyAgents = companyAgentRows as Array<{ + id: string; + companyId: string; + name: string; + role: string; + title: string | null; + status: string; + reportsTo: string | null; + }>; + const graphIssueIds = graphIssues.map((issue) => issue.id); + const issuesById = new Map( + graphIssues.map((issue) => [issue.id, issue]), + ); + + const [ + activeRunRows, + wakeRows, + scheduledRetryRows, + interactionRows, + approvalRows, + handoffMap, + ] = await Promise.all([ + graphIssueIds.length === 0 + ? Promise.resolve([]) + : dbOrTx + .select({ + companyId: heartbeatRuns.companyId, + issueId: sql`coalesce( + ${heartbeatRuns.contextSnapshot} ->> 'issueId', + ${heartbeatRuns.contextSnapshot} ->> 'taskId' + )`, + agentId: heartbeatRuns.agentId, + status: heartbeatRuns.status, + }) + .from(heartbeatRuns) + .where( + and( + eq(heartbeatRuns.companyId, companyId), + inArray(heartbeatRuns.status, [ + ...BLOCKED_INBOX_ACTIVE_RUN_STATUSES, + ]), + inArray( + sql`coalesce( + ${heartbeatRuns.contextSnapshot} ->> 'issueId', + ${heartbeatRuns.contextSnapshot} ->> 'taskId' + )`, + graphIssueIds, + ), + ), + ), + graphIssueIds.length === 0 + ? Promise.resolve([]) + : dbOrTx + .select({ + companyId: agentWakeupRequests.companyId, + issueId: sql`coalesce( + ${agentWakeupRequests.payload} ->> 'issueId', + ${agentWakeupRequests.payload} ->> 'taskId', + ${agentWakeupRequests.payload} -> '_paperclipWakeContext' ->> 'issueId', + ${agentWakeupRequests.payload} -> '_paperclipWakeContext' ->> 'taskId' + )`, + agentId: agentWakeupRequests.agentId, + status: agentWakeupRequests.status, + }) + .from(agentWakeupRequests) + .where( + and( + eq(agentWakeupRequests.companyId, companyId), + inArray(agentWakeupRequests.status, [ + ...BLOCKED_INBOX_ACTIVE_WAKE_STATUSES, + ]), + inArray( + sql`coalesce( + ${agentWakeupRequests.payload} ->> 'issueId', + ${agentWakeupRequests.payload} ->> 'taskId', + ${agentWakeupRequests.payload} -> '_paperclipWakeContext' ->> 'issueId', + ${agentWakeupRequests.payload} -> '_paperclipWakeContext' ->> 'taskId' + )`, + graphIssueIds, + ), + ), + ), + graphIssueIds.length === 0 + ? Promise.resolve([]) + : dbOrTx + .select({ + companyId: heartbeatRuns.companyId, + issueId: sql`coalesce( + ${heartbeatRuns.contextSnapshot} ->> 'issueId', + ${heartbeatRuns.contextSnapshot} ->> 'taskId' + )`, + agentId: heartbeatRuns.agentId, + status: heartbeatRuns.status, + }) + .from(heartbeatRuns) + .where( + and( + eq(heartbeatRuns.companyId, companyId), + eq(heartbeatRuns.status, "scheduled_retry"), + inArray( + sql`coalesce( + ${heartbeatRuns.contextSnapshot} ->> 'issueId', + ${heartbeatRuns.contextSnapshot} ->> 'taskId' + )`, + graphIssueIds, + ), + ), + ), + graphIssueIds.length === 0 + ? Promise.resolve([]) + : dbOrTx + .select({ + id: issueThreadInteractions.id, + issueId: issueThreadInteractions.issueId, + kind: issueThreadInteractions.kind, + createdAt: issueThreadInteractions.createdAt, + }) + .from(issueThreadInteractions) + .where( + and( + eq(issueThreadInteractions.companyId, companyId), + inArray(issueThreadInteractions.status, [ + ...BLOCKED_INBOX_PENDING_INTERACTION_STATUSES, + ]), + inArray(issueThreadInteractions.issueId, graphIssueIds), + ), + ), + graphIssueIds.length === 0 + ? Promise.resolve([]) + : dbOrTx + .select({ + approvalId: approvals.id, + issueId: issueApprovals.issueId, + createdAt: approvals.createdAt, + }) + .from(issueApprovals) + .innerJoin(approvals, eq(issueApprovals.approvalId, approvals.id)) + .where( + and( + eq(issueApprovals.companyId, companyId), + eq(approvals.companyId, companyId), + inArray(approvals.status, [ + ...BLOCKED_INBOX_PENDING_APPROVAL_STATUSES, + ]), + inArray(issueApprovals.issueId, graphIssueIds), + ), + ), + listSuccessfulRunHandoffMapForIssues(dbOrTx, companyId, rowIssueIds, { + hydrateLiveness: false, + }), + ]); + + const pendingInteractions = ( + interactionRows as BlockedInboxInteractionRow[] + ).map((row) => ({ + companyId, + issueId: row.issueId, + status: "pending", + })); + const pendingApprovals = (approvalRows as BlockedInboxApprovalRow[]).map( + (row) => ({ + companyId, + issueId: row.issueId, + status: "pending", + }), + ); + + const openRecoveryIssues = graphIssues + .filter((issue) => + BLOCKED_INBOX_RECOVERY_ORIGIN_KINDS.includes( + issue.originKind as (typeof BLOCKED_INBOX_RECOVERY_ORIGIN_KINDS)[number], + ), + ) + .filter( + (issue) => + !BLOCKED_INBOX_TERMINAL_STATUSES.includes( + issue.status as (typeof BLOCKED_INBOX_TERMINAL_STATUSES)[number], + ), + ) + .flatMap((issue) => { + const entries = [{ companyId, issueId: issue.id, status: issue.status }]; + if (issue.originKind === "harness_liveness_escalation") { + const parsed = parseIssueGraphLivenessIncidentKey(issue.originId); + if (parsed?.companyId === companyId) { + entries.push({ + companyId, + issueId: parsed.issueId, + status: issue.status, + }); + entries.push({ + companyId, + issueId: parsed.leafIssueId, + status: issue.status, + }); + } + } else if ( + issue.originKind === "stranded_issue_recovery" && + issue.originId + ) { + entries.push({ + companyId, + issueId: issue.originId, + status: issue.status, + }); + } + return entries; + }); + + const findings = classifyIssueGraphLiveness({ + externalResponseWaits: await listExternalReviewWaits(dbOrTx, companyId, graphIssues.filter(issue => issue.status === "in_review").map(issue => issue.id)), + issues: graphIssues.map((issue) => ({ + id: issue.id, + companyId: issue.companyId, + identifier: issue.identifier, + title: issue.title, + status: issue.status, + projectId: issue.projectId, + goalId: issue.goalId, + parentId: issue.parentId, + assigneeAgentId: issue.assigneeAgentId, + assigneeUserId: issue.assigneeUserId, + createdByAgentId: issue.createdByAgentId, + createdByUserId: issue.createdByUserId, + conversationAgentId: issue.conversationAgentId, + conversationUserId: issue.conversationUserId, + conversationState: issue.conversationState, + executionPolicy: issue.executionPolicy, + executionState: issue.executionState, + monitorNextCheckAt: issue.monitorNextCheckAt, + monitorAttemptCount: issue.monitorAttemptCount, + })), + relations: graphRelations, + agents: companyAgents, + activeRuns: ( + activeRunRows as Array<{ + companyId: string; + issueId: string | null; + agentId: string | null; + status: string; + }> + ).flatMap((row) => + row.issueId + ? [ + { + companyId: row.companyId, + issueId: row.issueId, + agentId: row.agentId, + status: row.status, + }, + ] + : [], + ), + queuedWakeRequests: [ + ...(wakeRows as Array<{ + companyId: string; + issueId: string | null; + agentId: string | null; + status: string; + }>), + ...(scheduledRetryRows as Array<{ + companyId: string; + issueId: string | null; + agentId: string | null; + status: string; + }>), + ].flatMap((row) => + row.issueId + ? [ + { + companyId: row.companyId, + issueId: row.issueId, + agentId: row.agentId, + status: row.status, + }, + ] + : [], + ), + pendingInteractions, + pendingApprovals, + openRecoveryIssues, + now: new Date(), + }); + const findingByIssueId = new Map(); + for (const finding of findings) { + if (!findingByIssueId.has(finding.issueId)) + findingByIssueId.set(finding.issueId, finding); + } + + const interactionByIssueId = new Map(); + for (const row of interactionRows as BlockedInboxInteractionRow[]) { + if (!interactionByIssueId.has(row.issueId)) + interactionByIssueId.set(row.issueId, row); + } + const approvalByIssueId = new Map(); + for (const row of approvalRows as BlockedInboxApprovalRow[]) { + if (!approvalByIssueId.has(row.issueId)) + approvalByIssueId.set(row.issueId, row); + } + const liveHandoffRunIssueIds = new Set( + [ + ...(activeRunRows as Array<{ issueId: string | null }>), + ...(scheduledRetryRows as Array<{ issueId: string | null }>), + ].flatMap((row) => (row.issueId ? [row.issueId] : [])), + ); + const liveHandoffWakeIssueIds = new Set( + (wakeRows as Array<{ issueId: string | null }>).flatMap((row) => + row.issueId ? [row.issueId] : [], + ), + ); + + for (const row of issueRows) { + if ( + row.companyId !== companyId || + BLOCKED_INBOX_TERMINAL_STATUSES.includes( + row.status as (typeof BLOCKED_INBOX_TERMINAL_STATUSES)[number], + ) || + row.hiddenAt + ) { + continue; + } + const source = issueRef(row); + const handoff = handoffMap.get(row.id); + const hasLiveHandoffContinuation = Boolean( + (handoff?.state === "required" || handoff?.state === "escalated") && + (liveHandoffRunIssueIds.has(row.id) || + liveHandoffWakeIssueIds.has(row.id)), + ); + if ( + handoff && + !hasLiveHandoffContinuation && + (handoff.required || handoff.state === "escalated") + ) { + result.set( + row.id, + attentionBase({ + state: "missing_disposition", + reason: "missing_successful_run_disposition", + severity: "high", + stoppedSinceAt: handoff.createdAt ?? row.updatedAt, + owner: { + type: row.assigneeAgentId + ? "agent" + : row.assigneeUserId + ? "user" + : "unknown", + agentId: row.assigneeAgentId, + userId: row.assigneeUserId, + label: null, + }, + action: { + label: "Choose disposition", + detail: + "Choose exactly one final disposition: done, cancelled, review/input, blocked with owner, delegated follow-up, or queued continuation.", + }, + sourceIssue: source, + }), + ); + continue; + } + + if ( + BLOCKED_INBOX_RECOVERY_ORIGIN_KINDS.includes( + row.originKind as (typeof BLOCKED_INBOX_RECOVERY_ORIGIN_KINDS)[number], + ) + ) { + let sourceIssue: IssueBlockedInboxIssueRef | null = null; + let leafIssue: IssueBlockedInboxIssueRef | null = null; + if (row.originKind === "harness_liveness_escalation") { + const parsed = parseIssueGraphLivenessIncidentKey(row.originId); + if (parsed?.companyId === companyId) { + sourceIssue = issueRef(issuesById.get(parsed.issueId)); + leafIssue = issueRef(issuesById.get(parsed.leafIssueId)); + } + } else if (row.originKind === "stranded_issue_recovery" && row.originId) { + sourceIssue = issueRef(issuesById.get(row.originId)); + } + result.set( + row.id, + attentionBase({ + state: "recovery_open", + reason: "open_recovery_issue", + severity: "high", + stoppedSinceAt: row.createdAt, + owner: { + type: row.assigneeAgentId + ? "agent" + : row.assigneeUserId + ? "user" + : "unknown", + agentId: row.assigneeAgentId, + userId: row.assigneeUserId, + label: null, + }, + action: { + label: "Resolve recovery", + detail: + "Restore a live path for the source work or record why this recovery issue is a false positive.", + }, + sourceIssue: sourceIssue ?? source, + leafIssue, + recoveryIssue: source, + }), + ); + continue; + } + + const interaction = interactionByIssueId.get(row.id); + if (interaction) { + const isUserQuestion = + interaction.kind === "ask_user_questions" && + Boolean(row.assigneeUserId); + result.set( + row.id, + attentionBase({ + state: "awaiting_decision", + reason: isUserQuestion + ? "pending_user_decision" + : "pending_board_decision", + severity: "medium", + stoppedSinceAt: interaction.createdAt, + owner: isUserQuestion + ? { + type: "user", + agentId: null, + userId: row.assigneeUserId, + label: null, + } + : { type: "board", agentId: null, userId: null, label: "Board" }, + action: { + label: isUserQuestion ? "Answer question" : "Answer confirmation", + detail: + "Respond to the pending issue-thread interaction so the assignee has a live next action.", + }, + sourceIssue: source, + interactionId: interaction.id, + }), + ); + continue; + } + + const approval = approvalByIssueId.get(row.id); + if (approval) { + result.set( + row.id, + attentionBase({ + state: "awaiting_decision", + reason: "pending_board_decision", + severity: "medium", + stoppedSinceAt: approval.createdAt, + owner: { type: "board", agentId: null, userId: null, label: "Board" }, + action: { + label: "Decide approval", + detail: + "Approve, reject, or request revision on the linked approval.", + }, + sourceIssue: source, + approvalId: approval.approvalId, + }), + ); + continue; + } + + const finding = findingByIssueId.get(row.id); + if (finding) { + const leaf = + finding.dependencyPath.length > 1 + ? issuesById.get( + finding.dependencyPath[finding.dependencyPath.length - 1]! + .issueId, + ) + : issuesById.get(finding.recoveryIssueId); + const ownerAgentId = + finding.state === "blocked_by_unassigned_issue" + ? null + : (finding.recommendedOwnerAgentId ?? + row.assigneeAgentId ?? + leaf?.assigneeAgentId ?? + null); + result.set( + row.id, + attentionBase({ + state: "needs_attention", + reason: finding.state as IssueBlockedInboxAttention["reason"], + severity: + finding.state === "blocked_by_assigned_backlog_issue" || + finding.state === "in_review_without_action_path" + ? "high" + : finding.severity === "critical" + ? "critical" + : "high", + stoppedSinceAt: leaf?.updatedAt ?? row.updatedAt, + owner: { + type: ownerAgentId + ? "agent" + : leaf?.assigneeUserId + ? "user" + : "unknown", + agentId: ownerAgentId, + userId: leaf?.assigneeUserId ?? null, + label: null, + }, + action: { + label: (() => { + switch (finding.state) { + case "blocked_by_unassigned_issue": + return "Assign blocker"; + case "blocked_by_assigned_backlog_issue": + return "Resume parked blocker"; + case "blocked_by_uninvokable_assignee": + return "Assign active owner"; + case "blocked_by_cancelled_issue": + return "Replace blocker"; + case "invalid_review_participant": + return "Repair review participant"; + case "in_review_without_action_path": + return "Choose review path"; + } + })(), + detail: finding.recommendedAction, + }, + sourceIssue: source, + leafIssue: issueRef(leaf), + recoveryIssue: issueRef(issuesById.get(finding.recoveryIssueId)), + sampleIssueIdentifier: leaf?.identifier ?? finding.identifier, + }), + ); + continue; + } + + const hasMonitor = Boolean( + row.monitorNextCheckAt && row.monitorNextCheckAt.getTime() > Date.now(), + ); + const external = + row.status === "blocked" && !hasMonitor + ? externalWaitFromDescription(row.description) + : null; + if (external) { + result.set( + row.id, + attentionBase({ + state: "external_wait", + reason: "external_owner_action", + severity: "medium", + stoppedSinceAt: row.updatedAt, + owner: { type: "external", agentId: null, userId: null, label: null }, + action: { + label: "External owner action", + detail: null, + }, + sourceIssue: source, + externalDetailsRedacted: true, + }), + ); + continue; + } + + const blockerAttention = await listIssueBlockerAttentionMap( + dbOrTx, + companyId, + [row], + ); + const blockerState = blockerAttention.get(row.id); + if ( + row.status === "blocked" && + (blockerState?.state === "needs_attention" || + blockerState?.state === "stalled") + ) { + result.set( + row.id, + attentionBase({ + state: "needs_attention", + reason: "blocked_chain_stalled", + severity: "high", + stoppedSinceAt: row.updatedAt, + owner: { type: "unknown", agentId: null, userId: null, label: null }, + action: { + label: "Inspect blocker chain", + detail: + "Inspect the stalled blocker or review leaf and make the next owner/action explicit.", + }, + sourceIssue: source, + sampleIssueIdentifier: + blockerState.sampleStalledBlockerIdentifier ?? + blockerState.sampleBlockerIdentifier, + }), + ); + } + } + + return result; +} + +function parseIssueAssigneeAgentFilter( + assigneeAgentId: IssueFilters["assigneeAgentId"], +): string | null | undefined { + const normalizedRaw = + typeof assigneeAgentId === "string" + ? assigneeAgentId.trim() + : assigneeAgentId; + const normalized = normalizedRaw === "" ? undefined : normalizedRaw; + if (typeof normalized !== "string") return normalized; + return normalized.toLowerCase() === "null" ? null : normalized; +} + +function assertValidAssigneeAgentFilter( + assigneeAgentFilter: string | null | undefined, +) { + if ( + typeof assigneeAgentFilter === "string" && + !isUuidLike(assigneeAgentFilter) + ) { + throw unprocessable("assigneeAgentId must be a UUID or 'null'"); + } +} + +function creationVisibilityCondition(companyId: string, filters?: IssueFilters) { + if (filters?.includeAgentCreatedTasks !== false) return undefined; + return filters.creationVisibilityUserId + ? or(humanCreatedIssueCondition(), touchedByUserCondition(companyId, filters.creationVisibilityUserId))! + : humanCreatedIssueCondition(); +} + +async function blockedInboxIssueConditions( + dbOrTx: any, + companyId: string, + filters?: IssueFilters, +) { + const conditions = [ + eq(issues.companyId, companyId), + visibleIssueCondition(), + notInArray(issues.status, [...BLOCKED_INBOX_TERMINAL_STATUSES]), + ]; + if (filters?.readCondition) conditions.push(filters.readCondition); + const creationCondition = creationVisibilityCondition(companyId, filters); + if (creationCondition) conditions.push(creationCondition); + const touchedByUserId = filters?.touchedByUserId?.trim() || undefined; + const inboxArchivedByUserId = + filters?.inboxArchivedByUserId?.trim() || undefined; + const unreadForUserId = filters?.unreadForUserId?.trim() || undefined; + const contextUserId = + unreadForUserId ?? touchedByUserId ?? inboxArchivedByUserId; + + if (filters?.createdFromIssueId) { + conditions.push(createdFromIssueCondition(companyId, filters.createdFromIssueId)); + } + if (filters?.descendantOf) { + conditions.push(sql` + ${issues.id} IN ( + WITH RECURSIVE descendants(id) AS ( + SELECT ${issues.id} + FROM ${issues} + WHERE ${issues.companyId} = ${companyId} + AND ${issues.parentId} = ${filters.descendantOf} + UNION + SELECT ${issues.id} + FROM ${issues} + JOIN descendants ON ${issues.parentId} = descendants.id + WHERE ${issues.companyId} = ${companyId} + ) + SELECT id FROM descendants + ) + `); + } + const lowTrustCondition = lowTrustBoundaryIssueCondition( + companyId, + filters?.lowTrustBoundary, + ); + if (lowTrustCondition) conditions.push(lowTrustCondition); + const statuses = parseStatusFilter(filters?.status); + if (statuses.length > 0) { + conditions.push( + statuses.length === 1 + ? eq(issues.status, statuses[0]!) + : inArray(issues.status, statuses), + ); + } + const assigneeAgentFilter = parseIssueAssigneeAgentFilter( + filters?.assigneeAgentId, + ); + assertValidAssigneeAgentFilter(assigneeAgentFilter); + if (assigneeAgentFilter === null) { + conditions.push(isNull(issues.assigneeAgentId)); + } else if (assigneeAgentFilter) { + conditions.push(eq(issues.assigneeAgentId, assigneeAgentFilter)); + } + if (filters?.participantAgentId) + conditions.push( + participatedByAgentCondition(companyId, filters.participantAgentId), + ); + if (filters?.assigneeUserId) + conditions.push(eq(issues.assigneeUserId, filters.assigneeUserId)); + if (touchedByUserId) + conditions.push(touchedByUserCondition(companyId, touchedByUserId)); + if (inboxArchivedByUserId) + conditions.push( + inboxVisibleForUserCondition(companyId, inboxArchivedByUserId), + ); + if (unreadForUserId) + conditions.push(unreadForUserCondition(companyId, unreadForUserId)); + if (filters?.projectId) + conditions.push(eq(issues.projectId, filters.projectId)); + if (filters?.workspaceId) { + conditions.push( + or( + eq(issues.executionWorkspaceId, filters.workspaceId), + eq(issues.projectWorkspaceId, filters.workspaceId), + )!, + ); + } + if (filters?.executionWorkspaceId) + conditions.push( + eq(issues.executionWorkspaceId, filters.executionWorkspaceId), + ); + if (filters?.parentId) conditions.push(eq(issues.parentId, filters.parentId)); + if (filters?.originKind) + conditions.push(eq(issues.originKind, filters.originKind)); + if (filters?.originKindPrefix) + conditions.push(like(issues.originKind, `${filters.originKindPrefix}%`)); + if (filters?.originId) conditions.push(eq(issues.originId, filters.originId)); + if (filters?.hasPlanDocument !== undefined) { + conditions.push( + hasPlanDocumentCondition(companyId, filters.hasPlanDocument), + ); + } + if (!shouldIncludePluginOperationIssues(filters)) + conditions.push(nonPluginOperationIssueCondition()); + if (filters?.labelId) { + const labeledIssueIds = await dbOrTx + .select({ issueId: issueLabels.issueId }) + .from(issueLabels) + .where( + and( + eq(issueLabels.companyId, companyId), + eq(issueLabels.labelId, filters.labelId), + ), + ); + if (labeledIssueIds.length === 0) + return { conditions: [sql`false`], contextUserId }; + conditions.push( + inArray( + issues.id, + labeledIssueIds.map((row: { issueId: string }) => row.issueId), + ), + ); + } + if ( + filters?.excludeRoutineExecutions && + !filters?.originKind && + !filters?.originId + ) { + conditions.push(ne(issues.originKind, "routine_execution")); + } + + return { conditions, contextUserId }; +} + +async function listBlockedInboxIssues( + dbOrTx: any, + companyId: string, + filters?: IssueFilters, +): Promise< + Array< + IssueWithLabelsAndRun & { + blockedBy?: IssueRelationIssueSummary[]; + blockerAttention?: IssueBlockerAttention; + reviewAttention?: IssueReviewAttention; + blockedInboxAttention: IssueBlockedInboxAttention; + liveDescendantCount?: number; + lastActivityAt: Date; + myLastTouchAt?: Date | null; + lastExternalCommentAt?: Date | null; + isUnreadForMe?: boolean; + } + > +> { + const { conditions, contextUserId } = await blockedInboxIssueConditions( + dbOrTx, + companyId, + filters, + ); + + const rows = ( + await dbOrTx + .select(issueListSelect) + .from(issues) + .where(and(...conditions)) + .orderBy( + desc(issueCanonicalLastActivityAtExpr(companyId)), + desc(issues.updatedAt), + desc(issues.id), + ) + ).map((row: any) => ({ + ...row, + description: decodeDatabaseTextPreview( + row.description, + ISSUE_LIST_DESCRIPTION_MAX_CHARS, + ), + })); + const withLabels = await withIssueLabels(dbOrTx, rows); + const withRuns = withActiveRuns( + withLabels, + await activeRunMapForIssues(dbOrTx, withLabels), + ); + if (withRuns.length === 0) return []; + + const issueIds = withRuns.map((row) => row.id); + const includeLiveDescendantSummary = + filters?.includeLiveDescendantSummary === true; + const [ + statsRows, + readRows, + lastActivityRows, + blockedByMap, + blockerAttentionByIssueId, + reviewAttentionByIssueId, + blockedInboxAttentionByIssueId, + liveDescendantCountByIssueId, + ] = await Promise.all([ + contextUserId + ? userCommentStatsForIssues(dbOrTx, companyId, contextUserId, issueIds) + : Promise.resolve([]), + contextUserId + ? userReadStatsForIssues(dbOrTx, companyId, contextUserId, issueIds) + : Promise.resolve([]), + lastActivityStatsForIssues(dbOrTx, companyId, issueIds), + blockedByMapForIssues(dbOrTx, companyId, issueIds), + listIssueBlockerAttentionMap(dbOrTx, companyId, withRuns), + listIssueReviewAttentionMap(dbOrTx, companyId, withRuns), + listIssueBlockedInboxAttentionMap(dbOrTx, companyId, withRuns), + includeLiveDescendantSummary + ? liveDescendantCountMapForIssues(dbOrTx, companyId, issueIds) + : Promise.resolve(new Map()), + ]); + + const rawSearchInput = filters?.q?.trim() ?? ""; + const rawSearch = rawSearchInput.toLowerCase(); + const commentSearchMatchIssueIds = new Set(); + if (rawSearchInput) { + const containsPattern = `%${escapeLikePattern(rawSearchInput)}%`; + for (const issueIdChunk of chunkList( + issueIds, + ISSUE_LIST_RELATED_QUERY_CHUNK_SIZE, + )) { + const rows = await dbOrTx + .select({ issueId: issueComments.issueId }) + .from(issueComments) + .where( + and( + eq(issueComments.companyId, companyId), + inArray(issueComments.issueId, issueIdChunk), + isNull(issueComments.deletedAt), + sql`${issueComments.body} ILIKE ${containsPattern} ESCAPE '\\'`, + ), + ); + for (const row of rows as Array<{ issueId: string }>) + commentSearchMatchIssueIds.add(row.issueId); + } + } + const statsByIssueId = new Map(statsRows.map((row) => [row.issueId, row])); + const readByIssueId = new Map( + readRows.map((row) => [row.issueId, row.myLastReadAt]), + ); + const lastActivityByIssueId = new Map( + lastActivityRows.map((row) => [row.issueId, row]), + ); + + const enriched = withRuns + .flatMap((row) => { + const blockedInboxAttention = blockedInboxAttentionByIssueId.get(row.id); + if (!blockedInboxAttention) return []; + if ( + rawSearch && + !blockedInboxSearchText(blockedInboxAttention, row).includes( + rawSearch, + ) && + !commentSearchMatchIssueIds.has(row.id) + ) + return []; + + const activity = lastActivityByIssueId.get(row.id); + const lastActivityAt = + latestIssueActivityAt( + row.updatedAt, + activity?.latestCommentAt ?? null, + activity?.latestLogAt ?? null, + ) ?? row.updatedAt; + return [ + { + ...row, + description: blockedInboxResponseDescription( + blockedInboxAttention, + row, + ), + blockedBy: blockedByMap.get(row.id) ?? [], + lastActivityAt, + ...(blockerAttentionByIssueId.has(row.id) + ? { blockerAttention: blockerAttentionByIssueId.get(row.id) } + : {}), + reviewAttention: + reviewAttentionByIssueId.get(row.id) ?? reviewAttentionNone(), + blockedInboxAttention, + ...(includeLiveDescendantSummary + ? { + liveDescendantCount: + liveDescendantCountByIssueId.get(row.id) ?? 0, + } + : {}), + ...(contextUserId + ? deriveIssueUserContext(row, contextUserId, { + isUnreadForMe: statsByIssueId.get(row.id)?.isUnreadForMe ?? false, + myLastCommentAt: + statsByIssueId.get(row.id)?.myLastCommentAt ?? null, + myLastReadAt: readByIssueId.get(row.id) ?? null, + lastExternalCommentAt: + statsByIssueId.get(row.id)?.lastExternalCommentAt ?? null, + }) + : {}), + }, + ]; + }) + .sort(compareBlockedInboxRows); + + const offset = + typeof filters?.offset === "number" && Number.isFinite(filters.offset) + ? Math.max(0, Math.floor(filters.offset)) + : 0; + const limit = + typeof filters?.limit === "number" && Number.isFinite(filters.limit) + ? Math.max(1, Math.floor(filters.limit)) + : undefined; + return limit === undefined + ? enriched.slice(offset) + : enriched.slice(offset, offset + limit); +} + +async function countBlockedInboxIssues( + dbOrTx: any, + companyId: string, + filters?: IssueFilters, +): Promise { + const { conditions } = await blockedInboxIssueConditions( + dbOrTx, + companyId, + filters, + ); + const rows = (await dbOrTx + .select() + .from(issues) + .where(and(...conditions))) as IssueRow[]; + if (rows.length === 0) return 0; + + const blockedInboxAttentionByIssueId = + await listIssueBlockedInboxAttentionMap(dbOrTx, companyId, rows); + const rawSearchInput = filters?.q?.trim() ?? ""; + const rawSearch = rawSearchInput.toLowerCase(); + const commentSearchMatchIssueIds = new Set(); + if (rawSearchInput) { + const issueIds = rows.map((row) => row.id); + const containsPattern = `%${escapeLikePattern(rawSearchInput)}%`; + for (const issueIdChunk of chunkList( + issueIds, + ISSUE_LIST_RELATED_QUERY_CHUNK_SIZE, + )) { + const commentRows = await dbOrTx + .select({ issueId: issueComments.issueId }) + .from(issueComments) + .where( + and( + eq(issueComments.companyId, companyId), + inArray(issueComments.issueId, issueIdChunk), + isNull(issueComments.deletedAt), + sql`${issueComments.body} ILIKE ${containsPattern} ESCAPE '\\'`, + ), + ); + for (const row of commentRows as Array<{ issueId: string }>) + commentSearchMatchIssueIds.add(row.issueId); + } + } + + return rows.reduce((count: number, row: IssueRow) => { + const attention = blockedInboxAttentionByIssueId.get(row.id); + if (!attention) return count; + if ( + rawSearch && + !blockedInboxSearchText(attention, row).includes(rawSearch) && + !commentSearchMatchIssueIds.has(row.id) + ) + return count; + return count + 1; + }, 0); +} + +export async function readIssueCommentRunLogText(run: { + runId?: string | null; + logStore: string | null; + logRef: string | null; + logBytes: number | null; +}, db?: Db) { + if (!isRunLogStoreType(run.logStore) || !run.logRef) return ""; + // A timed-out finalization leaves size unknown even when earlier entries + // exist. Read those logs within the same byte budget as a known-size log. + if (run.logBytes !== null && (!Number.isFinite(run.logBytes) || run.logBytes <= 0)) return ""; + + const logRef = run.logRef; + const storeType = run.logStore; + const store = getRunLogStore(); + let offset = 0; + let content = ""; + let nextOffset: number | undefined = 0; + const controller = new AbortController(); + let readTimer: NodeJS.Timeout | undefined; + + const readChunks = async () => { + if (db && storeType === "s3_segments") await recoverSettledNativeRunLog(db, store, logRef, undefined, controller.signal) + .catch(() => { controller.signal.throwIfAborted(); }); + while (nextOffset !== undefined) { + controller.signal.throwIfAborted(); + const remainingBytes = + ISSUE_COMMENT_RUN_LOG_DERIVATION_MAX_LOG_BYTES - + Buffer.byteLength(content, "utf8"); + if (remainingBytes <= 0) break; + const chunk = await store.read( + { store: storeType, logRef }, + { + offset, + limitBytes: Math.min(ISSUE_COMMENT_RUN_LOG_DERIVATION_CHUNK_BYTES, remainingBytes), + signal: controller.signal, + }, + ); + controller.signal.throwIfAborted(); + content += chunk.content; + nextOffset = chunk.nextOffset; + offset = chunk.nextOffset ?? 0; + } + }; + + try { + await Promise.race([ + readChunks(), + new Promise((_resolve, reject) => { + readTimer = setTimeout(() => { + const reason = new DOMException("Attribution log read timed out", "TimeoutError"); + // Cancellation closes storage work where supported, but filesystem + // I/O can delay stream destruction. Keep the response deadline too. + reject(reason); + controller.abort(reason); + }, ISSUE_COMMENT_RUN_LOG_DERIVATION_TIMEOUT_MS); + readTimer.unref?.(); + }), + ]); + } catch (err) { + // Attribution enriches already-authorized comments. Missing, failed, or + // stalled storage must not prevent listing them; keep any evidence read. + // Do not log raw provider errors, which can contain credentialed URLs. + logger.warn( + { runId: run.runId ?? undefined, logRef, status: err instanceof HttpError ? err.status : undefined }, + "could not read heartbeat run log while deriving optional issue comment metadata", + ); + } finally { + clearTimeout(readTimer); + } + + return content; +} + +/** Share task-before-comment lock order with receipt completion and task deletion. */ +async function lockIssueForCommentMutation(tx: DbTransaction, commentId: string) { + const [comment] = await tx.select({ issueId: issueComments.issueId, companyId: issueComments.companyId }) + .from(issueComments).where(eq(issueComments.id, commentId)); + if (!comment) return false; + const [issue] = await tx.select({ id: issues.id }).from(issues).where(and( + eq(issues.id, comment.issueId), eq(issues.companyId, comment.companyId), + )).for("no key update"); + return Boolean(issue); +} + +export function issueService(db: Db) { + const instanceSettings = instanceSettingsService(db); + const treeControlSvc = issueTreeControlService(db); + + function provisionalTitleFromDescription(description: string) { + const simpleTitle = description.trim().replace(/\s+/g, " ").slice(0, 120); + try { + type MarkdownNode = { + type: string; + alt?: string | null; + children?: MarkdownNode[]; + position?: { start: { offset?: number }; end: { offset?: number } }; + }; + const imageRanges: Array<{ start: number; end: number }> = []; + const imageAlts: string[] = []; + const visit = (node: MarkdownNode) => { + if (node.type === "image" || node.type === "imageReference") { + const start = node.position?.start.offset; + const end = node.position?.end.offset; + if (start !== undefined && end !== undefined) imageRanges.push({ start, end }); + if (node.alt?.trim()) imageAlts.push(node.alt.trim()); + } + node.children?.forEach(visit); + }; + visit(parseMarkdown(description) as MarkdownNode); + const withoutImages = imageRanges + .sort((a, b) => b.start - a.start) + .reduce((text, range) => `${text.slice(0, range.start)} ${text.slice(range.end)}`, description); + const plainText = markdownToPlainText(withoutImages).trim().replace(/\s+/g, " "); + const fallback = imageRanges.length > 0 + ? imageAlts.join(" ") || "Image" + : simpleTitle; + return (plainText || fallback).slice(0, 120); + } catch { + return simpleTitle; + } + } + + function normalizeCreateIssueTitle(title: string) { + return title.trim().replace(/\s+/g, " ").toLowerCase(); + } + + async function getIssueByUuid(id: string) { + const row = await retryIdempotentDatabaseOperation(() => db + .select({ ...getTableColumns(issues), externalConversationState: externalConversationStateSql() }) + .from(issues) + .where(eq(issues.id, id)) + .then((rows) => rows[0] ?? null)); + if (!row) return null; + const [enriched] = await withIssueLabels(db, [row]); + return enriched; + } + + async function getIssueByIdentifier(identifier: string) { + const row = await retryIdempotentDatabaseOperation(() => db + .select({ ...getTableColumns(issues), externalConversationState: externalConversationStateSql() }) + .from(issues) + .where(eq(issues.identifier, identifier.toUpperCase())) + .then((rows) => rows[0] ?? null)); + if (!row) return null; + const [enriched] = await withIssueLabels(db, [row]); + return enriched; + } + + async function projectHistoricalRunComments< + T extends { body: string; createdByRunId: string | null }, + >(comments: T[]): Promise { + const runIds = [ + ...new Set( + comments.flatMap((comment) => + comment.createdByRunId && comment.body === LEGACY_WITHHELD_RUN_COMMENT + ? [comment.createdByRunId] + : [], + ), + ), + ]; + if (runIds.length === 0) return comments; + const runResults = await db + .select({ id: heartbeatRuns.id, resultJson: heartbeatRuns.resultJson }) + .from(heartbeatRuns) + .where(inArray(heartbeatRuns.id, runIds)); + const resultByRunId = new Map( + runResults.map((run) => [run.id, parseObject(run.resultJson)]), + ); + return comments.map((comment) => { + if (!comment.createdByRunId) return comment; + const body = projectHistoricalHeartbeatRunComment( + comment.body, + resultByRunId.get(comment.createdByRunId), + ); + return body === comment.body ? comment : { ...comment, body }; + }); + } + + async function getCurrentScheduledRetriesForIssues( + issueIds: string[], + companyId: string, + dbOrTx: DbReader = db, + ): Promise> { + const uniqueIssueIds = [...new Set(issueIds)]; + if (uniqueIssueIds.length === 0) return new Map(); + + const contextIssueId = sql`${heartbeatRuns.contextSnapshot} ->> 'issueId'`; + const rows = await dbOrTx + .select({ + issueId: contextIssueId, + runId: heartbeatRuns.id, + status: heartbeatRuns.status, + agentId: heartbeatRuns.agentId, + agentName: agents.name, + retryOfRunId: heartbeatRuns.retryOfRunId, + scheduledRetryAt: heartbeatRuns.scheduledRetryAt, + scheduledRetryAttempt: heartbeatRuns.scheduledRetryAttempt, + scheduledRetryReason: heartbeatRuns.scheduledRetryReason, + error: heartbeatRuns.error, + errorCode: heartbeatRuns.errorCode, + }) + .from(heartbeatRuns) + .innerJoin(agents, eq(heartbeatRuns.agentId, agents.id)) + .where( + and( + eq(heartbeatRuns.companyId, companyId), + inArray(heartbeatRuns.status, [ + "scheduled_retry", + "queued", + "running", + ]), + isNotNull(heartbeatRuns.scheduledRetryReason), + inArray(contextIssueId, uniqueIssueIds), + ), + ) + .orderBy( + sql`case ${heartbeatRuns.status} + when 'running' then 0 + when 'queued' then 1 + else 2 + end`, + asc(heartbeatRuns.scheduledRetryAt), + asc(heartbeatRuns.createdAt), + asc(heartbeatRuns.id), + ); + + const currentByIssueId = new Map(); + for (const row of rows) { + if (currentByIssueId.has(row.issueId)) continue; + const status = row.status; + if ( + status !== "scheduled_retry" && + status !== "queued" && + status !== "running" + ) + continue; + currentByIssueId.set(row.issueId, { ...row, status }); + } + return currentByIssueId; + } + + async function getCurrentScheduledRetryForIssue( + issueId: string, + companyId: string, + ): Promise { + const currentByIssueId = await getCurrentScheduledRetriesForIssues( + [issueId], + companyId, + ); + return currentByIssueId.get(issueId) ?? null; + } + + function deriveIssueCommentAuthorType(comment: { + authorType?: string | null; + authorAgentId?: string | null; + authorUserId?: string | null; + }): IssueCommentAuthorType { + const explicit = issueCommentAuthorTypeSchema.safeParse(comment.authorType); + if (explicit.success) return explicit.data; + if (comment.authorAgentId) return "agent"; + if (comment.authorUserId) return "user"; + return "system"; + } + + function assertIssueCommentAuthorTypeAllowed( + actor: { agentId?: string | null; userId?: string | null }, + authorType: IssueCommentAuthorType, + ) { + if (actor.agentId && authorType !== "agent") { + throw unprocessable("Comment authorType must match authenticated actor"); + } + if (actor.userId && authorType !== "user") { + throw unprocessable("Comment authorType must match authenticated actor"); + } + if (!actor.agentId && !actor.userId && authorType !== "system") { + throw unprocessable( + "System comments cannot use user or agent authorType without an author id", + ); + } + } + + function redactIssueComment< + T extends { + body: string; + authorType?: string | null; + authorAgentId?: string | null; + authorUserId?: string | null; + presentation?: unknown; + metadata?: unknown; + deletedAt?: Date | string | null; + deletedByType?: "agent" | "user" | null; + deletedByAgentId?: string | null; + deletedByUserId?: string | null; + deletedByRunId?: string | null; + }, + >( + comment: T, + censorUsernameInLogs: boolean, + ): T & { + authorType: IssueCommentAuthorType; + presentation: IssueCommentPresentation | null; + metadata: IssueCommentMetadata | null; + } { + const deletedAt = comment.deletedAt ?? null; + if (deletedAt) { + return { + ...comment, + authorType: deriveIssueCommentAuthorType(comment), + body: "", + presentation: null, + metadata: null, + deletedAt, + deletedByType: comment.deletedByType ?? null, + deletedByAgentId: comment.deletedByAgentId ?? null, + deletedByUserId: comment.deletedByUserId ?? null, + deletedByRunId: comment.deletedByRunId ?? null, + }; + } + + return { + ...comment, + authorType: deriveIssueCommentAuthorType(comment), + body: redactCurrentUserText(comment.body, { + enabled: censorUsernameInLogs, + }), + presentation: issueCommentPresentationSchema + .nullable() + .catch(null) + .parse(comment.presentation ?? null), + metadata: issueCommentMetadataSchema + .nullable() + .catch(null) + .parse(comment.metadata ?? null), + }; + } + + + // Persist a resolved attribution so subsequent reads stop re-scanning run + // logs (and old "Board" threads stay fixed durably). Best-effort: a write + // failure must never break the read path. The `IS NULL` guard keeps this + // idempotent and avoids clobbering a value another reader just stored. + async function persistDerivedIssueCommentAttribution( + derivedByCommentId: ReadonlyMap, + ) { + if (derivedByCommentId.size === 0) return; + // One bulk `UPDATE ... FROM (VALUES ...)` so the read path is never blocked + // on N sequential round-trips for a large legacy thread. The `IS NULL` guard + // keeps this idempotent and avoids clobbering a value another reader just + // stored. Best-effort: a write failure must never break the read path. + const rows = [...derivedByCommentId].map( + ([commentId, derived]) => + sql`(${commentId}::uuid, ${derived.derivedAuthorAgentId}::uuid, ${derived.derivedCreatedByRunId}::uuid, ${derived.derivedAuthorSource}::text)`, + ); + try { + await db.execute(sql` + UPDATE ${issueComments} AS c + SET derived_author_agent_id = v.agent_id, + derived_created_by_run_id = v.run_id, + derived_author_source = v.source + FROM (VALUES ${sql.join(rows, sql`, `)}) AS v(comment_id, agent_id, run_id, source) + WHERE c.id = v.comment_id AND c.derived_author_agent_id IS NULL + `); + } catch (err) { + logger.warn( + { err, commentIds: [...derivedByCommentId.keys()] }, + "failed to persist derived issue-comment attribution", + ); + } + } + + async function enrichCommentsWithDerivedAgentAttribution< + T extends { + id: string; + companyId: string; + issueId: string; + authorAgentId?: string | null; + authorUserId?: string | null; + createdByRunId?: string | null; + derivedAuthorAgentId?: string | null; + createdAt: Date | string; + }, + >(comments: readonly T[]) { + // Candidates: a non-human author, no stored agent, and not already resolved + // by a previous read / the backfill migration. + const preliminary = comments.filter( + (comment) => + !comment.authorAgentId && + !!comment.authorUserId && + !comment.derivedAuthorAgentId, + ); + if (preliminary.length === 0) return comments; + + const companyId = comments[0]?.companyId ?? null; + const issueId = comments[0]?.issueId ?? null; + if (!companyId || !issueId) return comments; + + // Guard: never reattribute a comment whose author maps to a genuine user + // profile. Only the non-human sentinels agents post under (e.g. + // `local-board`) are eligible — even though `local-board` is itself a row in + // the `user` table, so a plain "exists in user table" check would wrongly + // exclude every mis-attributed agent comment. + const nonSentinelAuthorUserIds = [ + ...new Set( + preliminary + .map((comment) => comment.authorUserId) + .filter( + (id): id is string => + !!id && !NON_HUMAN_SENTINEL_AUTHOR_USER_IDS.has(id), + ), + ), + ]; + const genuineUserIds = nonSentinelAuthorUserIds.length + ? new Set( + ( + await db + .select({ id: authUsers.id }) + .from(authUsers) + .where(inArray(authUsers.id, nonSentinelAuthorUserIds)) + ).map((row) => row.id), + ) + : new Set(); + // `preliminary` already guarantees a truthy `authorUserId`, so only the two + // "not a genuine user" arms are live: the explicit non-human sentinel, or an + // author id absent from the `user` table. + const candidates = preliminary.filter( + (comment) => + NON_HUMAN_SENTINEL_AUTHOR_USER_IDS.has(comment.authorUserId!) || + !genuineUserIds.has(comment.authorUserId!), + ); + if (candidates.length === 0) return comments; + + const minCommentCreatedAtMs = candidates.reduce( + (min, comment) => { + const timestamp = toTimestampMs(comment.createdAt); + if (timestamp === null) return min; + return min === null ? timestamp : Math.min(min, timestamp); + }, + null, + ); + const maxCommentCreatedAtMs = candidates.reduce( + (max, comment) => { + const timestamp = toTimestampMs(comment.createdAt); + if (timestamp === null) return max; + return max === null ? timestamp : Math.max(max, timestamp); + }, + null, + ); + if (minCommentCreatedAtMs === null || maxCommentCreatedAtMs === null) + return comments; + + const minCommentCreatedAt = new Date(minCommentCreatedAtMs).toISOString(); + const maxCommentCreatedAt = new Date( + maxCommentCreatedAtMs + ISSUE_COMMENT_RUN_LOG_DERIVATION_END_SLACK_MS, + ).toISOString(); + + // The runs the comments' own `createdByRunId` point at — fetched + // unconditionally so the lossless run-id tier resolves even when a run is + // not otherwise associated with the issue. + const ownRunIds = [ + ...new Set( + candidates + .map((comment) => comment.createdByRunId) + .filter((id): id is string => !!id), + ), + ]; + + const runs = await db + .select({ + runId: heartbeatRuns.id, + agentId: heartbeatRuns.agentId, + createdAt: heartbeatRuns.createdAt, + startedAt: heartbeatRuns.startedAt, + finishedAt: heartbeatRuns.finishedAt, + logStore: heartbeatRuns.logStore, + logRef: heartbeatRuns.logRef, + logBytes: heartbeatRuns.logBytes, + }) + .from(heartbeatRuns) + .where( + and( + eq(heartbeatRuns.companyId, companyId), + or( + and( + or( + sql`${heartbeatRuns.contextSnapshot} ->> 'issueId' = ${issueId}`, + sql`exists ( + select 1 + from ${activityLog} + where ${activityLog.companyId} = ${companyId} + and ${activityLog.entityType} = 'issue' + and ${activityLog.entityId} = ${issueId} + and ${activityLog.runId} = ${heartbeatRuns.id} + )`, + ), + sql`coalesce(${heartbeatRuns.finishedAt}, ${heartbeatRuns.createdAt}) >= ${minCommentCreatedAt}::timestamptz`, + sql`coalesce(${heartbeatRuns.startedAt}, ${heartbeatRuns.createdAt}) <= ${maxCommentCreatedAt}::timestamptz`, + ), + ownRunIds.length > 0 + ? inArray(heartbeatRuns.id, ownRunIds) + : sql`false`, + ), + ), + ) + .orderBy(desc(heartbeatRuns.createdAt)); + + if (runs.length === 0) return comments; + + // Pass 1: resolve the run-id tier, which never reads log bodies. Most + // comments resolve here, so we avoid object-storage reads entirely. + const runsWithoutLogs = runs.map((run) => ({ ...run, logContent: "" })); + const derivedByCommentId = new Map( + deriveIssueCommentRunLogAttribution(candidates, runsWithoutLogs), + ); + + // Pass 2: for comments still unresolved after the run-id tier, read the logs + // of any run whose window overlaps such a comment, to look for the explicit + // `comment id:` post marker. The marker is a lossless signal regardless of + // how many runs overlap, so we do not short-circuit on the single-run case. + const unresolved = candidates.filter( + (comment) => !derivedByCommentId.has(comment.id), + ); + if (unresolved.length > 0) { + const runIdsToRead = new Set(); + for (const run of runs) { + const runStartMs = toTimestampMs(run.startedAt ?? run.createdAt); + const runEndMs = toTimestampMs(run.finishedAt ?? run.createdAt); + if (runStartMs === null || runEndMs === null) continue; + for (const comment of unresolved) { + const commentCreatedAtMs = toTimestampMs(comment.createdAt); + if (commentCreatedAtMs === null) continue; + if ( + commentCreatedAtMs >= runStartMs && + commentCreatedAtMs <= + runEndMs + ISSUE_COMMENT_RUN_LOG_DERIVATION_END_SLACK_MS + ) { + runIdsToRead.add(run.runId); + break; + } + } + } + + if (runIdsToRead.size > 0) { + const runsToRead = runs.filter((run) => runIdsToRead.has(run.runId)); + const logByRunId = new Map(); + for ( + let index = 0; + index < runsToRead.length; + index += ISSUE_COMMENT_RUN_LOG_DERIVATION_MAX_PARALLEL_READS + ) { + const batch = runsToRead.slice( + index, + index + ISSUE_COMMENT_RUN_LOG_DERIVATION_MAX_PARALLEL_READS, + ); + await Promise.all( + batch.map(async (run) => { + logByRunId.set(run.runId, await readIssueCommentRunLogText(run, db)); + }), + ); + } + const runsWithLogs = runs.map((run) => ({ + ...run, + logContent: logByRunId.get(run.runId) ?? "", + })); + for (const [commentId, derived] of deriveIssueCommentRunLogAttribution( + unresolved, + runsWithLogs, + )) { + derivedByCommentId.set(commentId, derived); + } + } + } + + if (derivedByCommentId.size === 0) return comments; + + await persistDerivedIssueCommentAttribution(derivedByCommentId); + + return comments.map((comment) => { + const derived = derivedByCommentId.get(comment.id); + return derived ? { ...comment, ...derived } : comment; + }); + } + + async function isTreeHoldInteractionCheckoutAllowed( + companyId: string, + checkoutRunId: string | null, + _gate: ActiveIssueTreePauseHoldGate, + ) { + if (!checkoutRunId) return false; + const run = await db + .select({ + id: heartbeatRuns.id, + agentId: heartbeatRuns.agentId, + wakeupRequestId: heartbeatRuns.wakeupRequestId, + contextSnapshot: heartbeatRuns.contextSnapshot, + }) + .from(heartbeatRuns) + .where( + and( + eq(heartbeatRuns.id, checkoutRunId), + eq(heartbeatRuns.companyId, companyId), + ), + ) + .then((rows) => rows[0] ?? null); + const issueId = readStringFromRecord(run?.contextSnapshot, "issueId"); + if (!run || !issueId) return false; + return isVerifiedIssueTreeControlInteractionWake(db, { + companyId, + issueId, + agentId: run.agentId, + runId: run.id, + wakeupRequestId: run.wakeupRequestId, + contextSnapshot: run.contextSnapshot as + Record | null | undefined, + }); + } + + async function assertAssignableUser(companyId: string, userId: string) { + const membership = await db + .select({ id: companyMemberships.id }) + .from(companyMemberships) + .where( + and( + eq(companyMemberships.companyId, companyId), + eq(companyMemberships.principalType, "user"), + eq(companyMemberships.principalId, userId), + eq(companyMemberships.status, "active"), + ), + ) + .then((rows) => rows[0] ?? null); + if (!membership) { + throw notFound("Assignee user not found"); + } + } + + async function assertValidProjectWorkspace( + companyId: string, + projectId: string | null | undefined, + projectWorkspaceId: string, + dbOrTx: DbReader = db, + ) { + const workspace = await dbOrTx + .select({ + id: projectWorkspaces.id, + companyId: projectWorkspaces.companyId, + projectId: projectWorkspaces.projectId, + }) + .from(projectWorkspaces) + .where(eq(projectWorkspaces.id, projectWorkspaceId)) + .then((rows) => rows[0] ?? null); + if (!workspace) throw notFound("Project workspace not found"); + if (workspace.companyId !== companyId) + throw unprocessable("Project workspace must belong to same company"); + if (projectId && workspace.projectId !== projectId) { + throw unprocessable( + "Project workspace must belong to the selected project", + ); + } + return workspace; + } + + async function assertValidExecutionWorkspace( + companyId: string, + projectId: string | null | undefined, + executionWorkspaceId: string, + dbOrTx: DbReader = db, + ) { + const workspace = await dbOrTx + .select({ + id: executionWorkspaces.id, + companyId: executionWorkspaces.companyId, + projectId: executionWorkspaces.projectId, + projectWorkspaceId: executionWorkspaces.projectWorkspaceId, + }) + .from(executionWorkspaces) + .where(eq(executionWorkspaces.id, executionWorkspaceId)) + .then((rows) => rows[0] ?? null); + if (!workspace) throw notFound("Execution workspace not found"); + if (workspace.companyId !== companyId) + throw unprocessable("Execution workspace must belong to same company"); + return workspace; + } + + async function assertValidLabelIds( + companyId: string, + labelIds: string[], + dbOrTx: any = db, + ) { + if (labelIds.length === 0) return; + const existing = await dbOrTx + .select({ id: labels.id }) + .from(labels) + .where( + and(eq(labels.companyId, companyId), inArray(labels.id, labelIds)), + ); + if (existing.length !== new Set(labelIds).size) { + throw unprocessable("One or more labels are invalid for this company"); + } + } + + async function syncIssueLabels( + issueId: string, + companyId: string, + labelIds: string[], + dbOrTx: any = db, + ) { + const deduped = [...new Set(labelIds)]; + await assertValidLabelIds(companyId, deduped, dbOrTx); + await dbOrTx.delete(issueLabels).where(eq(issueLabels.issueId, issueId)); + if (deduped.length === 0) return; + await dbOrTx.insert(issueLabels).values( + deduped.map((labelId) => ({ + issueId, + labelId, + companyId, + })), + ); + } + + async function getIssueRelationSummaryMap( + companyId: string, + issueIds: string[], + dbOrTx: DbReader = db, + ): Promise> { + const uniqueIssueIds = [...new Set(issueIds)]; + const empty = new Map(); + for (const issueId of uniqueIssueIds) { + empty.set(issueId, { blockedBy: [], blocks: [] }); + } + if (uniqueIssueIds.length === 0) return empty; + + const [blockedByRows, blockingRows] = await Promise.all([ + dbOrTx + .select({ + currentIssueId: issueRelations.relatedIssueId, + relatedId: issues.id, + identifier: issues.identifier, + title: issues.title, + status: issues.status, + priority: issues.priority, + assigneeAgentId: issues.assigneeAgentId, + assigneeUserId: issues.assigneeUserId, + }) + .from(issueRelations) + .innerJoin(issues, eq(issueRelations.issueId, issues.id)) + .where( + and( + eq(issueRelations.companyId, companyId), + eq(issueRelations.type, "blocks"), + inArray(issueRelations.relatedIssueId, uniqueIssueIds), + ), + ), + dbOrTx + .select({ + currentIssueId: issueRelations.issueId, + relatedId: issues.id, + identifier: issues.identifier, + title: issues.title, + status: issues.status, + priority: issues.priority, + assigneeAgentId: issues.assigneeAgentId, + assigneeUserId: issues.assigneeUserId, + }) + .from(issueRelations) + .innerJoin(issues, eq(issueRelations.relatedIssueId, issues.id)) + .where( + and( + eq(issueRelations.companyId, companyId), + eq(issueRelations.type, "blocks"), + inArray(issueRelations.issueId, uniqueIssueIds), + ), + ), + ]); + + for (const row of blockedByRows) { + empty + .get(row.currentIssueId) + ?.blockedBy.push(summarizeIssueRelationRow(row)); + } + for (const row of blockingRows) { + empty + .get(row.currentIssueId) + ?.blocks.push(summarizeIssueRelationRow(row)); + } + + const terminalByRoot = await terminalExplicitBlockersByRoot( + companyId, + [...empty.values()].flatMap((relations) => relations.blockedBy), + dbOrTx, + ); + + for (const relations of empty.values()) { + relations.blockedBy.sort((a, b) => a.title.localeCompare(b.title)); + for (const blocker of relations.blockedBy) { + const terminalBlockers = terminalByRoot.get(blocker.id); + if (terminalBlockers && terminalBlockers.length > 0) { + blocker.terminalBlockers = terminalBlockers; + } + } + relations.blocks.sort((a, b) => a.title.localeCompare(b.title)); + } + + const relationSummaries: IssueRelationIssueSummary[] = []; + const collectRelationSummary = (summary: IssueRelationIssueSummary) => { + relationSummaries.push(summary); + for (const terminal of summary.terminalBlockers ?? []) + collectRelationSummary(terminal); + }; + for (const relations of empty.values()) { + for (const blocker of relations.blockedBy) + collectRelationSummary(blocker); + for (const blocking of relations.blocks) collectRelationSummary(blocking); + } + const scheduledRetryByIssueId = await getCurrentScheduledRetriesForIssues( + relationSummaries.map((summary) => summary.id), + companyId, + dbOrTx, + ); + for (const summary of relationSummaries) { + summary.scheduledRetry = scheduledRetryByIssueId.get(summary.id) ?? null; + } + + return empty; + } + + async function withIssueRelationSummaries( + companyId: string, + rows: T[], + dbOrTx: DbReader = db, + ): Promise> { + if (rows.length === 0) return []; + const relationMap = await getIssueRelationSummaryMap( + companyId, + rows.map((row) => row.id), + dbOrTx, + ); + return rows.map((row) => ({ + ...row, + ...(relationMap.get(row.id) ?? { blockedBy: [], blocks: [] }), + })); + } + + async function assertNoBlockingCycles( + companyId: string, + issueId: string, + blockerIssueIds: string[], + dbOrTx: DbReader = db, + ) { + if (blockerIssueIds.length === 0) return; + + const rows = await dbOrTx + .select({ + blockerIssueId: issueRelations.issueId, + blockedIssueId: issueRelations.relatedIssueId, + }) + .from(issueRelations) + .where( + and( + eq(issueRelations.companyId, companyId), + eq(issueRelations.type, "blocks"), + ), + ); + + const adjacency = new Map(); + for (const row of rows) { + const list = adjacency.get(row.blockerIssueId) ?? []; + list.push(row.blockedIssueId); + adjacency.set(row.blockerIssueId, list); + } + + for (const blockerIssueId of blockerIssueIds) { + const queue = [...(adjacency.get(issueId) ?? [])]; + const visited = new Set([issueId]); + while (queue.length > 0) { + const current = queue.shift()!; + if (current === blockerIssueId) { + throw unprocessable("Blocking relations cannot contain cycles"); + } + if (visited.has(current)) continue; + visited.add(current); + queue.push(...(adjacency.get(current) ?? [])); + } + } + } + + async function syncBlockedByIssueIds( + issueId: string, + companyId: string, + blockedByIssueIds: string[], + actor: { agentId?: string | null; userId?: string | null } = {}, + dbOrTx: any = db, + ) { + const deduped = [...new Set(blockedByIssueIds)]; + if (deduped.some((candidate) => candidate === issueId)) { + throw unprocessable("Issue cannot be blocked by itself"); + } + + if (deduped.length > 0) { + const lockedIssueIds = [issueId, ...deduped].sort(); + await dbOrTx.execute( + sql`SELECT ${issues.id} FROM ${issues} + WHERE ${and(eq(issues.companyId, companyId), inArray(issues.id, lockedIssueIds))} + ORDER BY ${issues.id} + FOR UPDATE`, + ); + const relatedIssues = await dbOrTx + .select({ id: issues.id }) + .from(issues) + .where( + and(eq(issues.companyId, companyId), inArray(issues.id, deduped)), + ); + if (relatedIssues.length !== deduped.length) { + throw unprocessable( + "Blocked-by issues must belong to the same company", + ); + } + await assertNoBlockingCycles(companyId, issueId, deduped, dbOrTx); + } + + await dbOrTx + .delete(issueRelations) + .where( + and( + eq(issueRelations.companyId, companyId), + eq(issueRelations.relatedIssueId, issueId), + eq(issueRelations.type, "blocks"), + ), + ); + + if (deduped.length === 0) return; + + await dbOrTx.insert(issueRelations).values( + deduped.map((blockerIssueId) => ({ + companyId, + issueId: blockerIssueId, + relatedIssueId: issueId, + type: "blocks", + createdByAgentId: actor.agentId ?? null, + createdByUserId: actor.userId ?? null, + })), + ); + } + + async function isTerminalOrMissingHeartbeatRun( + runId: string, + dbOrTx: DbReader = db, + ) { + return heartbeatRunIsTerminalOrMissing(dbOrTx, runId); + } + + async function adoptStaleCheckoutRun(input: { + issueId: string; + actorAgentId: string; + actorRunId: string; + expectedCheckoutRunId: string; + }) { + return db.transaction(async (tx) => { + const lockedIssue = await tx + .select({ + id: issues.id, + companyId: issues.companyId, + status: issues.status, + assigneeAgentId: issues.assigneeAgentId, + checkoutRunId: issues.checkoutRunId, + executionRunId: issues.executionRunId, + }) + .from(issues) + .where(eq(issues.id, input.issueId)) + .for("update") + .then((rows) => rows[0] ?? null); + if (!lockedIssue) { + return { adopted: null, latest: null }; + } + + if ( + lockedIssue.status !== "in_progress" || + lockedIssue.assigneeAgentId !== input.actorAgentId || + lockedIssue.checkoutRunId !== input.expectedCheckoutRunId + ) { + return { adopted: null, latest: lockedIssue }; + } + + await Promise.all([ + tx.execute( + sql`select ${heartbeatRuns.id} from ${heartbeatRuns} where ${heartbeatRuns.id} = ${input.expectedCheckoutRunId} for update`, + ), + tx.execute( + sql`select ${heartbeatRuns.id} from ${heartbeatRuns} where ${heartbeatRuns.id} = ${input.actorRunId} for update`, + ), + ]); + const [existingRun, actorRun] = await Promise.all([ + tx + .select() + .from(heartbeatRuns) + .where(eq(heartbeatRuns.id, input.expectedCheckoutRunId)) + .then((rows) => rows[0] ?? null), + tx + .select({ status: heartbeatRuns.status }) + .from(heartbeatRuns) + .where(eq(heartbeatRuns.id, input.actorRunId)) + .then((rows) => rows[0] ?? null), + ]); + const stale = + !existingRun || TERMINAL_HEARTBEAT_RUN_STATUSES.has(existingRun.status); + if (isExplicitContinuationRetryClaim(lockedIssue, existingRun)) { + return { adopted: null, latest: lockedIssue }; + } + if (lockedIssue.executionRunId && lockedIssue.executionRunId !== input.expectedCheckoutRunId) { + const executionRun = await tx.select().from(heartbeatRuns) + .where(eq(heartbeatRuns.id, lockedIssue.executionRunId)).for("update") + .then(rows => rows[0] ?? null); + if (isExplicitContinuationRetryClaim(lockedIssue, executionRun)) { + return { adopted: null, latest: lockedIssue }; + } + } + const actorLive = + actorRun && !TERMINAL_HEARTBEAT_RUN_STATUSES.has(actorRun.status); + if (!stale || !actorLive) { + return { adopted: null, latest: lockedIssue }; + } + + const now = new Date(); + await notifyStatusCardIssueChange(tx, lockedIssue.companyId); + const adopted = await tx + .update(issues) + .set({ + checkoutRunId: input.actorRunId, + executionRunId: input.actorRunId, + executionLockedAt: now, + updatedAt: now, + }) + .where( + and( + eq(issues.id, input.issueId), + eq(issues.status, "in_progress"), + eq(issues.assigneeAgentId, input.actorAgentId), + eq(issues.checkoutRunId, input.expectedCheckoutRunId), + ), + ) + .returning({ + id: issues.id, + status: issues.status, + assigneeAgentId: issues.assigneeAgentId, + checkoutRunId: issues.checkoutRunId, + executionRunId: issues.executionRunId, + }) + .then((rows) => rows[0] ?? null); + if (adopted) { + return { adopted, latest: adopted }; + } + + const latest = await tx + .select({ + id: issues.id, + status: issues.status, + assigneeAgentId: issues.assigneeAgentId, + checkoutRunId: issues.checkoutRunId, + executionRunId: issues.executionRunId, + }) + .from(issues) + .where(eq(issues.id, input.issueId)) + .then((rows) => rows[0] ?? null); + return { adopted: null, latest }; + }); + } + + async function adoptUnownedCheckoutRun(input: { + issueId: string; + actorAgentId: string; + actorRunId: string; + }) { + return db.transaction(async (tx) => { + await tx.execute( + sql`select ${heartbeatRuns.id} from ${heartbeatRuns} where ${heartbeatRuns.id} = ${input.actorRunId} for update`, + ); + const actorRun = await tx + .select({ status: heartbeatRuns.status, companyId: heartbeatRuns.companyId }) + .from(heartbeatRuns) + .where(eq(heartbeatRuns.id, input.actorRunId)) + .then((rows) => rows[0] ?? null); + if (!actorRun || TERMINAL_HEARTBEAT_RUN_STATUSES.has(actorRun.status)) + return null; + + await notifyStatusCardIssueChange(tx, actorRun.companyId); + const now = new Date(); + const adopted = await tx + .update(issues) + .set({ + checkoutRunId: input.actorRunId, + executionRunId: input.actorRunId, + executionLockedAt: now, + updatedAt: now, + }) + .where( + and( + eq(issues.id, input.issueId), + eq(issues.status, "in_progress"), + eq(issues.assigneeAgentId, input.actorAgentId), + isNull(issues.checkoutRunId), + or( + isNull(issues.executionRunId), + eq(issues.executionRunId, input.actorRunId), + ), + ), + ) + .returning({ + id: issues.id, + status: issues.status, + assigneeAgentId: issues.assigneeAgentId, + checkoutRunId: issues.checkoutRunId, + executionRunId: issues.executionRunId, + }) + .then((rows) => rows[0] ?? null); + + return adopted; + }); + } + + async function clearExecutionRunIfTerminal( + issueId: string, + ): Promise { + return db.transaction(async (tx) => { + await tx.execute( + sql`select ${issues.id} from ${issues} where ${issues.id} = ${issueId} for update`, + ); + const issue = await tx + .select({ id: issues.id, companyId: issues.companyId, executionRunId: issues.executionRunId }) + .from(issues) + .where(eq(issues.id, issueId)) + .then((rows) => rows[0] ?? null); + if (!issue?.executionRunId) return false; + + await tx.execute( + sql`select ${heartbeatRuns.id} from ${heartbeatRuns} where ${heartbeatRuns.id} = ${issue.executionRunId} for update`, + ); + const run = await tx + .select() + .from(heartbeatRuns) + .where(eq(heartbeatRuns.id, issue.executionRunId)) + .then((rows) => rows[0] ?? null); + if (run && !TERMINAL_HEARTBEAT_RUN_STATUSES.has(run.status)) return false; + if (isExplicitContinuationRetryClaim(issue, run)) return false; + + await notifyUnsafeWorkspaceExportRecovery(tx, { companyId: issue.companyId, issueId }); + await notifyNativeStatusReassessment(tx, { companyId: issue.companyId, issueId }); + const updated = await tx + .update(issues) + .set({ + executionRunId: null, + executionAgentNameKey: null, + executionLockedAt: null, + updatedAt: new Date(), + }) + .where( + and( + eq(issues.id, issueId), + eq(issues.executionRunId, issue.executionRunId), + ), + ) + .returning({ id: issues.id }) + .then((rows) => rows[0] ?? null); + + return Boolean(updated); + }); + } + + // Symmetric to clearExecutionRunIfTerminal. Clears checkoutRunId (and the + // bundled execution lock cols) when the row's checkoutRunId points at a + // heartbeat run that is terminal or no longer exists. No assignee/status + // precondition. Explicit retry claims remain owned by queue-first settlement. + async function clearCheckoutRunIfTerminal(issueId: string): Promise { + return db.transaction(async (tx) => { + await tx.execute( + sql`select ${issues.id} from ${issues} where ${issues.id} = ${issueId} for update`, + ); + const issue = await tx + .select({ + id: issues.id, + companyId: issues.companyId, + checkoutRunId: issues.checkoutRunId, + executionRunId: issues.executionRunId, + }) + .from(issues) + .where(eq(issues.id, issueId)) + .then((rows) => rows[0] ?? null); + if (!issue?.checkoutRunId) return false; + + await tx.execute( + sql`select ${heartbeatRuns.id} from ${heartbeatRuns} where ${heartbeatRuns.id} = ${issue.checkoutRunId} for update`, + ); + const run = await tx + .select() + .from(heartbeatRuns) + .where(eq(heartbeatRuns.id, issue.checkoutRunId)) + .then((rows) => rows[0] ?? null); + if (run && !TERMINAL_HEARTBEAT_RUN_STATUSES.has(run.status)) return false; + if (isExplicitContinuationRetryClaim(issue, run)) return false; + + if ( + issue.executionRunId && + issue.executionRunId !== issue.checkoutRunId + ) { + await tx.execute( + sql`select ${heartbeatRuns.id} from ${heartbeatRuns} where ${heartbeatRuns.id} = ${issue.executionRunId} for update`, + ); + const executionRun = await tx + .select() + .from(heartbeatRuns) + .where(eq(heartbeatRuns.id, issue.executionRunId)) + .then((rows) => rows[0] ?? null); + if ( + executionRun && + !TERMINAL_HEARTBEAT_RUN_STATUSES.has(executionRun.status) + ) + return false; + if (isExplicitContinuationRetryClaim(issue, executionRun)) return false; + } + + await notifyUnsafeWorkspaceExportRecovery(tx, { companyId: issue.companyId, issueId }); + await notifyNativeStatusReassessment(tx, { companyId: issue.companyId, issueId }); + const updated = await tx + .update(issues) + .set({ + checkoutRunId: null, + executionRunId: null, + executionAgentNameKey: null, + executionLockedAt: null, + updatedAt: new Date(), + }) + .where( + and( + eq(issues.id, issueId), + eq(issues.checkoutRunId, issue.checkoutRunId), + issue.executionRunId + ? eq(issues.executionRunId, issue.executionRunId) + : isNull(issues.executionRunId), + ), + ) + .returning({ id: issues.id }) + .then((rows) => rows[0] ?? null); + + return Boolean(updated); + }); + } + + async function addStopRelayCommentIfNeeded( + child: typeof issues.$inferSelect, + dbOrTx: any = db, + ) { + if ( + !child.parentId || + (child.status !== "blocked" && child.status !== "cancelled") + ) + return null; + + const relayKey = `issue-stop-relay:${child.id}:${child.status}`; + await dbOrTx.execute( + sql`select pg_advisory_xact_lock(hashtextextended(${relayKey}, 0))`, + ); + + const childIdentifier = child.identifier?.trim() || child.id; + const childPrefix = childIdentifier.split("-")[0] || "PAP"; + const body = `System relay: [${childIdentifier}](/${childPrefix}/issues/${childIdentifier}) transitioned to \`${child.status}\`.`; + const existingRelay = await dbOrTx + .select({ id: issueComments.id }) + .from(issueComments) + .where( + and( + eq(issueComments.companyId, child.companyId), + eq(issueComments.issueId, child.parentId), + eq(issueComments.authorType, "system"), + eq(issueComments.body, body), + ), + ) + .limit(1) + .then((rows: Array<{ id: string }>) => rows[0] ?? null); + if (existingRelay) return null; + + const parent = await dbOrTx + .select({ + id: issues.id, + companyId: issues.companyId, + assigneeAgentId: issues.assigneeAgentId, + status: issues.status, + }) + .from(issues) + .where( + and( + eq(issues.id, child.parentId), + eq(issues.companyId, child.companyId), + ), + ) + .then( + ( + rows: Array<{ + id: string; + companyId: string; + assigneeAgentId: string | null; + status: string; + }>, + ) => rows[0] ?? null, + ); + if (!parent) return null; + + const [comment] = await insertIssueComment(dbOrTx, { + companyId: child.companyId, + issueId: parent.id, + authorType: "system", + body, + }); + await dbOrTx + .update(issues) + .set({ updatedAt: new Date() }) + .where(eq(issues.id, parent.id)); + + return { comment, parent }; + } + + async function archiveInbox( + companyId: string, + issueId: string, + userId: string, + archivedAt: Date = new Date(), + attribution?: { + archivedByActorType: "user" | "agent"; + archivedByAgentId?: string | null; + archivedByRunId?: string | null; + }, + dbOrTx: any = db, + ) { + const runArchive = async (tx: typeof dbOrTx) => { + // Completion locks the issue before archiving. Take the FK's parent lock + // first too, or an insert can hold the archive key while waiting on that + // issue and deadlock with completion's archive UPSERT. SHARE also holds + // companyId stable; different users can still archive concurrently. + const [issue] = await tx + .select({ id: issues.id }) + .from(issues) + .where(and(eq(issues.companyId, companyId), eq(issues.id, issueId))) + .for("share"); + if (!issue) throw notFound("Issue not found"); + + const now = new Date(); + const [row] = await tx + .insert(issueInboxArchives) + .values({ + companyId, + issueId, + userId, + archivedByActorType: attribution?.archivedByActorType ?? "user", + archivedByAgentId: attribution?.archivedByAgentId ?? null, + archivedByRunId: attribution?.archivedByRunId ?? null, + archivedAt, + updatedAt: now, + }) + .onConflictDoUpdate({ + target: [ + issueInboxArchives.companyId, + issueInboxArchives.issueId, + issueInboxArchives.userId, + ], + set: { + archivedAt, + archivedByActorType: attribution?.archivedByActorType ?? "user", + archivedByAgentId: attribution?.archivedByAgentId ?? null, + archivedByRunId: attribution?.archivedByRunId ?? null, + updatedAt: now, + }, + // A request that waited behind completion must not replace the newer + // archive with its earlier request time and resurface the done task. + setWhere: lte(issueInboxArchives.archivedAt, archivedAt), + }) + .returning(); + if (row) return row; + // ON CONFLICT holds this row lock even when setWhere skips the update. + // Return the newer state, including its matching actor attribution. + const [existing] = await tx + .select() + .from(issueInboxArchives) + .where(and( + eq(issueInboxArchives.companyId, companyId), + eq(issueInboxArchives.issueId, issueId), + eq(issueInboxArchives.userId, userId), + )); + if (!existing) throw new Error("Inbox archive conflict row missing"); + return existing; + }; + return dbOrTx === db ? db.transaction(runArchive) : runArchive(dbOrTx); + } + + const service = { + clearExecutionRunIfTerminal, + clearCheckoutRunIfTerminal, + addStopRelayCommentIfNeeded, + + unreadInboxIssueIds: async (companyId: string, userId: string, readCondition: SQL) => { + // Rank the visible Mine set before counting unread, just like the inbox. + // This projection never reads descriptions, run payloads, or recovery details. + const directlyParticipated = or( + and(eq(issues.createdByUserId, userId), humanCreatedIssueCondition()), + eq(issues.assigneeUserId, userId), + ); + const participationSources = unionAll( + db.select({ id: sql`${issues.id}`.as("id") }).from(issues).where(and( + eq(issues.companyId, companyId), + directlyParticipated, + )), + db.select({ id: sql`case when length(${activityLog.entityId}) = 36 and ${activityLog.entityId} ~ '^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$' then ${activityLog.entityId}::uuid end`.as("id") }).from(activityLog).where(and( + userParticipationActivityCondition(companyId, userId), + notInArray(activityLog.entityId, db.select({ id: sql`${issues.id}::text` }).from(issues) + .where(and(eq(issues.companyId, companyId), directlyParticipated))), + )), + db.select({ id: sql`${issueComments.issueId}`.as("id") }).from(issueComments).where(and( + eq(issueComments.companyId, companyId), eq(issueComments.authorUserId, userId), directHumanCommentCondition(), + )), + db.select({ id: sql`${issues.id}`.as("id") }).from(issueThreadInteractions) + .innerJoin(issues, and(eq(issues.id, issueThreadInteractions.issueId), eq(issues.companyId, companyId))) + .where(pendingUserRequestRowCondition(companyId, userId)), + ).as("mine_participation_sources"); + const participated = db.selectDistinct({ id: participationSources.id }).from(participationSources).as("mine_participants"); + const eligible = db.select({ id: issues.id, updatedAt: issues.updatedAt }) + .from(participated).innerJoin(issues, eq(issues.id, sql`"mine_participants"."id"`)).where(and( + eq(issues.companyId, companyId), + visibleIssueCondition(), + isNull(issues.conversationAgentId), + nonPluginOperationIssueCondition(), + ne(issues.status, "cancelled"), + inboxVisibleForUserCondition(companyId, userId), + readCondition, + )).orderBy(asc(issues.id)).as("issues"); + const rows = await db.select({ id: eligible.id }).from(eligible) + .orderBy(desc(issueCanonicalLastActivityAtExpr(companyId)), desc(eligible.updatedAt), desc(eligible.id)).limit(100); + if (rows.length === 0) return []; + // Keep correlated predicates in WHERE (as in the list's user stats). + // A single-table Drizzle SELECT projection unqualifies nested columns. + return db.select({ id: issues.id }).from(issues).where(and( + eq(issues.companyId, companyId), + inArray(issues.id, rows.map((row) => row.id)), + unreadForUserCondition(companyId, userId), + readCondition, + )).then((unread) => unread.map((row) => row.id)); + }, + + list: async (companyId: string, filters?: IssueFilters) => { + if (filters?.sortField === "id" && filters.attention) { + throw unprocessable("ID ordering is not supported for blocked attention lists"); + } + if (filters?.afterId !== undefined && ( + !isUuidLike(filters.afterId) || filters.sortField !== "id" || + filters.sortDir !== "asc" || (filters.offset ?? 0) !== 0 + )) { + throw unprocessable("afterId requires a UUID, ascending ID order and no offset"); + } + if (filters?.attention === "blocked") { + return listBlockedInboxIssues(db, companyId, { + ...filters, + includeBlockedBy: true, + includeBlockedInboxAttention: true, + }); + } + + const conditions = [ + eq(issues.companyId, companyId), + visibleIssueCondition(), + ]; + if (filters?.issueIds) { + if (filters.issueIds.length === 0) return []; + conditions.push(inArray(issues.id, filters.issueIds)); + } + if (filters?.parentIds) { + if (filters.parentIds.length === 0) return []; + conditions.push(inArray(issues.parentId, filters.parentIds)); + } + if (!filters?.q?.trim() && !filters?.issueIds) { + conditions.push(isNull(issues.conversationAgentId)); + if (!filters?.touchedByUserId && !filters?.unreadForUserId && !filters?.inboxArchivedByUserId) { + conditions.push(nonIdleSlackIssueCondition()); + } + } + if (filters?.afterId) conditions.push(gt(issues.id, filters.afterId)); + if (filters?.readCondition) conditions.push(filters.readCondition); + const creationCondition = creationVisibilityCondition(companyId, filters); + if (creationCondition) conditions.push(creationCondition); + const assigneeAgentFilter = parseIssueAssigneeAgentFilter( + filters?.assigneeAgentId, + ); + assertValidAssigneeAgentFilter(assigneeAgentFilter); + const limit = + typeof filters?.limit === "number" && Number.isFinite(filters.limit) + ? Math.max(1, Math.floor(filters.limit)) + : undefined; + const offset = + typeof filters?.offset === "number" && Number.isFinite(filters.offset) + ? Math.max(0, Math.floor(filters.offset)) + : 0; + const touchedByUserId = filters?.touchedByUserId?.trim() || undefined; + const inboxArchivedByUserId = + filters?.inboxArchivedByUserId?.trim() || undefined; + const unreadForUserId = filters?.unreadForUserId?.trim() || undefined; + const contextUserId = + unreadForUserId ?? touchedByUserId ?? inboxArchivedByUserId; + const includeBlockedBy = filters?.includeBlockedBy === true; + const includeBlockedInboxAttention = + filters?.includeBlockedInboxAttention === true; + const includeLiveDescendantSummary = + filters?.includeLiveDescendantSummary === true; + const rawSearch = filters?.q?.trim() ?? ""; + const hasSearch = rawSearch.length > 0; + const taskSearch = parseTaskSearch(rawSearch); + if (filters?.createdFromIssueId) { + conditions.push(createdFromIssueCondition(companyId, filters.createdFromIssueId)); + } + if (filters?.descendantOf) { + conditions.push(sql` + ${issues.id} IN ( + WITH RECURSIVE descendants(id) AS ( + SELECT ${issues.id} + FROM ${issues} + WHERE ${issues.companyId} = ${companyId} + AND ${issues.parentId} = ${filters.descendantOf} + UNION + SELECT ${issues.id} + FROM ${issues} + JOIN descendants ON ${issues.parentId} = descendants.id + WHERE ${issues.companyId} = ${companyId} + ) + SELECT id FROM descendants + ) + `); + } + const lowTrustCondition = lowTrustBoundaryIssueCondition( + companyId, + filters?.lowTrustBoundary, + ); + if (lowTrustCondition) conditions.push(lowTrustCondition); + const statuses = parseStatusFilter(filters?.status); + if (statuses.length === 1) { + conditions.push(eq(issues.status, statuses[0])); + } else if (statuses.length > 1) { + conditions.push(inArray(issues.status, statuses)); + } + if (assigneeAgentFilter === null) { + conditions.push(isNull(issues.assigneeAgentId)); + } else if (assigneeAgentFilter) { + conditions.push(eq(issues.assigneeAgentId, assigneeAgentFilter)); + } + if (filters?.participantAgentId) { + conditions.push( + participatedByAgentCondition(companyId, filters.participantAgentId), + ); + } + if (filters?.assigneeUserId) { + conditions.push(eq(issues.assigneeUserId, filters.assigneeUserId)); + } + if (touchedByUserId) { + conditions.push(touchedByUserCondition(companyId, touchedByUserId)); + } + if (inboxArchivedByUserId) { + conditions.push( + inboxVisibleForUserCondition(companyId, inboxArchivedByUserId), + ); + } + if (unreadForUserId) { + conditions.push(unreadForUserCondition(companyId, unreadForUserId)); + } + if (filters?.projectId) + conditions.push(eq(issues.projectId, filters.projectId)); + if (filters?.workspaceId) { + conditions.push( + or( + eq(issues.executionWorkspaceId, filters.workspaceId), + eq(issues.projectWorkspaceId, filters.workspaceId), + )!, + ); + } + if (filters?.executionWorkspaceId) { + conditions.push( + eq(issues.executionWorkspaceId, filters.executionWorkspaceId), + ); + } + if (filters?.parentId) + conditions.push(eq(issues.parentId, filters.parentId)); + if (filters?.originKind) + conditions.push(eq(issues.originKind, filters.originKind)); + if (filters?.originKindPrefix) + conditions.push( + like(issues.originKind, `${filters.originKindPrefix}%`), + ); + if (filters?.originId) + conditions.push(eq(issues.originId, filters.originId)); + if (filters?.hasPlanDocument !== undefined) { + conditions.push( + hasPlanDocumentCondition(companyId, filters.hasPlanDocument), + ); + } + if (!shouldIncludePluginOperationIssues(filters)) { + conditions.push(nonPluginOperationIssueCondition()); + } + if (filters?.labelId) { + const labeledIssueIds = await db + .select({ issueId: issueLabels.issueId }) + .from(issueLabels) + .where( + and( + eq(issueLabels.companyId, companyId), + eq(issueLabels.labelId, filters.labelId), + ), + ); + if (labeledIssueIds.length === 0) return []; + conditions.push( + inArray( + issues.id, + labeledIssueIds.map((row) => row.issueId), + ), + ); + } + if (filters?.updatedSince) { + const since = new Date(filters.updatedSince); + if (Number.isFinite(since.getTime())) { + conditions.push(gt(issues.updatedAt, since)); + } + } + if ( + filters?.excludeRoutineExecutions && + !filters?.originKind && + !filters?.originId + ) { + conditions.push(ne(issues.originKind, "routine_execution")); + } + const priorityOrder = sql`CASE ${issues.priority} WHEN 'critical' THEN 0 WHEN 'high' THEN 1 WHEN 'medium' THEN 2 WHEN 'low' THEN 3 ELSE 4 END`; + const searchOrder = sql`-task_search.score`; + const selectedFields = filters?.listProjection ? { + ...issueListSelect, + description: sql`NULL`, + descriptionTruncated: sql`false`, + } : issueListSelect; + const issueSource = (filters?.oneChildPerParent + ? db.selectDistinctOn([issues.parentId], selectedFields) + : db.select(selectedFields)).from(issues); + const searchedSource = hasSearch + ? issueSource.innerJoin(sql`( + ${taskSearchCtes(companyId, taskSearch, true, and(...conditions), filters?.readCondition)} + SELECT m.id, ${taskSearchScore(taskSearch)} AS score FROM matched m + ) task_search`, sql`task_search.id = ${issues.id}`) + : issueSource; + const baseQuery = searchedSource + .where(and(...conditions)) + .orderBy( + ...(filters?.oneChildPerParent ? [asc(issues.parentId)] : []), + ...issueListOrderBy(companyId, { + hasSearch, + priorityOrder, + searchOrder, + sortField: filters?.sortField, + sortDir: filters?.sortDir, + }), + ); + const pageQuery = + offset > 0 + ? limit === undefined + ? baseQuery.offset(offset) + : baseQuery.limit(limit).offset(offset) + : limit === undefined + ? baseQuery + : baseQuery.limit(limit); + const rows = (await pageQuery).map((row) => ({ + ...row, + description: decodeDatabaseTextPreview( + row.description, + ISSUE_LIST_DESCRIPTION_MAX_CHARS, + ), + })); + const withLabels = await withIssueLabels(db, rows); + const runMap = filters?.listProjection === "index" ? new Map() : await activeRunMapForIssues(db, withLabels); + const withRuns = withActiveRuns(withLabels, runMap); + if (withRuns.length === 0) { + return withRuns; + } + + const issueIds = withRuns.map((row) => row.id); + const [ + statsRows, + readRows, + lastActivityRows, + archiveRows, + blockedByMap, + liveDescendantCountByIssueId, + ] = await Promise.all([ + contextUserId + ? userCommentStatsForIssues(db, companyId, contextUserId, issueIds) + : Promise.resolve([]), + contextUserId + ? userReadStatsForIssues(db, companyId, contextUserId, issueIds) + : Promise.resolve([]), + lastActivityStatsForIssues(db, companyId, issueIds), + contextUserId + ? inboxArchiveRowsForIssues(db, companyId, contextUserId, issueIds) + : Promise.resolve([]), + includeBlockedBy + ? blockedByMapForIssues(db, companyId, issueIds) + : Promise.resolve(new Map()), + includeLiveDescendantSummary + ? liveDescendantCountMapForIssues(db, companyId, issueIds) + : Promise.resolve(new Map()), + ]); + const statsByIssueId = new Map( + statsRows.map((row) => [row.issueId, row]), + ); + const lastActivityByIssueId = new Map( + lastActivityRows.map((row) => [row.issueId, row]), + ); + const archiveByIssueId = new Map( + archiveRows.map((row) => [row.issueId, row]), + ); + const [ + blockerAttentionByIssueId, + reviewAttentionByIssueId, + blockedInboxAttentionByIssueId, + ] = await Promise.all([ + filters?.listProjection === "index" ? Promise.resolve(new Map()) : listIssueBlockerAttentionMap(db, companyId, withRuns), + filters?.listProjection === "index" ? Promise.resolve(new Map()) : listIssueReviewAttentionMap(db, companyId, withRuns), + includeBlockedInboxAttention + ? listIssueBlockedInboxAttentionMap(db, companyId, withRuns) + : Promise.resolve(new Map()), + ]); + + if (!contextUserId) { + return withRuns.map((row) => { + const activity = lastActivityByIssueId.get(row.id); + const lastActivityAt = + latestIssueActivityAt( + row.updatedAt, + activity?.latestCommentAt ?? null, + activity?.latestLogAt ?? null, + ) ?? row.updatedAt; + return { + ...row, + ...(includeBlockedBy + ? { blockedBy: blockedByMap.get(row.id) ?? [] } + : {}), + lastActivityAt, + ...(blockerAttentionByIssueId.has(row.id) + ? { blockerAttention: blockerAttentionByIssueId.get(row.id) } + : {}), + reviewAttention: + reviewAttentionByIssueId.get(row.id) ?? reviewAttentionNone(), + ...(includeBlockedInboxAttention + ? { + blockedInboxAttention: + blockedInboxAttentionByIssueId.get(row.id) ?? null, + } + : {}), + ...(includeLiveDescendantSummary + ? { + liveDescendantCount: + liveDescendantCountByIssueId.get(row.id) ?? 0, + } + : {}), + }; + }); + } + + const readByIssueId = new Map( + readRows.map((row) => [row.issueId, row.myLastReadAt]), + ); + + return withRuns.map((row) => { + const activity = lastActivityByIssueId.get(row.id); + const lastActivityAt = + latestIssueActivityAt( + row.updatedAt, + activity?.latestCommentAt ?? null, + activity?.latestLogAt ?? null, + ) ?? row.updatedAt; + return { + ...row, + ...activeInboxArchiveFields( + archiveByIssueId.get(row.id), + lastActivityAt, + ), + ...(includeBlockedBy + ? { blockedBy: blockedByMap.get(row.id) ?? [] } + : {}), + lastActivityAt, + ...(blockerAttentionByIssueId.has(row.id) + ? { blockerAttention: blockerAttentionByIssueId.get(row.id) } + : {}), + reviewAttention: + reviewAttentionByIssueId.get(row.id) ?? reviewAttentionNone(), + ...(includeBlockedInboxAttention + ? { + blockedInboxAttention: + blockedInboxAttentionByIssueId.get(row.id) ?? null, + } + : {}), + ...(includeLiveDescendantSummary + ? { + liveDescendantCount: + liveDescendantCountByIssueId.get(row.id) ?? 0, + } + : {}), + ...deriveIssueUserContext(row, contextUserId, { + isUnreadForMe: statsByIssueId.get(row.id)?.isUnreadForMe ?? false, + myLastCommentAt: + statsByIssueId.get(row.id)?.myLastCommentAt ?? null, + myLastReadAt: readByIssueId.get(row.id) ?? null, + lastExternalCommentAt: + statsByIssueId.get(row.id)?.lastExternalCommentAt ?? null, + }), + }; + }); + }, + + count: async (companyId: string, filters?: IssueFilters) => { + if (filters?.attention === "blocked") { + return countBlockedInboxIssues(db, companyId, filters); + } + + const conditions = [eq(issues.companyId, companyId), visibleIssueCondition()]; + if (filters?.readCondition) conditions.push(filters.readCondition); + const creationCondition = creationVisibilityCondition(companyId, filters); + if (creationCondition) conditions.push(creationCondition); + if (!filters?.q?.trim()) { + conditions.push(isNull(issues.conversationAgentId)); + if (!filters?.touchedByUserId && !filters?.unreadForUserId && !filters?.inboxArchivedByUserId) { + conditions.push(nonIdleSlackIssueCondition()); + } + } + const statuses = parseStatusFilter(filters?.status); + if (statuses.length === 1) + conditions.push(eq(issues.status, statuses[0]!)); + else if (statuses.length > 1) + conditions.push(inArray(issues.status, statuses)); + const assigneeAgentFilter = parseIssueAssigneeAgentFilter( + filters?.assigneeAgentId, + ); + assertValidAssigneeAgentFilter(assigneeAgentFilter); + if (assigneeAgentFilter === null) { + conditions.push(isNull(issues.assigneeAgentId)); + } else if (assigneeAgentFilter) { + conditions.push(eq(issues.assigneeAgentId, assigneeAgentFilter)); + } + if (filters?.assigneeUserId) + conditions.push(eq(issues.assigneeUserId, filters.assigneeUserId)); + if (filters?.projectId) + conditions.push(eq(issues.projectId, filters.projectId)); + if (filters?.workspaceId) { + conditions.push( + or( + eq(issues.executionWorkspaceId, filters.workspaceId), + eq(issues.projectWorkspaceId, filters.workspaceId), + )!, + ); + } + if (filters?.executionWorkspaceId) + conditions.push( + eq(issues.executionWorkspaceId, filters.executionWorkspaceId), + ); + if (filters?.parentId) + conditions.push(eq(issues.parentId, filters.parentId)); + if (filters?.originKind) + conditions.push(eq(issues.originKind, filters.originKind)); + if (filters?.originKindPrefix) + conditions.push( + like(issues.originKind, `${filters.originKindPrefix}%`), + ); + if (filters?.originId) + conditions.push(eq(issues.originId, filters.originId)); + if (filters?.hasPlanDocument !== undefined) { + conditions.push( + hasPlanDocumentCondition(companyId, filters.hasPlanDocument), + ); + } + if (!shouldIncludePluginOperationIssues(filters)) + conditions.push(nonPluginOperationIssueCondition()); + const [row] = await db + .select({ count: sql`count(*)` }) + .from(issues) + .where(and(...conditions)); + return Number(row?.count ?? 0); + }, + + countUnreadTouchedByUser: async ( + companyId: string, + userId: string, + status?: string | readonly string[], + ) => { + const conditions = [ + eq(issues.companyId, companyId), + visibleIssueCondition(), + nonPluginOperationIssueCondition(), + unreadForUserCondition(companyId, userId), + ]; + const statuses = parseStatusFilter(status); + if (statuses.length === 1) { + conditions.push(eq(issues.status, statuses[0])); + } else if (statuses.length > 1) { + conditions.push(inArray(issues.status, statuses)); + } + const [row] = await db + .select({ count: sql`count(*)` }) + .from(issues) + .where(and(...conditions)); + return Number(row?.count ?? 0); + }, + + markRead: async ( + companyId: string, + issueId: string, + userId: string, + readAt: Date = new Date(), + ) => { + const now = new Date(); + const [row] = await db + .insert(issueReadStates) + .values({ + companyId, + issueId, + userId, + lastReadAt: readAt, + updatedAt: now, + }) + .onConflictDoUpdate({ + target: [ + issueReadStates.companyId, + issueReadStates.issueId, + issueReadStates.userId, + ], + set: { + lastReadAt: readAt, + updatedAt: now, + }, + }) + .returning(); + return row; + }, + + markUnread: async (companyId: string, issueId: string, userId: string) => { + const deleted = await db + .delete(issueReadStates) + .where( + and( + eq(issueReadStates.companyId, companyId), + eq(issueReadStates.issueId, issueId), + eq(issueReadStates.userId, userId), + ), + ) + .returning(); + return deleted.length > 0; + }, + + archiveInbox, + + /** + * Seed inbox archives for a batch of freshly imported issues so a company + * import does not flood the importing user's inbox. Imported issues are + * historical work, not new inbox items, but the inbox "mine" query surfaces + * every touched-and-not-archived issue; a 1000-task import would otherwise + * bury the inbox. Seeding a per-user inbox archive keeps them hidden via + * `inboxVisibleForUserCondition`, while genuine new activity on an imported + * issue still resurfaces it. This runs only on import (mirroring how + * `pauseAutomations` threads an import-only suppression) and never touches + * normal issue creation. Rows carry the same "user"-attributed shape a + * manual inbox archive writes, batched to stay under Postgres bind limits. + */ + archiveImportedInbox: async ( + companyId: string, + issueIds: string[], + userId: string, + archivedAt: Date = new Date(), + ): Promise => { + if (issueIds.length === 0) return; + const now = new Date(); + const rows = issueIds.map((issueId) => ({ + companyId, + issueId, + userId, + archivedByActorType: "user" as const, + archivedByAgentId: null, + archivedByRunId: null, + archivedAt, + createdAt: now, + updatedAt: now, + })); + await insertRowsInChunks(db, issueInboxArchives, rows); + }, + + unarchiveInbox: async ( + companyId: string, + issueId: string, + userId: string, + ) => { + const [row] = await db + .delete(issueInboxArchives) + .where( + and( + eq(issueInboxArchives.companyId, companyId), + eq(issueInboxArchives.issueId, issueId), + eq(issueInboxArchives.userId, userId), + ), + ) + .returning(); + return row ?? null; + }, + + getActiveInboxArchiveFields: async ( + issue: Pick, + userId: string, + ) => { + const [[activity], [archive]] = await Promise.all([ + lastActivityStatsForIssues(db, issue.companyId, [issue.id]), + inboxArchiveRowsForIssues(db, issue.companyId, userId, [issue.id]), + ]); + const lastActivityAt = + latestIssueActivityAt( + issue.updatedAt, + activity?.latestCommentAt ?? null, + activity?.latestLogAt ?? null, + ) ?? issue.updatedAt; + return activeInboxArchiveFields(archive, lastActivityAt); + }, + + /** + * Walk the full parent chain from `parentIssueId` (inclusive) looking for + * a still-open ancestor created by `agentId`. Used to refuse agent + * delegation cycles: an agent assigning a new child to the agent that + * created an open ancestor is handing the same work back to its own + * delegator (A→B→A hot-potato). One recursive query covers the entire + * chain regardless of depth; `UNION` (not `UNION ALL`) deduplicates + * revisited rows, so a parent graph corrupted into a loop terminates + * instead of recursing forever. + */ + findOpenAncestorCreatedByAgent: async ( + parentIssueId: string, + agentId: string, + ): Promise<{ + id: string; + identifier: string | null; + parentId: string | null; + createdByAgentId: string | null; + status: string; + } | null> => { + const rows = await db.execute(sql` + WITH RECURSIVE ancestors(id, parent_id) AS ( + SELECT id, parent_id FROM issues WHERE id = ${parentIssueId} + UNION + SELECT parent.id, parent.parent_id + FROM issues parent + JOIN ancestors ON parent.id = ancestors.parent_id + ) + SELECT i.id, i.identifier, i.parent_id, i.created_by_agent_id, i.status + FROM issues i + JOIN ancestors a ON a.id = i.id + WHERE i.created_by_agent_id = ${agentId} + AND i.status NOT IN ('done', 'cancelled') + LIMIT 1 + `); + const first = (Array.isArray(rows) ? rows[0] : null) as Record< + string, + unknown + > | null; + if (!first) return null; + return { + id: String(first.id), + identifier: + typeof first.identifier === "string" ? first.identifier : null, + parentId: typeof first.parent_id === "string" ? first.parent_id : null, + createdByAgentId: + typeof first.created_by_agent_id === "string" + ? first.created_by_agent_id + : null, + status: String(first.status), + }; + }, + + getById: async (raw: string) => { + const id = raw.trim(); + const identifier = normalizeIssueReferenceIdentifier(id); + if (identifier) { + return getIssueByIdentifier(identifier); + } + if (!isUuidLike(id)) { + return null; + } + return getIssueByUuid(id); + }, + + getByIdForUpdate: async (id: string, dbOrTx: any) => { + return dbOrTx + .select() + .from(issues) + .where(eq(issues.id, id)) + .for("update") + .then((rows: Array) => rows[0] ?? null); + }, + + getByIdentifier: async (identifier: string) => { + return getIssueByIdentifier(identifier); + }, + + getCurrentScheduledRetry: async (issueId: string) => { + const issue = await db + .select({ id: issues.id, companyId: issues.companyId }) + .from(issues) + .where(eq(issues.id, issueId)) + .then((rows) => rows[0] ?? null); + if (!issue) throw notFound("Issue not found"); + return getCurrentScheduledRetryForIssue(issue.id, issue.companyId); + }, + + getRelationSummaries: async (issueId: string) => { + const issue = await db + .select({ id: issues.id, companyId: issues.companyId }) + .from(issues) + .where(eq(issues.id, issueId)) + .then((rows) => rows[0] ?? null); + if (!issue) throw notFound("Issue not found"); + const relations = await getIssueRelationSummaryMap( + issue.companyId, + [issueId], + db, + ); + return relations.get(issueId) ?? { blockedBy: [], blocks: [] }; + }, + + getBlockerDiagnostics: async ( + issueId: string, + maxBlockers = ISSUE_BLOCKER_DIAGNOSTICS_MAX_BLOCKERS, + ) => { + const issue = await db + .select({ id: issues.id, companyId: issues.companyId }) + .from(issues) + .where(eq(issues.id, issueId)) + .then((rows) => rows[0] ?? null); + if (!issue) throw notFound("Issue not found"); + + const cappedMax = Math.max( + 0, + Math.min(maxBlockers, ISSUE_BLOCKER_DIAGNOSTICS_MAX_BLOCKERS), + ); + const blockerRows = await db + .select({ + id: issues.id, + companyId: issues.companyId, + projectId: issues.projectId, + parentId: issues.parentId, + identifier: issues.identifier, + title: issues.title, + status: issues.status, + priority: issues.priority, + assigneeAgentId: issues.assigneeAgentId, + assigneeUserId: issues.assigneeUserId, + }) + .from(issueRelations) + .innerJoin(issues, eq(issueRelations.issueId, issues.id)) + .where( + and( + eq(issueRelations.companyId, issue.companyId), + eq(issueRelations.type, "blocks"), + eq(issueRelations.relatedIssueId, issue.id), + eq(issues.companyId, issue.companyId), + ), + ) + .orderBy(asc(issues.title), asc(issues.id)) + .limit(cappedMax + 1); + + const readiness = await listIssueDependencyReadinessMap( + db, + issue.companyId, + [issue.id], + ); + + return { + blockers: blockerRows.slice( + 0, + cappedMax, + ) as IssueBlockerDiagnosticsIssueRow[], + readiness: + readiness.get(issue.id) ?? createIssueDependencyReadiness(issue.id), + truncated: blockerRows.length > cappedMax, + }; + }, + + getWakeDiagnostics: async ( + issueId: string, + opts?: { + maxWakeRequests?: number; + maxActivityRecords?: number; + lookbackDays?: number; + }, + ) => { + const issue = await db + .select({ id: issues.id, companyId: issues.companyId }) + .from(issues) + .where(eq(issues.id, issueId)) + .then((rows) => rows[0] ?? null); + if (!issue) throw notFound("Issue not found"); + + const maxWakeRequests = Math.max( + 0, + Math.min( + opts?.maxWakeRequests ?? ISSUE_WAKE_DIAGNOSTICS_MAX_WAKE_REQUESTS, + ISSUE_WAKE_DIAGNOSTICS_MAX_WAKE_REQUESTS, + ), + ); + const maxActivityRecords = Math.max( + 0, + Math.min( + opts?.maxActivityRecords ?? + ISSUE_WAKE_DIAGNOSTICS_MAX_ACTIVITY_RECORDS, + ISSUE_WAKE_DIAGNOSTICS_MAX_ACTIVITY_RECORDS, + ), + ); + const lookbackDays = Math.max( + 1, + Math.min( + opts?.lookbackDays ?? ISSUE_WAKE_DIAGNOSTICS_LOOKBACK_DAYS, + ISSUE_WAKE_DIAGNOSTICS_LOOKBACK_DAYS, + ), + ); + const since = new Date(Date.now() - lookbackDays * 24 * 60 * 60 * 1000); + + const wakeRows = await db + .select({ + agentId: agentWakeupRequests.agentId, + source: agentWakeupRequests.source, + reason: agentWakeupRequests.reason, + status: agentWakeupRequests.status, + coalescedCount: agentWakeupRequests.coalescedCount, + runId: agentWakeupRequests.runId, + requestedAt: agentWakeupRequests.requestedAt, + claimedAt: agentWakeupRequests.claimedAt, + finishedAt: agentWakeupRequests.finishedAt, + error: agentWakeupRequests.error, + }) + .from(agentWakeupRequests) + .where( + and( + eq(agentWakeupRequests.companyId, issue.companyId), + gte(agentWakeupRequests.requestedAt, since), + wakeRequestTargetsIssue(issue.id), + ), + ) + .orderBy( + desc(agentWakeupRequests.requestedAt), + desc(agentWakeupRequests.createdAt), + ) + .limit(maxWakeRequests + 1); + + const activityRows = await db + .select({ + action: activityLog.action, + entityType: activityLog.entityType, + entityId: activityLog.entityId, + agentId: activityLog.agentId, + runId: activityLog.runId, + details: activityLog.details, + createdAt: activityLog.createdAt, + }) + .from(activityLog) + .where( + and( + eq(activityLog.companyId, issue.companyId), + gte(activityLog.createdAt, since), + inArray(activityLog.action, [ + ...ISSUE_WAKE_DIAGNOSTICS_ACTIVITY_ACTIONS, + ]), + wakeDiagnosticActivityTargetsIssue(issue.id), + ), + ) + .orderBy(desc(activityLog.createdAt)) + .limit(maxActivityRecords + 1); + + return { + wakeRequests: wakeRows.slice( + 0, + maxWakeRequests, + ) as IssueWakeDiagnosticsWakeRequestRow[], + activityRecords: activityRows.slice( + 0, + maxActivityRecords, + ) as IssueWakeDiagnosticsActivityRow[], + truncatedWakeRequests: wakeRows.length > maxWakeRequests, + truncatedActivityRecords: activityRows.length > maxActivityRecords, + caps: { + maxWakeRequests: ISSUE_WAKE_DIAGNOSTICS_MAX_WAKE_REQUESTS, + maxActivityRecords: ISSUE_WAKE_DIAGNOSTICS_MAX_ACTIVITY_RECORDS, + lookbackDays: ISSUE_WAKE_DIAGNOSTICS_LOOKBACK_DAYS, + }, + }; + }, + + getSubtreeDiagnostics: async ( + issueId: string, + opts?: { + maxDepth?: number; + maxNodes?: number; + maxBlockersPerNode?: number; + maxWakeRequestsPerNode?: number; + maxActivityRecordsPerNode?: number; + lookbackDays?: number; + }, + ) => { + const issue = await db + .select({ id: issues.id, companyId: issues.companyId }) + .from(issues) + .where(eq(issues.id, issueId)) + .then((rows) => rows[0] ?? null); + if (!issue) throw notFound("Issue not found"); + + const maxDepth = Math.max( + 0, + Math.min( + opts?.maxDepth ?? ISSUE_SUBTREE_DIAGNOSTICS_MAX_DEPTH, + ISSUE_SUBTREE_DIAGNOSTICS_MAX_DEPTH, + ), + ); + const maxNodes = Math.max( + 1, + Math.min( + opts?.maxNodes ?? ISSUE_SUBTREE_DIAGNOSTICS_MAX_NODES, + ISSUE_SUBTREE_DIAGNOSTICS_MAX_NODES, + ), + ); + const maxBlockersPerNode = Math.max( + 0, + Math.min( + opts?.maxBlockersPerNode ?? + ISSUE_SUBTREE_DIAGNOSTICS_MAX_BLOCKERS_PER_NODE, + ISSUE_SUBTREE_DIAGNOSTICS_MAX_BLOCKERS_PER_NODE, + ), + ); + const maxWakeRequestsPerNode = Math.max( + 0, + Math.min( + opts?.maxWakeRequestsPerNode ?? + ISSUE_SUBTREE_DIAGNOSTICS_MAX_WAKE_REQUESTS_PER_NODE, + ISSUE_SUBTREE_DIAGNOSTICS_MAX_WAKE_REQUESTS_PER_NODE, + ), + ); + const maxActivityRecordsPerNode = Math.max( + 0, + Math.min( + opts?.maxActivityRecordsPerNode ?? + ISSUE_SUBTREE_DIAGNOSTICS_MAX_ACTIVITY_RECORDS_PER_NODE, + ISSUE_SUBTREE_DIAGNOSTICS_MAX_ACTIVITY_RECORDS_PER_NODE, + ), + ); + const lookbackDays = Math.max( + 1, + Math.min( + opts?.lookbackDays ?? ISSUE_WAKE_DIAGNOSTICS_LOOKBACK_DAYS, + ISSUE_WAKE_DIAGNOSTICS_LOOKBACK_DAYS, + ), + ); + const since = new Date(Date.now() - lookbackDays * 24 * 60 * 60 * 1000); + const sinceIso = since.toISOString(); + + const rawSubtreeRows = + await db.execute(sql` + WITH RECURSIVE issue_tree AS ( + SELECT + id, + company_id, + project_id, + parent_id, + identifier, + title, + status, + priority, + assignee_agent_id, + assignee_user_id, + created_at, + updated_at, + 0 AS depth, + ARRAY[id] AS path + FROM issues + WHERE company_id = ${issue.companyId} + AND id = ${issue.id} + AND hidden_at IS NULL + AND harness_kind IS NULL + UNION ALL + SELECT + child.id, + child.company_id, + child.project_id, + child.parent_id, + child.identifier, + child.title, + child.status, + child.priority, + child.assignee_agent_id, + child.assignee_user_id, + child.created_at, + child.updated_at, + issue_tree.depth + 1, + issue_tree.path || child.id + FROM issues child + JOIN issue_tree ON child.parent_id = issue_tree.id + WHERE child.company_id = ${issue.companyId} + AND child.hidden_at IS NULL + AND child.harness_kind IS NULL + AND issue_tree.depth < ${maxDepth + 1} + AND NOT child.id = ANY(issue_tree.path) + ) + SELECT + id, + company_id AS "companyId", + project_id AS "projectId", + parent_id AS "parentId", + identifier, + title, + status, + priority, + assignee_agent_id AS "assigneeAgentId", + assignee_user_id AS "assigneeUserId", + created_at AS "createdAt", + updated_at AS "updatedAt", + depth::int AS depth + FROM issue_tree + ORDER BY depth ASC, created_at ASC, id ASC + LIMIT ${maxNodes + 1} + `); + const subtreeRows = Array.from(rawSubtreeRows).map((row) => ({ + ...row, + depth: Number(row.depth), + })); + const rowsWithinDepth = subtreeRows.filter( + (row) => row.depth <= maxDepth, + ); + const nodes = rowsWithinDepth.slice( + 0, + maxNodes, + ) as IssueSubtreeDiagnosticsIssueRow[]; + const truncatedNodes = rowsWithinDepth.length > maxNodes; + const truncatedDepth = + truncatedNodes || subtreeRows.some((row) => row.depth > maxDepth); + const nodeIds = nodes.map((node) => node.id); + + const readiness = + nodeIds.length > 0 + ? await listIssueDependencyReadinessMap(db, issue.companyId, nodeIds) + : new Map(); + const blockersByIssueId = new Map< + string, + IssueSubtreeDiagnosticsBlockerRow[] + >(); + const wakeRequestsByIssueId = new Map< + string, + IssueSubtreeDiagnosticsWakeRequestRow[] + >(); + const activityRecordsByIssueId = new Map< + string, + IssueSubtreeDiagnosticsActivityRow[] + >(); + const truncatedBlockerIssueIds = new Set(); + const truncatedWakeIssueIds = new Set(); + const truncatedActivityIssueIds = new Set(); + + if (nodeIds.length > 0) { + const nodeIdValues = sql.join( + nodeIds.map((id) => sql`${id}`), + sql`, `, + ); + const rawBlockerRows = Array.from( + await db.execute(sql` + WITH blocker_rows AS ( + SELECT + blocker.id, + blocker.company_id AS "companyId", + blocker.project_id AS "projectId", + blocker.parent_id AS "parentId", + blocker.identifier, + blocker.title, + blocker.status, + blocker.priority, + blocker.assignee_agent_id AS "assigneeAgentId", + blocker.assignee_user_id AS "assigneeUserId", + relation.related_issue_id AS "blockedIssueId", + relation.created_at AS "relationCreatedAt", + row_number() OVER ( + PARTITION BY relation.related_issue_id + ORDER BY blocker.title ASC, blocker.id ASC + )::int AS "rowNumber" + FROM issue_relations relation + INNER JOIN issues blocker ON blocker.id = relation.issue_id + WHERE relation.company_id = ${issue.companyId} + AND relation.type = 'blocks' + AND blocker.company_id = ${issue.companyId} + AND blocker.hidden_at IS NULL + AND blocker.harness_kind IS NULL + AND relation.related_issue_id::text IN (${nodeIdValues}) + ) + SELECT * + FROM blocker_rows + WHERE "rowNumber" <= ${maxBlockersPerNode + 1} + ORDER BY "blockedIssueId" ASC, "rowNumber" ASC + `), + ) as IssueSubtreeDiagnosticsBlockerResultRow[]; + for (const row of rawBlockerRows) { + const normalized = { ...row, rowNumber: Number(row.rowNumber) }; + if (normalized.rowNumber > maxBlockersPerNode) { + truncatedBlockerIssueIds.add(normalized.blockedIssueId); + continue; + } + const rows = blockersByIssueId.get(normalized.blockedIssueId) ?? []; + rows.push(normalized); + blockersByIssueId.set(normalized.blockedIssueId, rows); + } + + const wakeTargetIssueIdSql = sql` + coalesce( + wake.payload ->> 'issueId', + wake.payload ->> 'taskId', + wake.payload -> '_paperclipWakeContext' ->> 'issueId', + wake.payload -> '_paperclipWakeContext' ->> 'taskId' + ) + `; + const rawWakeRows = Array.from( + await db.execute(sql` + WITH wake_rows AS ( + SELECT + ${wakeTargetIssueIdSql} AS "issueId", + wake.agent_id AS "agentId", + wake.source, + wake.reason, + wake.status, + wake.coalesced_count AS "coalescedCount", + wake.run_id AS "runId", + wake.requested_at AS "requestedAt", + wake.claimed_at AS "claimedAt", + wake.finished_at AS "finishedAt", + wake.error, + row_number() OVER ( + PARTITION BY ${wakeTargetIssueIdSql} + ORDER BY wake.requested_at DESC, wake.created_at DESC + )::int AS "rowNumber" + FROM agent_wakeup_requests wake + WHERE wake.company_id = ${issue.companyId} + AND wake.requested_at >= ${sinceIso}::timestamptz + AND ${wakeTargetIssueIdSql} IN (${nodeIdValues}) + ) + SELECT * + FROM wake_rows + WHERE "rowNumber" <= ${maxWakeRequestsPerNode + 1} + ORDER BY "issueId" ASC, "requestedAt" DESC + `), + ) as IssueSubtreeDiagnosticsWakeRequestResultRow[]; + for (const row of rawWakeRows) { + const normalized = { ...row, rowNumber: Number(row.rowNumber) }; + if (normalized.rowNumber > maxWakeRequestsPerNode) { + truncatedWakeIssueIds.add(normalized.issueId); + continue; + } + const rows = wakeRequestsByIssueId.get(normalized.issueId) ?? []; + rows.push(normalized); + wakeRequestsByIssueId.set(normalized.issueId, rows); + } + + const activityTargetIssueIdSql = sql` + coalesce( + CASE WHEN activity.entity_type = 'issue' THEN activity.entity_id ELSE NULL END, + activity.details ->> 'issueId', + activity.details ->> 'rootIssueId' + ) + `; + const activityActionValues = sql.join( + ISSUE_WAKE_DIAGNOSTICS_ACTIVITY_ACTIONS.map( + (action) => sql`${action}`, + ), + sql`, `, + ); + const rawActivityRows = Array.from( + await db.execute(sql` + WITH activity_rows AS ( + SELECT + ${activityTargetIssueIdSql} AS "issueId", + activity.action, + activity.entity_type AS "entityType", + activity.entity_id AS "entityId", + activity.agent_id AS "agentId", + activity.run_id AS "runId", + activity.details, + activity.created_at AS "createdAt", + row_number() OVER ( + PARTITION BY ${activityTargetIssueIdSql} + ORDER BY activity.created_at DESC, activity.id DESC + )::int AS "rowNumber" + FROM activity_log activity + WHERE activity.company_id = ${issue.companyId} + AND activity.created_at >= ${sinceIso}::timestamptz + AND activity.action IN (${activityActionValues}) + AND ${activityTargetIssueIdSql} IN (${nodeIdValues}) + ) + SELECT * + FROM activity_rows + WHERE "rowNumber" <= ${maxActivityRecordsPerNode + 1} + ORDER BY "issueId" ASC, "createdAt" DESC + `), + ) as IssueSubtreeDiagnosticsActivityResultRow[]; + for (const row of rawActivityRows) { + const normalized = { ...row, rowNumber: Number(row.rowNumber) }; + if (normalized.rowNumber > maxActivityRecordsPerNode) { + truncatedActivityIssueIds.add(normalized.issueId); + continue; + } + const rows = activityRecordsByIssueId.get(normalized.issueId) ?? []; + rows.push(normalized); + activityRecordsByIssueId.set(normalized.issueId, rows); + } + } + + return { + nodes, + blockersByIssueId, + readinessByIssueId: readiness, + wakeRequestsByIssueId, + activityRecordsByIssueId, + truncatedNodes, + truncatedDepth, + truncatedBlockerIssueIds, + truncatedWakeIssueIds, + truncatedActivityIssueIds, + caps: { + maxDepth, + maxNodes, + maxBlockersPerNode, + maxWakeRequestsPerNode, + maxActivityRecordsPerNode, + lookbackDays, + }, + }; + }, + + + + getDependencyReadiness: async ( + issueId: string, + dbOrTx: any = db, + options: { lockSources?: boolean } = {}, + ) => { + const issue = await dbOrTx + .select({ id: issues.id, companyId: issues.companyId }) + .from(issues) + .where(eq(issues.id, issueId)) + .then( + (rows: Array<{ id: string; companyId: string }>) => rows[0] ?? null, + ); + if (!issue) throw notFound("Issue not found"); + const readiness = await listIssueDependencyReadinessMap( + dbOrTx, + issue.companyId, + [issueId], + options.lockSources, + ); + return readiness.get(issueId) ?? createIssueDependencyReadiness(issueId); + }, + + listDependencyReadiness: async ( + companyId: string, + issueIds: string[], + dbOrTx: any = db, + ) => { + return listIssueDependencyReadinessMap(dbOrTx, companyId, issueIds); + }, + + listBlockerAttention: async ( + companyId: string, + issueRows: IssueBlockerAttentionInputNode[], + dbOrTx: any = db, + ) => { + return listIssueBlockerAttentionMap(dbOrTx, companyId, issueRows); + }, + + listReviewAttention: async ( + companyId: string, + issueRows: IssueReviewAttentionInput[], + dbOrTx: any = db, + ) => { + return listIssueReviewAttentionMap(dbOrTx, companyId, issueRows); + }, + + listWakeableBlockedDependents: async (blockerIssueId: string) => { + const blockerIssue = await db + .select({ id: issues.id, companyId: issues.companyId }) + .from(issues) + .where(eq(issues.id, blockerIssueId)) + .then((rows) => rows[0] ?? null); + if (!blockerIssue) return []; + + const candidates = await db + .select({ + id: issues.id, + assigneeAgentId: issues.assigneeAgentId, + status: issues.status, + blockedTransitionAt: issues.blockedTransitionAt, + }) + .from(issueRelations) + .innerJoin(issues, eq(issueRelations.relatedIssueId, issues.id)) + .where( + and( + eq(issueRelations.companyId, blockerIssue.companyId), + eq(issueRelations.type, "blocks"), + eq(issueRelations.issueId, blockerIssueId), + isNull(issues.conversationAgentId), + ), + ); + if (candidates.length === 0) return []; + + const wakeableCandidates = candidates.filter( + (candidate) => + candidate.assigneeAgentId && + !["backlog", "done", "cancelled"].includes(candidate.status), + ); + if (wakeableCandidates.length === 0) return []; + + // Defer to the unified readiness check so that a dependent only fires when + // (a) every blocker is done AND (b) every done blocker's workspace has + // recorded a successful workspace_finalize. The finalize hook also calls + // this function on completion, so a wake initially gated by an in-flight + // sync-back will re-fire once the restore lands locally. + const readinessMap = await listIssueDependencyReadinessMap( + db, + blockerIssue.companyId, + wakeableCandidates.map((candidate) => candidate.id), + ); + + return wakeableCandidates + .map((candidate) => { + const readiness = + readinessMap.get(candidate.id) ?? + createIssueDependencyReadiness(candidate.id); + return { candidate, readiness }; + }) + .filter( + ({ readiness }) => + readiness.isDependencyReady && readiness.blockerIssueIds.length > 0, + ) + .map(({ candidate, readiness }) => ({ + id: candidate.id, + assigneeAgentId: candidate.assigneeAgentId!, + blockerIssueIds: readiness.blockerIssueIds, + blockedTransitionAt: candidate.blockedTransitionAt, + })); + }, + + getWakeableParentAfterChildCompletion: async ( + parentIssueId: string, + completedChildResult?: { issueId: string; summary: string | null } | null, + ) => { + const parent = await db + .select({ + id: issues.id, + conversationAgentId: issues.conversationAgentId, + originKind: issues.originKind, + assigneeAgentId: issues.assigneeAgentId, + status: issues.status, + companyId: issues.companyId, + }) + .from(issues) + .where(eq(issues.id, parentIssueId)) + .then((rows) => rows[0] ?? null); + if (!parent || parent.conversationAgentId || !parent.assigneeAgentId || ["backlog", "cancelled"].includes(parent.status) || + (parent.status === "done" && parent.originKind !== "onboarding_first_task")) { + return null; + } + + const children = await db + .select({ + id: issues.id, + identifier: issues.identifier, + title: issues.title, + status: issues.status, + priority: issues.priority, + assigneeAgentId: issues.assigneeAgentId, + assigneeUserId: issues.assigneeUserId, + updatedAt: issues.updatedAt, + }) + .from(issues) + .where( + and( + eq(issues.companyId, parent.companyId), + eq(issues.parentId, parentIssueId), + ), + ) + .orderBy(asc(issues.issueNumber), asc(issues.createdAt)); + if (children.length === 0) return null; + if ( + !children.every( + (child) => child.status === "done" || child.status === "cancelled", + ) + ) { + return null; + } + + const childIdsForSummaries = children + .slice(0, MAX_CHILD_COMPLETION_SUMMARIES) + .map((child) => child.id); + const commentRows = + childIdsForSummaries.length > 0 + ? await db + .select({ + issueId: issueComments.issueId, + body: issueComments.body, + createdAt: issueComments.createdAt, + }) + .from(issueComments) + .where( + and( + eq(issueComments.companyId, parent.companyId), + inArray(issueComments.issueId, childIdsForSummaries), + isNull(issueComments.deletedAt), + ), + ) + .orderBy(desc(issueComments.createdAt), desc(issueComments.id)) + : []; + const latestCommentByIssueId = new Map(); + for (const comment of commentRows) { + if (!latestCommentByIssueId.has(comment.issueId)) { + latestCommentByIssueId.set(comment.issueId, comment.body); + } + } + const childIssueSummaries: ChildIssueCompletionSummary[] = children + .slice(0, MAX_CHILD_COMPLETION_SUMMARIES) + .map((child) => ({ + ...child, + summary: truncateInlineSummary( + child.id === completedChildResult?.issueId + ? (completedChildResult.summary ?? + latestCommentByIssueId.get(child.id)) + : latestCommentByIssueId.get(child.id), + ), + })); + + return { + onboardingCompletion: parent.originKind === "onboarding_first_task", + id: parent.id, + assigneeAgentId: parent.assigneeAgentId, + childIssueIds: children.map((child) => child.id), + childIssueSummaries, + childIssueSummaryTruncated: + children.length > childIssueSummaries.length, + }; + }, + + createChild: async (parentIssueId: string, data: IssueChildCreateInput) => { + const parent = await db + .select() + .from(issues) + .where(eq(issues.id, parentIssueId)) + .then((rows) => rows[0] ?? null); + if (!parent) throw notFound("Parent issue not found"); + await assertExecutionTaskParent(db, parent.companyId, parent.id); + + const idempotencyKey = data.idempotencyKey?.trim(); + if (idempotencyKey) { + const existingChild = await db + .select({ issue: issues }) + .from(issueCreateIdempotencyKeys) + .innerJoin(issues, eq(issueCreateIdempotencyKeys.issueId, issues.id)) + .where( + and( + eq(issueCreateIdempotencyKeys.companyId, parent.companyId), + eq(issueCreateIdempotencyKeys.idempotencyKey, idempotencyKey), + ), + ) + .limit(1) + .then((rows) => rows[0]?.issue ?? null); + if (existingChild) { + if (existingChild.parentId !== parent.id) { + throw conflict( + "Child creation idempotency key belongs to another parent issue", + ); + } + await data.assertCanReuseIssue?.(existingChild); + assertQualityReviewPolicyReuse(data.executionPolicy, existingChild.executionPolicy); + data.onDeduplicated?.("idempotency_key"); + const [enriched] = await withIssueLabels(db, [existingChild]); + const [withRelations] = await withIssueRelationSummaries( + parent.companyId, + [enriched], + db, + ); + return { + issue: withRelations, + parentBlockerAdded: false, + }; + } + } + + const [{ childCount }] = await db + .select({ childCount: sql`count(*)::int` }) + .from(issues) + .where( + and( + eq(issues.companyId, parent.companyId), + eq(issues.parentId, parent.id), + notInArray(issues.status, ["done", "cancelled"]), + ), + ); + if (childCount >= MAX_CHILD_ISSUES_CREATED_BY_HELPER) { + throw unprocessable( + `Parent issue already has the maximum ${MAX_CHILD_ISSUES_CREATED_BY_HELPER} active child issues for this helper`, + ); + } + + const { + acceptanceCriteria, + blockParentUntilDone, + executionWorkspaceInheritanceMode = "linkage", + actorAgentId, + actorUserId, + ...issueData + } = data; + + const inheritStrategyOnly = + executionWorkspaceInheritanceMode === "strategy_only"; + // A child may target another project. Parent workspace identity is only + // valid inside the parent's project, so do not forward it across that + // boundary; create() then resolves the target project's own workspaces. + const childProjectId = issueData.projectId ?? parent.projectId; + const childInheritsParentProject = childProjectId === parent.projectId; + const hasExplicitExecutionWorkspaceOverride = + issueData.projectWorkspaceId !== undefined || + issueData.executionWorkspaceId !== undefined || + issueData.executionWorkspacePreference !== undefined || + issueData.executionWorkspaceSettings !== undefined; + const inheritedPreRealizationWorkspaceSettings = + inheritStrategyOnly && !hasExplicitExecutionWorkspaceOverride + ? buildPreRealizationExecutionWorkspaceSettings( + parent.executionWorkspaceSettings, + ) + : null; + let child = await issueService(db).create(parent.companyId, { + ...issueData, + parentId: parent.id, + projectId: childProjectId, + projectWorkspaceId: + issueData.projectWorkspaceId !== undefined + ? issueData.projectWorkspaceId + : (inheritStrategyOnly && childInheritsParentProject + ? parent.projectWorkspaceId + : undefined), + goalId: issueData.goalId ?? parent.goalId, + actorResponsibleUserId: issueData.actorResponsibleUserId ?? null, + trustExplicitResponsibleUserId: + issueData.trustExplicitResponsibleUserId === true, + requestDepth: clampIssueRequestDepth( + Math.max( + clampIssueRequestDepth(parent.requestDepth) + 1, + issueData.requestDepth ?? 0, + ), + ), + description: appendAcceptanceCriteriaToDescription( + issueData.description, + acceptanceCriteria, + ), + ...(inheritedPreRealizationWorkspaceSettings + ? { + executionWorkspaceSettings: + inheritedPreRealizationWorkspaceSettings, + } + : {}), + ...(inheritStrategyOnly + ? { skipExecutionWorkspaceInheritance: true } + : { inheritExecutionWorkspaceFromIssueId: parent.id }), + }); + + if (blockParentUntilDone) { + const existingBlockers = await db + .select({ blockerIssueId: issueRelations.issueId }) + .from(issueRelations) + .where( + and( + eq(issueRelations.companyId, parent.companyId), + eq(issueRelations.relatedIssueId, parent.id), + eq(issueRelations.type, "blocks"), + ), + ); + await syncBlockedByIssueIds( + parent.id, + parent.companyId, + [ + ...new Set([ + ...existingBlockers.map((row) => row.blockerIssueId), + child.id, + ]), + ], + { agentId: actorAgentId ?? null, userId: actorUserId ?? null }, + ); + [child] = await withIssueRelationSummaries( + parent.companyId, + [child], + db, + ); + } + + return { + issue: child, + parentBlockerAdded: Boolean(blockParentUntilDone), + }; + }, + + decomposeAcceptedPlan: async ( + sourceIssueId: string, + data: AcceptedPlanDecompositionInput, + ) => { + const sourceIssue = await db + .select({ + id: issues.id, + companyId: issues.companyId, + projectId: issues.projectId, + goalId: issues.goalId, + }) + .from(issues) + .where(eq(issues.id, sourceIssueId)) + .then((rows) => rows[0] ?? null); + if (!sourceIssue) throw notFound("Source issue not found"); + + const requestFingerprint = + createAcceptedPlanDecompositionRequestFingerprint({ + acceptedPlanRevisionId: data.acceptedPlanRevisionId, + children: data.children, + }); + + const initialClaim = await db.transaction(async (tx) => { + await tx.execute( + sql`select ${issues.id} from ${issues} where ${issues.id} = ${sourceIssue.id} for update`, + ); + + const belongsToPlanDocument = await tx + .select({ revisionId: documentRevisions.id }) + .from(issueDocuments) + .innerJoin( + documentRevisions, + eq(issueDocuments.documentId, documentRevisions.documentId), + ) + .where( + and( + eq(issueDocuments.companyId, sourceIssue.companyId), + eq(issueDocuments.issueId, sourceIssue.id), + eq(issueDocuments.key, "plan"), + eq(documentRevisions.id, data.acceptedPlanRevisionId), + ), + ) + .then((rows) => rows[0] ?? null); + if (!belongsToPlanDocument) { + throw unprocessable( + "acceptedPlanRevisionId must belong to the source issue's plan document", + ); + } + + const acceptedInteraction = await findAcceptedPlanDocumentInteraction( + tx, + { + companyId: sourceIssue.companyId, + sourceIssueId: sourceIssue.id, + acceptedPlanRevisionId: data.acceptedPlanRevisionId, + }, + ); + if (!acceptedInteraction) { + throw unprocessable( + "acceptedPlanRevisionId must have an accepted plan confirmation", + ); + } + + const existing = await tx + .select() + .from(issuePlanDecompositions) + .where( + and( + eq(issuePlanDecompositions.companyId, sourceIssue.companyId), + eq(issuePlanDecompositions.sourceIssueId, sourceIssue.id), + eq( + issuePlanDecompositions.acceptedPlanRevisionId, + data.acceptedPlanRevisionId, + ), + ), + ) + .then((rows) => rows[0] ?? null); + + const now = new Date(); + if (!existing) { + const [created] = await tx + .insert(issuePlanDecompositions) + .values({ + companyId: sourceIssue.companyId, + sourceIssueId: sourceIssue.id, + acceptedPlanRevisionId: data.acceptedPlanRevisionId, + acceptedInteractionId: acceptedInteraction.id, + status: "in_flight", + requestFingerprint, + requestedChildCount: data.children.length, + requestedChildren: data.children as unknown as Record< + string, + unknown + >[], + childIssueIds: [], + ownerAgentId: data.actorAgentId ?? null, + ownerUserId: data.actorUserId ?? null, + ownerRunId: data.actorRunId ?? null, + updatedAt: now, + }) + .returning(); + if (!created) + throw new Error( + "Failed to create accepted-plan decomposition claim", + ); + return created; + } + + if (existing.requestFingerprint !== requestFingerprint) { + throw conflict( + "Accepted-plan decomposition already exists for this revision with a different child set", + ); + } + + return existing; + }); + + let currentClaim = initialClaim; + const newlyCreatedIssues: Array = []; + + while (true) { + const step = await db.transaction(async (tx) => { + await tx.execute( + sql`select ${issuePlanDecompositions.id} + from ${issuePlanDecompositions} + where ${issuePlanDecompositions.id} = ${currentClaim.id} + for update`, + ); + + const claim = await tx + .select() + .from(issuePlanDecompositions) + .where(eq(issuePlanDecompositions.id, currentClaim.id)) + .then((rows) => rows[0] ?? null); + if (!claim) + throw notFound("Accepted-plan decomposition claim not found"); + if (claim.requestFingerprint !== requestFingerprint) { + throw conflict( + "Accepted-plan decomposition already exists for this revision with a different child set", + ); + } + + const existingChildIssueIds = normalizeIssuePlanDecompositionChildIds( + claim.childIssueIds, + ); + if ( + claim.status === "completed" || + existingChildIssueIds.length >= data.children.length + ) { + const nextIds = existingChildIssueIds.slice( + 0, + data.children.length, + ); + if ( + claim.status === "completed" && + nextIds.length === data.children.length + ) { + return { + claim, + createdIssue: null, + }; + } + + const completedAt = claim.completedAt ?? new Date(); + const ownerPatch = await resolveAcceptedPlanClaimOwner({ + dbOrTx: tx, + claim, + actorAgentId: data.actorAgentId, + actorUserId: data.actorUserId, + actorRunId: data.actorRunId, + }); + const [completed] = await tx + .update(issuePlanDecompositions) + .set({ + status: "completed", + childIssueIds: nextIds, + completedAt, + ...ownerPatch, + updatedAt: completedAt, + }) + .where(eq(issuePlanDecompositions.id, claim.id)) + .returning(); + if (!completed) + throw new Error( + "Failed to complete accepted-plan decomposition claim", + ); + return { + claim: completed, + createdIssue: null, + }; + } + + const nextChildInput = data.children[existingChildIssueIds.length]; + if (!nextChildInput) { + throw new Error( + "Accepted-plan decomposition child cursor moved past the requested children", + ); + } + + const createdChild = await issueService( + tx as unknown as Db, + ).createChild(sourceIssue.id, { + ...nextChildInput, + executionWorkspaceInheritanceMode: "strategy_only", + }); + const nextIds = [...existingChildIssueIds, createdChild.issue.id]; + const now = new Date(); + const nextStatus = + nextIds.length === data.children.length ? "completed" : "in_flight"; + const ownerPatch = await resolveAcceptedPlanClaimOwner({ + dbOrTx: tx, + claim, + actorAgentId: data.actorAgentId, + actorUserId: data.actorUserId, + actorRunId: data.actorRunId, + }); + const [updatedClaim] = await tx + .update(issuePlanDecompositions) + .set({ + status: nextStatus, + childIssueIds: nextIds, + completedAt: nextStatus === "completed" ? now : null, + ...ownerPatch, + updatedAt: now, + }) + .where(eq(issuePlanDecompositions.id, claim.id)) + .returning(); + if (!updatedClaim) + throw new Error( + "Failed to persist accepted-plan decomposition progress", + ); + return { + claim: updatedClaim, + createdIssue: createdChild.issue, + }; + }); + + currentClaim = step.claim; + if (step.createdIssue) { + newlyCreatedIssues.push(step.createdIssue); + } + if (step.claim.status === "completed") break; + } + + const childIssueIds = normalizeIssuePlanDecompositionChildIds( + currentClaim.childIssueIds, + ); + const childIssueRows = + childIssueIds.length > 0 + ? await db + .select() + .from(issues) + .where( + and( + eq(issues.companyId, sourceIssue.companyId), + inArray(issues.id, childIssueIds), + ), + ) + : []; + const childIssueMap = new Map(childIssueRows.map((row) => [row.id, row])); + const orderedChildIssues = childIssueIds + .map((childIssueId) => childIssueMap.get(childIssueId)) + .filter((row): row is typeof issues.$inferSelect => Boolean(row)); + + const decomposition = serializeAcceptedPlanDecomposition(currentClaim); + + return { + decomposition, + childIssueIds: decomposition.childIssueIds, + childIssues: orderedChildIssues, + newlyCreatedIssues, + }; + }, + + listAcceptedPlanDecompositions: async (sourceIssueId: string) => { + const sourceIssue = await db + .select({ id: issues.id, companyId: issues.companyId }) + .from(issues) + .where(eq(issues.id, sourceIssueId)) + .then((rows) => rows[0] ?? null); + if (!sourceIssue) return []; + + const rows = await db + .select({ + decomposition: issuePlanDecompositions, + revisionNumber: documentRevisions.revisionNumber, + }) + .from(issuePlanDecompositions) + .leftJoin( + documentRevisions, + eq( + documentRevisions.id, + issuePlanDecompositions.acceptedPlanRevisionId, + ), + ) + .where( + and( + eq(issuePlanDecompositions.companyId, sourceIssue.companyId), + eq(issuePlanDecompositions.sourceIssueId, sourceIssue.id), + ), + ) + .orderBy(desc(issuePlanDecompositions.createdAt)); + + if (rows.length === 0) return []; + + const allChildIds = new Set(); + for (const row of rows) { + for (const childId of normalizeIssuePlanDecompositionChildIds( + row.decomposition.childIssueIds, + )) { + allChildIds.add(childId); + } + } + + const childIssueRows = + allChildIds.size > 0 + ? await db + .select({ + id: issues.id, + identifier: issues.identifier, + title: issues.title, + status: issues.status, + priority: issues.priority, + assigneeAgentId: issues.assigneeAgentId, + assigneeUserId: issues.assigneeUserId, + }) + .from(issues) + .where( + and( + eq(issues.companyId, sourceIssue.companyId), + inArray(issues.id, Array.from(allChildIds)), + ), + ) + : []; + const childIssueMap = new Map(childIssueRows.map((row) => [row.id, row])); + + return rows.map((row) => { + const decomposition = serializeAcceptedPlanDecomposition( + row.decomposition, + ); + const childIds = decomposition.childIssueIds; + return { + ...decomposition, + acceptedPlanRevisionNumber: row.revisionNumber ?? null, + childIssues: childIds + .map((childId) => childIssueMap.get(childId) ?? null) + .filter( + (entry): entry is NonNullable => entry !== null, + ), + }; + }); + }, + + listConversations: async (companyId: string, userId: string) => db.select().from(issues).where(and( + eq(issues.companyId, companyId), eq(issues.conversationUserId, userId), isNotNull(issues.conversationAgentId), + )).orderBy(desc(issues.updatedAt), asc(issues.id)), + + getConversation: async (companyId: string, agentId: string, userId: string) => db.select().from(issues).where(and( + eq(issues.companyId, companyId), eq(issues.conversationAgentId, agentId), eq(issues.conversationUserId, userId), + )).then((rows) => rows[0] ?? null), + + create: async ( + companyId: string, + data: IssueCreateInput, + dbOrTx: Db | DbTransaction = db, + ) => { + const { + workspaceSelection, workspaceSelectionSource = "explicit", workspaceSelectionActor, + initialPlan, + conversationHandoff, + labelIds: inputLabelIds, + blockedByIssueIds, + inheritExecutionWorkspaceFromIssueId, + skipExecutionWorkspaceInheritance, + watchdog, + watchdogActorRunId, + actorRunId, + actorResponsibleUserId, + trustExplicitResponsibleUserId, + idempotencyKey: rawIdempotencyKey, + allowDuplicate, + assertCanReuseIssue, + onDeduplicated, + ...issueData + } = data; + const explicitTitle = issueData.title?.trim(); + const provisionalTitle = issueData.description + ? provisionalTitleFromDescription(issueData.description) + : undefined; + const resolvedTitle = explicitTitle || provisionalTitle; + if (!resolvedTitle) throw unprocessable("Provide a title or task description"); + const titleNeedsGeneration = !explicitTitle; + // A prompt prefix is not a task identity: distinct requests can share it. + const deduplicateByTitle = allowDuplicate === false && !titleNeedsGeneration; + issueData.title = resolvedTitle; + const isolatedWorkspacesEnabled = ( + await instanceSettings.getExperimental() + ).enableIsolatedWorkspaces; + if (!isolatedWorkspacesEnabled) { + delete issueData.executionWorkspaceId; + delete issueData.executionWorkspacePreference; + delete issueData.executionWorkspaceSettings; + } + if (data.assigneeAgentId && data.assigneeUserId) { + throw unprocessable("Issue can only have one assignee"); + } + if (data.assigneeAgentId) { + await assertAssignableAgent(db, companyId, data.assigneeAgentId, { + kind: "work", + }); + } + if (data.assigneeUserId) { + await assertAssignableUser(companyId, data.assigneeUserId); + } + if ( + data.status === "in_progress" && + !data.assigneeAgentId && + !data.assigneeUserId + ) { + throw unprocessable("in_progress issues require an assignee"); + } + const persist = async (tx: DbTransaction) => { + await tx.execute(sql`select pg_advisory_xact_lock(hashtextextended(${`issue-privacy-tree:${companyId}`}, 0))`); + await assertExecutionTaskParent(tx as unknown as Db, companyId, issueData.parentId); + if (issueData.conversationAgentId && issueData.conversationUserId) { + const identity = `conversation:${companyId}:${issueData.conversationAgentId}:${issueData.conversationUserId}`; + await tx.execute(sql`select pg_advisory_xact_lock(hashtextextended(${identity}, 0))`); + const [existing] = await tx.select().from(issues).where(and(eq(issues.companyId, companyId), + eq(issues.conversationAgentId, issueData.conversationAgentId), eq(issues.conversationUserId, issueData.conversationUserId))); + if (existing) { + const [enriched] = await withIssueLabels(tx, [existing]); + const [withRelations] = await withIssueRelationSummaries(companyId, [enriched], tx); + return withRelations; + } + } + const idempotencyKey = rawIdempotencyKey?.trim() || null; + const normalizedTitle = normalizeCreateIssueTitle(resolvedTitle); + if (deduplicateByTitle) { + const titleGuardKey = `issue-create:title:${companyId}:${issueData.parentId ?? "root"}:${normalizedTitle}`; + await tx.execute( + sql`select pg_advisory_xact_lock(hashtextextended(${titleGuardKey}, 0))`, + ); + } + if (idempotencyKey) { + const idempotencyGuardKey = `issue-create:idempotency:${companyId}:${idempotencyKey}`; + await tx.execute( + sql`select pg_advisory_xact_lock(hashtextextended(${idempotencyGuardKey}, 0))`, + ); + } + + let existingIssue: typeof issues.$inferSelect | undefined; + let deduplicationReason: + "idempotency_key" | "recent_open_title" | null = null; + if (idempotencyKey) { + const idempotencyKeyRetentionCutoff = new Date( + Date.now() - ISSUE_CREATE_IDEMPOTENCY_KEY_RETENTION_MS, + ); + await tx.execute(sql` + delete from ${issueCreateIdempotencyKeys} + where ${issueCreateIdempotencyKeys.id} in ( + select ${issueCreateIdempotencyKeys.id} + from ${issueCreateIdempotencyKeys} + where ${issueCreateIdempotencyKeys.companyId} = ${companyId} + and ${issueCreateIdempotencyKeys.createdAt} < ${idempotencyKeyRetentionCutoff.toISOString()}::timestamptz + order by ${issueCreateIdempotencyKeys.createdAt} asc, ${issueCreateIdempotencyKeys.id} asc + limit ${ISSUE_CREATE_IDEMPOTENCY_KEY_CLEANUP_BATCH_SIZE} + ) + `); + + [existingIssue] = await tx + .select() + .from(issueCreateIdempotencyKeys) + .innerJoin( + issues, + eq(issueCreateIdempotencyKeys.issueId, issues.id), + ) + .where( + and( + eq(issueCreateIdempotencyKeys.companyId, companyId), + eq(issueCreateIdempotencyKeys.idempotencyKey, idempotencyKey), + ), + ) + .limit(1) + .then((rows) => rows.map((row) => row.issues)); + if (existingIssue) deduplicationReason = "idempotency_key"; + } + if (!existingIssue && deduplicateByTitle) { + [existingIssue] = await tx + .select() + .from(issues) + .where( + and( + eq(issues.companyId, companyId), + issueData.parentId + ? eq(issues.parentId, issueData.parentId) + : isNull(issues.parentId), + isNull(issues.hiddenAt), + notInArray(issues.status, ["done", "cancelled"]), + gte( + issues.createdAt, + new Date(Date.now() - 48 * 60 * 60 * 1000), + ), + sql`lower(regexp_replace(btrim(${issues.title}), '\\s+', ' ', 'g')) = ${normalizedTitle}`, + ), + ) + .orderBy(asc(issues.createdAt), asc(issues.id)) + .limit(1); + if (existingIssue) deduplicationReason = "recent_open_title"; + } + if (existingIssue) { + // A duplicate may have a different scope or assignee than the proposed task. + await assertCanReuseIssue?.(existingIssue); + assertQualityReviewPolicyReuse(issueData.executionPolicy, existingIssue.executionPolicy); + if (idempotencyKey) { + await tx + .insert(issueCreateIdempotencyKeys) + .values({ companyId, idempotencyKey, issueId: existingIssue.id }) + .onConflictDoNothing(); + } + if (deduplicationReason) onDeduplicated?.(deduplicationReason); + const [enriched] = await withIssueLabels(tx, [existingIssue]); + const [withRelations] = await withIssueRelationSummaries( + companyId, + [enriched], + tx, + ); + return withRelations; + } + + if (issueData.status === "done" && requiredQualityReview(issueData.executionPolicy)) { + throw unprocessable("A new task with required quality review cannot start completed", { code: "required_quality_review_incomplete" }); + } + await assertOnboardingFirstTaskCreationAllowed(tx as unknown as Db, { + companyId, + actorAgentId: issueData.createdByAgentId, + actorRunId, + blockedByIssueIds, + }); + + const defaultCompanyGoal = await getDefaultCompanyGoal(tx, companyId); + let projectWorkspaceId = issueData.projectWorkspaceId ?? null; + let executionWorkspaceId = issueData.executionWorkspaceId ?? null; + let executionWorkspacePreference = + issueData.executionWorkspacePreference ?? null; + let executionWorkspaceSettings = + (issueData.executionWorkspaceSettings as + Record | null | undefined) ?? null; + let inheritedWorkspaceSelection = false; + const workspaceInheritanceIssueId = skipExecutionWorkspaceInheritance + ? null + : (inheritExecutionWorkspaceFromIssueId ?? + issueData.parentId ?? + null); + const hasExplicitExecutionWorkspaceOverride = + (workspaceSelection !== undefined && workspaceSelectionSource === "explicit") || + issueData.projectWorkspaceId !== undefined || + issueData.executionWorkspaceId !== undefined || + issueData.executionWorkspacePreference !== undefined || + issueData.executionWorkspaceSettings !== undefined; + if (workspaceInheritanceIssueId) { + const workspaceSource = await getWorkspaceInheritanceIssue( + tx, + companyId, + workspaceInheritanceIssueId, + ); + if (issueData.projectId == null && workspaceSource.projectId) { + issueData.projectId = workspaceSource.projectId; + } + // Workspace linkage is only inheritable inside the source project. A + // cross-project child (for example, a Paperclip ID issue created from + // a Paperclip App parent) must fall through to its own project's + // default workspaces, otherwise the inherited ids fail the + // project-match assertions below and the create is impossible without + // the caller naming the target workspaces explicitly. + const inheritsSourceProject = + issueData.projectId == null || + issueData.projectId === workspaceSource.projectId; + if ( + inheritsSourceProject && + issueData.projectWorkspaceId === undefined && + projectWorkspaceId == null && + workspaceSource.projectWorkspaceId + ) { + projectWorkspaceId = workspaceSource.projectWorkspaceId; + inheritedWorkspaceSelection = true; + } + if ( + inheritsSourceProject && + !hasExplicitExecutionWorkspaceOverride && + workspaceSource.executionWorkspaceId + ) { + const sourceWorkspace = await tx + .select({ + id: executionWorkspaces.id, + mode: executionWorkspaces.mode, + }) + .from(executionWorkspaces) + .where( + eq( + executionWorkspaces.id, + workspaceSource.executionWorkspaceId, + ), + ) + .then((rows) => rows[0] ?? null); + if (sourceWorkspace) { + executionWorkspaceId = sourceWorkspace.id; + inheritedWorkspaceSelection = true; + executionWorkspacePreference = "reuse_existing"; + executionWorkspaceSettings = { + ...((workspaceSource.executionWorkspaceSettings as + Record | null | undefined) ?? {}), + mode: issueExecutionWorkspaceModeForPersistedWorkspace( + sourceWorkspace.mode, + ), + }; + } + } + } + const workspaceActor: AuthorizationActor = workspaceSelectionActor ?? (issueData.createdByUserId + ? { type: "board", userId: issueData.createdByUserId, companyIds: [companyId] } + : { type: "agent", agentId: issueData.createdByAgentId ?? issueData.assigneeAgentId, companyId }); + let workspaceIntent: TaskWorkspaceIntent | null = null; + if (workspaceSelection && (workspaceSelectionSource === "explicit" || !inheritedWorkspaceSelection)) { + const projection = await executionWorkspaceService(db).validateSelection({ companyId, + actor: workspaceActor, + selection: workspaceSelection, parentIssueId: issueData.parentId, + taskCreation: { parentIssueId: issueData.parentId, projectId: issueData.projectId, + assigneeAgentId: issueData.assigneeAgentId, executionPolicy: issueData.executionPolicy }, + assigneeAgentId: issueData.assigneeAgentId, assigneeUserId: issueData.assigneeUserId, assigneeAdapterOverrides: issueData.assigneeAdapterOverrides, projectId: issueData.projectId, executionPolicy: issueData.executionPolicy }, tx); + projectWorkspaceId = projection.projectWorkspaceId; + executionWorkspaceId = projection.executionWorkspaceId; + executionWorkspacePreference = projection.executionWorkspacePreference; + executionWorkspaceSettings = projection.executionWorkspaceSettings; + workspaceIntent = { version: 1, selection: workspaceSelection, source: workspaceSelectionSource }; + } + if (executionWorkspaceId) { + const workspace = await assertValidExecutionWorkspace(companyId, null, executionWorkspaceId, tx); + projectWorkspaceId = workspace.projectWorkspaceId; + } + const projectGoalId = await getProjectDefaultGoalId( + tx, + companyId, + issueData.projectId, + ); + // Cache the project policy lookup for this insert so the default + // workspace-settings block does not re-query the project row. + let projectPolicyCached: ReturnType< + typeof parseProjectExecutionWorkspacePolicy + > | null = null; + let projectPolicyLoaded = false; + const loadProjectPolicyOnce = async () => { + if (projectPolicyLoaded) return projectPolicyCached; + projectPolicyLoaded = true; + if (!issueData.projectId) return null; + const projectRow = await tx + .select({ + executionWorkspacePolicy: projects.executionWorkspacePolicy, + }) + .from(projects) + .where( + and( + eq(projects.id, issueData.projectId), + eq(projects.companyId, companyId), + ), + ) + .then((rows) => rows[0] ?? null); + projectPolicyCached = parseProjectExecutionWorkspacePolicy( + projectRow?.executionWorkspacePolicy, + ); + return projectPolicyCached; + }; + + if ( + executionWorkspaceSettings == null && + executionWorkspaceId == null && + issueData.projectId + ) { + executionWorkspaceSettings = + defaultIssueExecutionWorkspaceSettingsForProject( + gateProjectExecutionWorkspacePolicy( + await loadProjectPolicyOnce(), + isolatedWorkspacesEnabled, + ), + ) as Record | null; + } + if (!executionWorkspaceId && !projectWorkspaceId && issueData.projectId && workspaceIntent?.selection.kind !== "task_directory") { + const project = await tx + .select({ + executionWorkspacePolicy: projects.executionWorkspacePolicy, + }) + .from(projects) + .where( + and( + eq(projects.id, issueData.projectId), + eq(projects.companyId, companyId), + ), + ) + .then((rows) => rows[0] ?? null); + const projectPolicy = parseProjectExecutionWorkspacePolicy( + project?.executionWorkspacePolicy, + ); + projectWorkspaceId = await resolveProjectDefaultWorkspaceId(tx, { + companyId, projectId: issueData.projectId, policy: projectPolicy, + }); + if (projectWorkspaceId && isolatedWorkspacesEnabled) { + const sourceStrategy = await executionWorkspaceService(db).resolveSourceWorkspaceStrategy({ companyId, + assigneeAgentId: issueData.assigneeAgentId, assigneeAdapterOverrides: issueData.assigneeAdapterOverrides, + executionWorkspaceSettings, executionWorkspacePolicy: projectPolicy, sourceWorkspaceId: projectWorkspaceId, + typedSelection: false, projectId: issueData.projectId, executionPolicy: issueData.executionPolicy }, tx); + if (sourceStrategy === "git_worktree") workspaceIntent = { version: 1, source: "project", + selection: { kind: "configured_source", projectWorkspaceId, mode: "managed_isolated" } }; + } + } + if (projectWorkspaceId && workspaceSelectionActor) { + const [readableSource] = await tx.select({ id: projectWorkspaces.id, projectId: projectWorkspaces.projectId, executionWorkspacePolicy: projects.executionWorkspacePolicy }).from(projectWorkspaces) + .innerJoin(projects, eq(projects.id, projectWorkspaces.projectId)) + .where(and(eq(projectWorkspaces.id, projectWorkspaceId), eq(projectWorkspaces.companyId, companyId), await projectReadSqlCondition(tx, workspaceActor))); + if (!readableSource) throw notFound("Workspace source is unavailable or inaccessible"); + await executionWorkspaceService(db).resolveSourceWorkspaceStrategy({ companyId, + assigneeAgentId: issueData.assigneeAgentId, assigneeAdapterOverrides: issueData.assigneeAdapterOverrides, + executionWorkspaceSettings, executionWorkspacePolicy: readableSource.executionWorkspacePolicy, sourceWorkspaceId: projectWorkspaceId, + typedSelection: Boolean(workspaceIntent), projectId: issueData.projectId, executionPolicy: issueData.executionPolicy }, tx); + await assertTaskWorkspaceSourceProjectAccess(tx, workspaceActor, companyId, readableSource.projectId); + } + if (executionWorkspaceId && workspaceSelectionActor) { + const [readableWorkspace] = await tx.select({ id: executionWorkspaces.id, projectId: executionWorkspaces.projectId }).from(executionWorkspaces) + .where(and(eq(executionWorkspaces.id, executionWorkspaceId), eq(executionWorkspaces.companyId, companyId), await executionWorkspaceReadSqlCondition(tx, workspaceActor))); + if (!readableWorkspace) throw notFound("Workspace is unavailable or inaccessible"); + await assertExecutionWorkspaceRecordAccess(tx, workspaceActor, companyId, readableWorkspace.id); + } + if (projectWorkspaceId) { + await assertValidProjectWorkspace( + companyId, + null, + projectWorkspaceId, + tx, + ); + } + if ( + isolatedWorkspacesEnabled && + issueData.executionWorkspaceSettings !== undefined + ) { + assertExplicitPinnedWorktreeIssueRunnable({ + projectId: issueData.projectId ?? null, + projectWorkspaceId, + executionWorkspaceId, + executionWorkspacePreference, + executionWorkspaceSettings: issueData.executionWorkspaceSettings, + }); + } + // Self-correcting counter: use MAX(issue_number) + 1 if the counter + // has drifted below the actual max, preventing identifier collisions. + const [maxRow] = await tx + .select({ + maxNum: sql`coalesce(max(${issues.issueNumber}), 0)`, + }) + .from(issues) + .where(eq(issues.companyId, companyId)); + const currentMax = maxRow?.maxNum ?? 0; + + const [company] = await tx + .update(companies) + .set({ + issueCounter: sql`greatest(${companies.issueCounter}, ${currentMax}) + 1`, + }) + .where(eq(companies.id, companyId)) + .returning({ + issueCounter: companies.issueCounter, + issuePrefix: companies.issuePrefix, + }); + + const issueNumber = company.issueCounter; + const identifier = `${company.issuePrefix}-${issueNumber}`; + const responsibleUserId = await resolveResponsibleUserIdForIssueCreate( + tx, + companyId, + { + explicitResponsibleUserId: issueData.responsibleUserId ?? null, + createdByUserId: issueData.createdByUserId ?? null, + parentId: issueData.parentId ?? null, + originKind: issueData.originKind ?? "manual", + originRunId: issueData.originRunId ?? null, + actorRunId: actorRunId ?? null, + actorResponsibleUserId: actorResponsibleUserId ?? null, + trustExplicitResponsibleUserId: + trustExplicitResponsibleUserId === true, + }, + ); + + let visibility = issueData.visibility ?? "open"; + let privacyRootIssueId: string | null = null; + // Provenance survives standalone chat handoffs. This is an access edge, + // independent of the product's structural parent/child relationship. + const handoffSource = conversationHandoff ? await lockAcceptedChatHandoffSource(tx, { + companyId, ...conversationHandoff, + userId: issueData.createdByUserId, agentId: issueData.createdByAgentId, + }) : null; + let privacyParentIssueId = issueData.parentId ?? handoffSource?.id ?? null; + const sourceRunId = issueData.originRunId ?? actorRunId; + if (!privacyParentIssueId && sourceRunId) { + const [sourceRun] = await tx.select().from(heartbeatRuns) + .where(and(eq(heartbeatRuns.id, sourceRunId), eq(heartbeatRuns.companyId, companyId))); + const sourceId = sourceRun?.nativeIssueId ?? sourceRun?.issueId ?? sourceRun?.contextSnapshot?.issueId; + if (typeof sourceId === "string") privacyParentIssueId = sourceId; + } + let inheritedPrivateSubtree = false; + if (privacyParentIssueId) { + const [source] = await tx.select().from(issues) + .where(and(eq(issues.id, privacyParentIssueId), eq(issues.companyId, companyId))); + if (!source) privacyParentIssueId = null; + else if (source.visibility === "private" || (source.projectId && await tx.select({ id: projects.id }).from(projects) + .where(and(eq(projects.id, source.projectId), eq(projects.visibility, "private"))).limit(1).then(rows => rows.length > 0))) { + visibility = "private"; + privacyRootIssueId = source.privacyRootIssueId ?? source.id; + inheritedPrivateSubtree = true; + } + } + if (visibility === "private" && !privacyRootIssueId) { + issueData.id ??= randomUUID(); + privacyRootIssueId = issueData.id; + } + + if (issueData.projectId) { + const selectedProject = await tx + .select({ visibility: projects.visibility }) + .from(projects) + .where(and(eq(projects.id, issueData.projectId), eq(projects.companyId, companyId))) + .then((rows) => rows[0] ?? null); + if (!selectedProject) throw notFound("Project not found"); + if (selectedProject.visibility === "private") { + visibility = "private"; + if (!privacyRootIssueId) { + issueData.id ??= randomUUID(); + privacyRootIssueId = issueData.id; + } + } + } + + if (visibility === "private" && !issueData.projectId && !inheritedPrivateSubtree) { + if (!responsibleUserId) { + throw unprocessable("Private tasks require a responsible user"); + } + const personalProject = await ensurePersonalPrivateProject(tx, companyId, responsibleUserId); + issueData.projectId = personalProject.id; + } + + const values = { + ...issueData, + workspaceSelection: workspaceIntent, + visibility, + privacyRootIssueId, + privacyParentIssueId, + titleNeedsGeneration, + originRunId: issueData.originRunId ?? actorRunId ?? null, + responsibleUserId, + requestDepth: clampIssueRequestDepth(issueData.requestDepth), + originKind: issueData.originKind ?? "manual", + goalId: resolveIssueGoalId({ + projectId: issueData.projectId, + goalId: issueData.goalId, + projectGoalId, + defaultGoalId: defaultCompanyGoal?.id ?? null, + }), + projectWorkspaceId, + executionWorkspaceId, + executionWorkspacePreference, + executionWorkspaceSettings, + companyId, + issueNumber, + identifier, + } as typeof issues.$inferInsert; + if (values.status === "in_progress" && !values.startedAt) { + values.startedAt = new Date(); + } + if (values.status === "done") { + values.completedAt = new Date(); + } + if (values.status === "cancelled") { + values.cancelledAt = new Date(); + } + Object.assign( + values, + buildInitialIssueMonitorFields({ + policy: normalizeIssueExecutionPolicy( + issueData.executionPolicy ?? null, + ), + status: values.status ?? "backlog", + assigneeAgentId: values.assigneeAgentId ?? null, + assigneeUserId: values.assigneeUserId ?? null, + }), + ); + + if (values.parentId || values.executionWorkspaceId || values.status === "done" || values.status === "cancelled") { + await notifyTerminalWorkspaceWork(tx, values.executionWorkspaceId ? `workspace:${values.executionWorkspaceId}` + : values.parentId ? `issue:${values.parentId}` : undefined); + } + if (values.monitorNextCheckAt) await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors); + await notifyRunDispatch(tx, { companyId: values.companyId }); + await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers); + await notifyStatusCardIssueChange(tx, companyId); + const [issue] = await tx.insert(issues).values(values).returning(); + await attachOwnedDraftImages(tx, issue, issue.description, { userId: issueData.createdByUserId, agentId: issueData.createdByAgentId }); + if (issue.visibility === "private" && !inheritedPrivateSubtree && issueData.createdByAgentId) { + await tx.insert(issueAccessGrants).values({ + issueId: issue.id, + subjectType: "agent", + subjectId: issueData.createdByAgentId, + source: "explicit", + grantedByAgentId: issueData.createdByAgentId, + }).onConflictDoNothing(); + } + await ensureAssignmentIssueAccessGrant(tx, issue, null, { + agentId: issueData.createdByAgentId ?? null, + userId: issueData.createdByUserId ?? null, + }); + await recordChatHandoff(tx, issue, actorRunId); + if (handoffSource) await recordAcceptedChatHandoff(tx, issue, handoffSource); + if (idempotencyKey) { + await tx.insert(issueCreateIdempotencyKeys).values({ + companyId, + idempotencyKey, + issueId: issue.id, + }); + } + if (watchdog) { + await upsertIssueWatchdogForIssue(tx, companyId, issue.id, { + agentId: watchdog.agentId, + instructions: watchdog.instructions, + actor: { + agentId: issueData.createdByAgentId ?? null, + userId: issueData.createdByUserId ?? null, + runId: watchdogActorRunId ?? null, + }, + }); + } + if (inputLabelIds) { + await syncIssueLabels(issue.id, companyId, inputLabelIds, tx); + } + if (blockedByIssueIds !== undefined) { + await syncBlockedByIssueIds( + issue.id, + companyId, + blockedByIssueIds, + { + agentId: issueData.createdByAgentId ?? null, + userId: issueData.createdByUserId ?? null, + }, + tx, + ); + } + if (initialPlan?.trim()) { + await documentService(tx as unknown as Db).upsertIssueDocument({ + issueId: issue.id, key: "plan", title: "Plan", format: "markdown", body: initialPlan, + createdByAgentId: issueData.createdByAgentId, createdByUserId: issueData.createdByUserId, + createdByRunId: actorRunId, + sourceTrust: issue.sourceTrust, + }); + } + const [enriched] = await withIssueLabels(tx, [issue]); + const [withRelations] = await withIssueRelationSummaries( + companyId, + [enriched], + tx, + ); + return withRelations; + }; + if (dbOrTx === db) return db.transaction(persist); + return persist(dbOrTx as DbTransaction); + }, + + /** + * Batched issue insert for company import. + * + * Company import used to call {@link create} once per issue — each call a + * separate network round-trip that inserted the issue, then serialized the + * issue's comments/documents behind the returned id. This inserts a whole + * bundle of pre-resolved issues (ids already generated by the caller) in + * chunked multi-row statements, so a thousand-issue import issues a handful + * of statements instead of thousands. + * + * The per-issue derivations {@link create} performs are reproduced for the + * import subset: a single contiguous identifier range is allocated from the + * company counter, per-project goal/workspace/policy defaults are computed + * once and cached, assignable-agent and workspace validation run per + * distinct id, and monitor notes land un-armed on the row. Dedup, + * idempotency, watchdogs, workspace inheritance and blocked-by wiring — none + * of which import uses — are intentionally omitted. + */ + importIssues: async ( + companyId: string, + rows: ImportIssueRow[], + ): Promise => { + if (rows.length === 0) return; + const isolatedWorkspacesEnabled = ( + await instanceSettings.getExperimental() + ).enableIsolatedWorkspaces; + await db.transaction(async (tx) => { + // Self-correcting counter: seed from max(issue_number) so a drifted + // company counter cannot mint colliding identifiers, then reserve the + // whole range in one bump instead of one-per-issue. + const [maxRow] = await tx + .select({ + maxNum: sql`coalesce(max(${issues.issueNumber}), 0)`, + }) + .from(issues) + .where(eq(issues.companyId, companyId)); + const currentMax = maxRow?.maxNum ?? 0; + const [company] = await tx + .select({ + issueCounter: companies.issueCounter, + issuePrefix: companies.issuePrefix, + }) + .from(companies) + .where(eq(companies.id, companyId)); + if (!company) throw notFound("Target company not found"); + const base = Math.max(company.issueCounter ?? 0, currentMax); + await tx + .update(companies) + .set({ issueCounter: base + rows.length }) + .where(eq(companies.id, companyId)); + + const defaultCompanyGoal = await getDefaultCompanyGoal(tx, companyId); + const defaultGoalId = defaultCompanyGoal?.id ?? null; + + // Project-scoped derivations depend only on the project, so resolve each + // distinct project once and reuse it across that project's issues. + const projectDerivedCache = new Map< + string, + { + goalId: string | null; + defaultProjectWorkspaceId: string | null; + defaultExecutionWorkspaceSettings: Record | null; + } + >(); + const loadProjectDerived = async (projectId: string) => { + const cached = projectDerivedCache.get(projectId); + if (cached) return cached; + const projectRow = await tx + .select({ + goalId: projects.goalId, + executionWorkspacePolicy: projects.executionWorkspacePolicy, + }) + .from(projects) + .where( + and( + eq(projects.id, projectId), + eq(projects.companyId, companyId), + ), + ) + .then((r) => r[0] ?? null); + const policy = parseProjectExecutionWorkspacePolicy( + projectRow?.executionWorkspacePolicy, + ); + const defaultProjectWorkspaceId = await resolveProjectDefaultWorkspaceId(tx, { + companyId, projectId, policy, + }); + const defaultExecutionWorkspaceSettings = + defaultIssueExecutionWorkspaceSettingsForProject( + gateProjectExecutionWorkspacePolicy( + policy, + isolatedWorkspacesEnabled, + ), + ) as Record | null; + const derived = { + goalId: projectRow?.goalId ?? null, + defaultProjectWorkspaceId, + defaultExecutionWorkspaceSettings, + }; + projectDerivedCache.set(projectId, derived); + return derived; + }; + + const validatedAgentIds = new Set(); + const validatedWorkspaceKeys = new Set(); + const issueRows: Array> = []; + const labelRows: Array<{ + issueId: string; + labelId: string; + companyId: string; + }> = []; + + let counter = base; + for (const row of rows) { + await assertExecutionTaskParent(tx as unknown as Db, companyId, row.parentId); + counter += 1; + const issueNumber = counter; + const identifier = `${company.issuePrefix}-${issueNumber}`; + + if (row.assigneeAgentId) { + if (!validatedAgentIds.has(row.assigneeAgentId)) { + await assertAssignableAgent( + tx as unknown as Db, + companyId, + row.assigneeAgentId, + { kind: "work" }, + ); + validatedAgentIds.add(row.assigneeAgentId); + } + } + if (row.status === "in_progress" && !row.assigneeAgentId) { + throw unprocessable("in_progress issues require an assignee"); + } + + const projectId = row.projectId ?? null; + let projectWorkspaceId = row.projectWorkspaceId ?? null; + // Isolated-workspace fields are gated the same way create() gates them: + // when the experiment is off the imported settings are dropped. + let executionWorkspaceSettings = isolatedWorkspacesEnabled + ? (row.executionWorkspaceSettings ?? null) + : null; + let projectGoalId: string | null = null; + if (projectId) { + const derived = await loadProjectDerived(projectId); + projectGoalId = derived.goalId; + if (!projectWorkspaceId) + projectWorkspaceId = derived.defaultProjectWorkspaceId; + if (executionWorkspaceSettings == null) { + executionWorkspaceSettings = + derived.defaultExecutionWorkspaceSettings; + } + } + if (projectWorkspaceId) { + const workspaceKey = `${projectId ?? ""}:${projectWorkspaceId}`; + if (!validatedWorkspaceKeys.has(workspaceKey)) { + await assertValidProjectWorkspace( + companyId, + projectId, + projectWorkspaceId, + tx, + ); + validatedWorkspaceKeys.add(workspaceKey); + } + } + + const goalId = resolveIssueGoalId({ + projectId, + goalId: null, + projectGoalId, + defaultGoalId, + }); + + issueRows.push({ + id: row.id, + companyId, + issueNumber, + identifier, + title: row.title, + description: row.description ?? null, + assigneeAgentId: row.assigneeAgentId ?? null, + status: row.status, + priority: row.priority, + billingCode: row.billingCode ?? null, + assigneeAdapterOverrides: row.assigneeAdapterOverrides ?? null, + projectId, + projectWorkspaceId, + executionWorkspaceSettings, + goalId, + responsibleUserId: null, + requestDepth: clampIssueRequestDepth(undefined), + originKind: "manual", + // The caller resolves parentId against ids in this same batch, so a + // parent always lands in the same insert (rows arrive parents-first). + parentId: row.parentId ?? null, + // Preserved bundle timestamps win; without them createdAt/updatedAt + // fall back to the insert time (the old defaultNow() behavior). + createdAt: row.createdAt ?? new Date(), + updatedAt: row.updatedAt ?? new Date(), + // Imported in-progress work did not start at import time; fabricating + // startedAt here would misrepresent its active episode. + // Only a bundle-carried startedAt is written. + startedAt: row.startedAt ?? null, + completedAt: + row.completedAt ?? (row.status === "done" ? new Date() : null), + cancelledAt: + row.cancelledAt ?? + (row.status === "cancelled" ? new Date() : null), + monitorNotes: row.monitorNotes ?? null, + monitorScheduledBy: row.monitorScheduledBy ?? null, + }); + for (const labelId of new Set(row.labelIds ?? [])) { + labelRows.push({ issueId: row.id, labelId, companyId }); + } + } + + await notifyStatusCardIssueChange(tx, companyId); + await insertRowsInChunks(tx, issues, issueRows); + await insertRowsInChunks(tx, issueLabels, labelRows); + }); + }, + + /** + * Batched comment insert for company import. Comment ids are pre-generated + * by the caller so attachments can reference them without a round-trip. + */ + addImportedComments: async ( + rows: ImportIssueCommentRow[], + ): Promise => { + if (rows.length === 0) return; + const censorUsernameInLogs = (await instanceSettings.getGeneral()) + .censorUsernameInLogs; + await db.transaction(async (tx) => { + for (const companyId of new Set(rows.map(row => row.companyId))) await notifyStatusCardIssueChange(tx, companyId); + const commentRows = rows.map((row) => { + const createdAt = row.createdAt ? new Date(row.createdAt) : null; + return { + id: row.id, + companyId: row.companyId, + issueId: row.issueId, + authorAgentId: row.authorAgentId ?? null, + authorUserId: row.authorUserId ?? null, + authorType: row.authorType, + createdByRunId: null, + body: redactCurrentUserText(row.body, { + enabled: censorUsernameInLogs, + }), + presentation: row.presentation ?? null, + metadata: row.metadata ?? null, + sourceTrust: null, + createdAt: + createdAt && !Number.isNaN(createdAt.getTime()) + ? createdAt + : new Date(), + }; + }); + await insertRowsInChunks(tx, issueComments, commentRows); + // Mirror addComment's recency bump, once per affected issue — but never + // backwards. An issue imported with a preserved updatedAt keeps it + // unless a newer imported comment outdates it; issues without preserved + // timestamps carry an insert-time updatedAt, so GREATEST reproduces the + // old "bump to now" behavior for them. + const bumpAtByIssueId = new Map(); + for (const row of commentRows) { + const existing = bumpAtByIssueId.get(row.issueId); + if (!existing || row.createdAt > existing) + bumpAtByIssueId.set(row.issueId, row.createdAt); + } + const bumpEntries = [...bumpAtByIssueId.entries()]; + for ( + let start = 0; + start < bumpEntries.length; + start += DEFAULT_INSERT_CHUNK_ROWS + ) { + const chunk = bumpEntries.slice( + start, + start + DEFAULT_INSERT_CHUNK_ROWS, + ); + await tx.execute(sql` + update ${issues} + set updated_at = greatest(${issues.updatedAt}, bumps.bump_at) + from (values ${sql.join( + chunk.map( + ([issueId, bumpAt]) => + sql`(${issueId}::uuid, ${bumpAt.toISOString()}::timestamptz)`, + ), + sql`, `, + )}) as bumps(issue_id, bump_at) + where ${issues.id} = bumps.issue_id + `); + } + }); + }, + + /** + * Batched attachment insert for company import: each row mints an asset and + * links it to its issue (and optionally comment) in two chunked statements. + */ + addImportedAttachments: async ( + rows: ImportIssueAttachmentRow[], + ): Promise => { + if (rows.length === 0) return; + await db.transaction(async (tx) => { + const assetRows: Array> = []; + const attachmentRows: Array> = []; + for (const row of rows) { + const assetId = randomUUID(); + assetRows.push({ + id: assetId, + companyId: row.companyId, + provider: row.provider, + objectKey: row.objectKey, + contentType: row.contentType, + byteSize: row.byteSize, + sha256: row.sha256, + originalFilename: row.originalFilename ?? null, + createdByAgentId: row.createdByAgentId ?? null, + createdByUserId: row.createdByUserId ?? null, + }); + attachmentRows.push({ + companyId: row.companyId, + issueId: row.issueId, + assetId, + issueCommentId: row.issueCommentId ?? null, + }); + } + await insertRowsInChunks(tx, assets, assetRows); + await insertRowsInChunks(tx, issueAttachments, attachmentRows); + }); + }, + + update: async ( + id: string, + data: Partial & { + labelIds?: string[]; + blockedByIssueIds?: string[]; + actorAgentId?: string | null; + actorRunId?: string | null; + /** Server-owned transfer being applied; never accepted from HTTP input. */ + caseHandoffId?: string; + actorRunStopId?: string | null; + actorUserId?: string | null; + companyGuard?: string; + workspaceSelectionActor?: AuthorizationActor; + expectedExecutionPolicy?: typeof issues.$inferInsert.executionPolicy; + expectedStatusVersion?: number; + }, + dbOrTx: any = db, + postCommitActivityPublications?: ActivityPublication[], + postCommitActions?: IssuePostCommitAction[], + options: { bindRuntimeSharedWorkspace?: boolean } = {}, + ) => { + const ownedActivityPublications: ActivityPublication[] = []; + const activityPublications = + postCommitActivityPublications ?? ownedActivityPublications; + const ownedPostCommitActions: IssuePostCommitAction[] = []; + const queuedPostCommitActions = + postCommitActions ?? ownedPostCommitActions; + // A caller that supplies `companyGuard` gets the company added to + // every read, lock, and write predicate below. A check before this + // call is not a boundary: `issues.company_id` can change between + // that check and this write, so the predicate must carry the + // company itself. + const idPredicate = + data.companyGuard !== undefined + ? and(eq(issues.id, id), eq(issues.companyId, data.companyGuard)) + : eq(issues.id, id); + const existing = await dbOrTx + .select() + .from(issues) + .where(idPredicate) + .then((rows: Array) => rows[0] ?? null); + if (!existing) return null; + if (data.parentId !== undefined && data.parentId !== existing.parentId) { + await assertExecutionTaskParent(dbOrTx, existing.companyId, data.parentId); + } + if (existing.conversationAgentId) { + if ((data.assigneeAgentId !== undefined && data.assigneeAgentId !== existing.conversationAgentId) + || data.assigneeUserId || data.conversationAgentId !== undefined || data.conversationUserId !== undefined + || data.conversationState !== undefined || data.conversationSessionGeneration !== undefined + || data.conversationBoundaryCommentId !== undefined || data.status === "done" || data.status === "cancelled") { + throw unprocessable("Conversation identity is fixed; finish the reply instead of completing or reassigning the conversation"); + } + } + + const { + labelIds: nextLabelIds, + blockedByIssueIds, + actorAgentId, + actorRunId, + caseHandoffId, + actorRunStopId, + actorUserId, + companyGuard, + workspaceSelectionActor, + expectedExecutionPolicy, + expectedStatusVersion, + ...issueData + } = data; + if (expectedStatusVersion !== undefined && ( + issueData.status !== "todo" || Object.keys(issueData).some(key => key !== "status") || + nextLabelIds !== undefined || blockedByIssueIds !== undefined || expectedExecutionPolicy !== undefined + )) { + throw unprocessable("expectedStatusVersion requires a status-only backlog-to-todo promotion"); + } + // An explicit edit claims the title, even if it keeps the same text. + if (issueData.title !== undefined) issueData.titleNeedsGeneration = false; + if ( + issueData.assigneeAgentId !== undefined && + issueData.assigneeAgentId !== existing.assigneeAgentId + ) { + const externalBinding = await dbOrTx + .select({ id: chatConversations.id }) + .from(chatConversations) + .where( + and( + eq(chatConversations.companyId, existing.companyId), + eq(chatConversations.issueId, existing.id), + ), + ) + .limit(1) + .then((rows: Array<{ id: string }>) => rows[0] ?? null); + if (externalBinding) { + throw conflict( + "Agent assignment cannot change while this task is bound to an external channel", + { + code: "chat_binding_agent_locked", + conversationId: externalBinding.id, + }, + ); + } + } + const isolatedWorkspacesEnabled = ( + await instanceSettings.getExperimental() + ).enableIsolatedWorkspaces; + if (options.bindRuntimeSharedWorkspace) { + const workspaceId = issueData.executionWorkspaceId ?? existing.executionWorkspaceId; + if (!workspaceId) { + throw unprocessable("Runtime workspace binding requires an existing shared workspace"); + } + const [workspace] = await dbOrTx + .select({ mode: executionWorkspaces.mode }) + .from(executionWorkspaces) + .where(and( + eq(executionWorkspaces.id, workspaceId), + eq(executionWorkspaces.companyId, existing.companyId), + )); + if ( + workspace?.mode !== "shared_workspace" || + (issueData.executionWorkspacePreference ?? existing.executionWorkspacePreference) !== "reuse_existing" || + (issueData.executionWorkspaceSettings ?? existing.executionWorkspaceSettings)?.mode !== "shared_workspace" + ) { + throw unprocessable("Runtime workspace binding requires an existing shared workspace"); + } + } + // Warm sandbox continuity is runtime bookkeeping, independent of the + // opt-in UI for creating isolated worktrees. Public updates still obey + // the feature gate; only the internal shared-workspace binding bypasses it. + if (!isolatedWorkspacesEnabled && !options.bindRuntimeSharedWorkspace) { + delete issueData.executionWorkspaceId; + delete issueData.executionWorkspacePreference; + delete issueData.executionWorkspaceSettings; + } + + if (issueData.status) { + assertTransition(existing.status, issueData.status); + } + + const patch: Partial = { + ...issueData, + updatedAt: new Date(), + }; + if (issueData.visibility === "private") { + patch.privacyRootIssueId = existing.privacyRootIssueId ?? existing.id; + } else if (issueData.visibility === "open") { + patch.privacyRootIssueId = null; + } + if (issueData.projectId) { + const selectedProject = await dbOrTx + .select({ visibility: projects.visibility }) + .from(projects) + .where(and(eq(projects.id, issueData.projectId), eq(projects.companyId, existing.companyId))) + .then((rows: Array<{ visibility: string }>) => rows[0] ?? null); + if (!selectedProject) throw notFound("Project not found"); + if (selectedProject.visibility === "private") { + patch.visibility = "private"; + patch.privacyRootIssueId = existing.privacyRootIssueId ?? existing.id; + } + } + if (existing.status !== "blocked" && issueData.status === "blocked") { + patch.blockedTransitionAt = patch.updatedAt; + patch.blockedOwnerNotifiedAt = null; + } else if ( + existing.status === "blocked" && + issueData.status && + issueData.status !== "blocked" + ) { + patch.unblockDescriptor = null; + patch.blockedTransitionAt = null; + patch.blockedOwnerNotifiedAt = null; + } + if (issueData.requestDepth !== undefined) { + patch.requestDepth = clampIssueRequestDepth(issueData.requestDepth); + } + + const nextAssigneeAgentId = + issueData.assigneeAgentId !== undefined + ? issueData.assigneeAgentId + : existing.assigneeAgentId; + const nextAssigneeUserId = + issueData.assigneeUserId !== undefined + ? issueData.assigneeUserId + : existing.assigneeUserId; + + if (nextAssigneeAgentId && nextAssigneeUserId) { + throw unprocessable("Issue can only have one assignee"); + } + if ( + patch.status === "in_progress" && + !nextAssigneeAgentId && + !nextAssigneeUserId + ) { + throw unprocessable("in_progress issues require an assignee"); + } + if (patch.status === "in_progress") { + const dependencyReadiness = + blockedByIssueIds === undefined + ? ( + await listIssueDependencyReadinessMap( + dbOrTx, + existing.companyId, + [id], + ) + ).get(id) + : null; + const unresolvedBlockerIssueIds = + blockedByIssueIds !== undefined + ? await listUnresolvedBlockerIssueIds( + dbOrTx, + existing.companyId, + blockedByIssueIds, + ) + : (dependencyReadiness?.unresolvedBlockerIssueIds ?? []); + if (unresolvedBlockerIssueIds.length > 0) { + const unresolvedBlockers = await listUnresolvedBlockerDetails( + dbOrTx, + existing.companyId, + unresolvedBlockerIssueIds, + dependencyReadiness?.pendingFinalizeBlockerIssueIds, + ); + throw unprocessable("Issue is blocked by unresolved blockers", { + unresolvedBlockerIssueIds, + unresolvedBlockers, + }); + } + } + const shouldValidateNextAssignee = + Boolean(nextAssigneeAgentId) && + (issueData.assigneeAgentId !== undefined || + patch.status === "in_progress"); + if (shouldValidateNextAssignee) { + await assertAssignableAgent( + dbOrTx as Db, + existing.companyId, + nextAssigneeAgentId, + { kind: "work" }, + ); + } + if (issueData.assigneeUserId) { + await assertAssignableUser( + existing.companyId, + issueData.assigneeUserId, + ); + } + let nextProjectId = + issueData.projectId !== undefined + ? issueData.projectId + : existing.projectId; + const nextProjectWorkspaceId = + issueData.projectWorkspaceId !== undefined + ? issueData.projectWorkspaceId + : existing.projectWorkspaceId; + const nextExecutionWorkspaceId = + issueData.executionWorkspaceId !== undefined + ? issueData.executionWorkspaceId + : existing.executionWorkspaceId; + const nextExecutionWorkspacePreference = + issueData.executionWorkspacePreference !== undefined + ? issueData.executionWorkspacePreference + : existing.executionWorkspacePreference; + const nextExecutionWorkspaceSettings = + issueData.executionWorkspaceSettings !== undefined + ? parseIssueExecutionWorkspaceSettings( + issueData.executionWorkspaceSettings, + ) + : parseIssueExecutionWorkspaceSettings( + existing.executionWorkspaceSettings, + ); + if (issueData.executionWorkspaceSettings !== undefined) { + patch.executionWorkspaceSettings = nextExecutionWorkspaceSettings + ? { ...nextExecutionWorkspaceSettings } + : null; + } + if (nextProjectWorkspaceId) await assertValidProjectWorkspace(existing.companyId, null, nextProjectWorkspaceId); + if (nextExecutionWorkspaceId) await assertValidExecutionWorkspace(existing.companyId, null, nextExecutionWorkspaceId); + if ( + isolatedWorkspacesEnabled && + issueData.executionWorkspaceSettings !== undefined + ) { + assertExplicitPinnedWorktreeIssueRunnable({ + projectId: nextProjectId ?? null, + projectWorkspaceId: nextProjectWorkspaceId ?? null, + executionWorkspaceId: nextExecutionWorkspaceId ?? null, + executionWorkspacePreference: + nextExecutionWorkspacePreference ?? null, + executionWorkspaceSettings: issueData.executionWorkspaceSettings, + }); + } + + applyStatusSideEffects(issueData.status, patch); + if (issueData.status && issueData.status !== "done") { + patch.completedAt = null; + } + if (issueData.status && issueData.status !== "cancelled") { + patch.cancelledAt = null; + } + if (issueData.status && issueData.status !== "in_progress") { + patch.checkoutRunId = null; + patch.executionRunId = null; + patch.executionAgentNameKey = null; + patch.executionLockedAt = null; + } + if ( + (issueData.assigneeAgentId !== undefined && + issueData.assigneeAgentId !== existing.assigneeAgentId) || + (issueData.assigneeUserId !== undefined && + issueData.assigneeUserId !== existing.assigneeUserId) + ) { + patch.checkoutRunId = null; + patch.executionRunId = null; + patch.executionAgentNameKey = null; + patch.executionLockedAt = null; + } + + const runUpdate = async (tx: any) => { + const changesPrivacy = issueData.visibility !== undefined || issueData.parentId !== undefined || issueData.projectId !== undefined; + if (changesPrivacy) await tx.execute(sql`select pg_advisory_xact_lock(hashtextextended(${`issue-privacy-tree:${existing.companyId}`}, 0))`); + // The receipt baseline must be read under the same row lock as the + // write. Otherwise a concurrent update can be mistaken for a change + // made by this request. + const receiptExisting = await tx + .select() + .from(issues) + .where(idPredicate) + .for("update") + .then((rows: Array) => rows[0] ?? null); + if (!receiptExisting) return null; + if (issueData.executionPolicy !== undefined) assertQualityReviewPolicyMutation({ previousPolicy: receiptExisting.executionPolicy, nextPolicy: issueData.executionPolicy, actorAgentId, actorUserId }); + if ((patch.status ?? receiptExisting.status) === "done" && + (receiptExisting.status !== "done" || issueData.executionPolicy !== undefined)) { + await assertRequiredQualityReviewComplete(tx as Db, { ...receiptExisting, executionPolicy: issueData.executionPolicy === undefined ? receiptExisting.executionPolicy : issueData.executionPolicy }, true); + } + if (actorRunId) await assertNoCaseHandoffMutation(tx, actorRunId, id); + if (expectedStatusVersion !== undefined && ( + receiptExisting.statusVersion !== expectedStatusVersion || receiptExisting.status !== "backlog" + )) { + throw conflict("The task status or assignment changed. Read the task again before starting work.", { + code: "issue_status_version_changed", + }); + } + if (expectedStatusVersion !== undefined) { + const policy = normalizeIssueExecutionPolicy(receiptExisting.executionPolicy ?? null); + const transition = applyIssueExecutionPolicyTransition({ + issue: receiptExisting, + policy, + previousPolicy: policy, + requestedStatus: "todo", + requestedAssigneePatch: {}, + actor: { agentId: actorAgentId ?? null, userId: actorUserId ?? null }, + allowBoardOverride: workspaceSelectionActor?.type === "board" && workspaceSelectionActor.source !== "mcp_oauth", + }); + if (transition.decision || + (transition.patch.status !== undefined && transition.patch.status !== "todo") || + (transition.patch.assigneeAgentId !== undefined && transition.patch.assigneeAgentId !== receiptExisting.assigneeAgentId) || + (transition.patch.assigneeUserId !== undefined && transition.patch.assigneeUserId !== receiptExisting.assigneeUserId)) { + throw unprocessable("expectedStatusVersion requires a status-only backlog-to-todo promotion"); + } + Object.assign(patch, transition.patch); + } + if (issueData.executionWorkspaceId !== undefined || issueData.projectWorkspaceId !== undefined || + issueData.executionWorkspacePreference !== undefined || issueData.executionWorkspaceSettings !== undefined) { + Object.assign(patch, await executionWorkspaceService(db).prepareTaskWorkspaceUpdate({ + companyId: receiptExisting.companyId, issueId: receiptExisting.id, patch, + }, tx)); + } + if (expectedExecutionPolicy !== undefined && !isDeepStrictEqual( + receiptExisting.executionPolicy ?? null, + expectedExecutionPolicy, + )) { + throw conflict("The task execution settings changed. Try cancelling the monitor again.", { + code: "execution_policy_changed", + }); + } + if (changesPrivacy) { + // Privacy inherited through task ancestry can authorize saved notices + // on descendants as well as this task. + await notifyChatInboundWakeNotices(tx, { companyId: receiptExisting.companyId }); + const nextProjectId = issueData.projectId !== undefined ? issueData.projectId : receiptExisting.projectId; + const [privacyProject] = nextProjectId ? await tx.select().from(projects) + .where(and(eq(projects.id, nextProjectId), eq(projects.companyId, existing.companyId))) : []; + if (privacyProject?.visibility === "private") { + if (issueData.visibility === "open") { + if (!privacyProject.personalOwnerUserId) throw unprocessable("Move this task out of its private project before making it public"); + patch.projectId = null; + } else { + patch.visibility = "private"; + patch.privacyRootIssueId = receiptExisting.privacyRootIssueId ?? receiptExisting.id; + } + } + const privacyParentId = issueData.parentId !== undefined ? issueData.parentId : receiptExisting.privacyParentIssueId; + const [privacyParent] = privacyParentId ? await tx.select().from(issues) + .where(and(eq(issues.id, privacyParentId), eq(issues.companyId, existing.companyId))) : []; + patch.privacyParentIssueId = privacyParent?.id ?? null; + if (privacyParent && (privacyParent.visibility === "private" || (privacyParent.projectId && await tx.select({ id: projects.id }).from(projects) + .where(and(eq(projects.id, privacyParent.projectId), eq(projects.visibility, "private"))).limit(1).then((rows: Array<{ id: string }>) => rows.length > 0)))) { + if (issueData.visibility === "open") throw unprocessable("A task cannot be made public while it inherits private access"); + patch.visibility = "private"; + patch.privacyRootIssueId = privacyParent.privacyRootIssueId ?? privacyParent.id; + } else if (issueData.visibility === "private") { + patch.privacyRootIssueId = receiptExisting.id; + } + } + if (actorAgentId && actorRunId) { + // Recheck under a run lock: a request admitted before Stop must not + // commit a late Done after cancellation revoked its credentials. + const actorRun = await assertAgentRunWriteAllowed(tx, receiptExisting.companyId, { + agentId: actorAgentId, runId: actorRunId, stopId: actorRunStopId, + }); + assertLegacyRunTaskOwnership(actorRun, receiptExisting); + } + if (blockedByIssueIds !== undefined) { + await assertOnboardingFirstTaskDependenciesAllowed(tx as Db, { + companyId: receiptExisting.companyId, + actorAgentId, + actorRunId, + blockedByIssueIds, + parentIssueId: receiptExisting.parentId, + }); + } + if (actorAgentId && patch.status === "done") { + const [review] = await tx.select({ id: toolActionRequests.id }).from(toolActionRequests).where(and(eq(toolActionRequests.companyId, existing.companyId), eq(toolActionRequests.issueId, id), inArray(toolActionRequests.status, ["pending", "approved", "executing"]))).limit(1); + if (review) throw conflict("This task is waiting for a connection review. Finish unrelated work, then yield in_review without retrying the governed call.", { code: "tool_review_pending", actionRequestId: review.id }); + } + + const [previousLabelsByIssueId, previousRelationSummaries] = await Promise.all([ + nextLabelIds !== undefined + ? labelMapForIssues(tx, [id]) + : Promise.resolve(new Map()), + blockedByIssueIds !== undefined + ? getIssueRelationSummaryMap(existing.companyId, [id], tx) + : Promise.resolve(new Map()), + ]); + const defaultCompanyGoal = await getDefaultCompanyGoal(tx, existing.companyId); + const [currentProjectGoalId, nextProjectGoalId] = await Promise.all([ + getProjectDefaultGoalId(tx, existing.companyId, existing.projectId), + getProjectDefaultGoalId( + tx, + existing.companyId, + issueData.projectId !== undefined + ? issueData.projectId + : existing.projectId, + ), + ]); + + patch.goalId = resolveNextIssueGoalId({ + currentProjectId: existing.projectId, + currentGoalId: existing.goalId, + currentProjectGoalId, + projectId: issueData.projectId, + goalId: issueData.goalId, + projectGoalId: nextProjectGoalId, + defaultGoalId: defaultCompanyGoal?.id ?? null, + }); + // Ownership changes invalidate observed handoff versions even if status + // stays the same, including an A -> B -> A assignment race. + if (issueData.status === "done" || issueData.status === "cancelled") { + await cancelCaseHandoffsForIssue(tx, { companyId: receiptExisting.companyId, issueId: id, + reason: `Task was marked ${issueData.status}`, agentTransfersOnly: issueData.status === "done", publications: activityPublications }); + } + if ((issueData.assigneeAgentId !== undefined && issueData.assigneeAgentId !== receiptExisting.assigneeAgentId) + || (issueData.assigneeUserId !== undefined && issueData.assigneeUserId !== receiptExisting.assigneeUserId)) { + await cancelCaseHandoffsForIssue(tx, { companyId: receiptExisting.companyId, issueId: id, + reason: "Task assignment changed", exceptId: caseHandoffId, publications: activityPublications }); + patch.statusVersion = sql`${issues.statusVersion} + 1` as unknown as number; + // Invalidate human direction at the common assignment boundary, including + // plugin/service writes that do not go through HTTP run cancellation. + // Keep the requester attribution for audit; cancellation revokes its use. + await notifyChatInboundWakeNotices(tx, { companyId: receiptExisting.companyId }); + await tx.update(agentWakeupRequests).set({ status: "cancelled", finishedAt: new Date(), updatedAt: new Date() }) + .where(and(eq(agentWakeupRequests.companyId, receiptExisting.companyId), + eq(agentWakeupRequests.requestedByActorType, "user"), + sql`coalesce(${agentWakeupRequests.payload}->>'issueId', ${agentWakeupRequests.payload}->>'taskId', + ${agentWakeupRequests.payload}->'_paperclipWakeContext'->>'issueId', + ${agentWakeupRequests.payload}->'_paperclipWakeContext'->>'taskId') = ${id}`)); + } + // Reasserting Blocked or changing its blockers is a fresh decision even + // when the status string stays the same. Invalidate recovery's prior + // status receipt without treating comment recency as blocking intent. + if (receiptExisting.status === "blocked" && + (issueData.status === "blocked" || + (issueData.status === undefined && + (blockedByIssueIds !== undefined || issueData.unblockDescriptor !== undefined)))) { + patch.statusVersion = sql`${issues.statusVersion} + 1` as unknown as number; + } + await notifyStatusCardIssueChange(tx, receiptExisting.companyId); + if ((patch.status !== undefined && patch.status !== receiptExisting.status) + || patch.completedAt !== undefined || patch.cancelledAt !== undefined + || issueData.parentId !== undefined || issueData.executionWorkspaceId !== undefined) { + await notifyTerminalWorkspaceWork(tx, `issue:${id}`); + } + if (patch.status !== undefined || patch.assigneeAgentId !== undefined || patch.assigneeUserId !== undefined + || patch.hiddenAt !== undefined || patch.conversationAgentId !== undefined) { + await notifyRunDispatch(tx, { companyId: receiptExisting.companyId }); + await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers); + } + if (patch.monitorNextCheckAt !== undefined || (receiptExisting.monitorNextCheckAt && + (patch.status !== undefined || patch.assigneeAgentId !== undefined || patch.assigneeUserId !== undefined))) { + await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors); + } + if (patch.status === "done" || patch.status === "cancelled") { + await notifyDecisionTargetWork(tx, `issue:${id}`); + } + if ((patch.status !== undefined && patch.status !== receiptExisting.status) || + (patch.assigneeAgentId !== undefined && patch.assigneeAgentId !== receiptExisting.assigneeAgentId) || + (patch.executionState !== undefined && parseObject(receiptExisting.executionState).status === "pending" && + parseObject(patch.executionState).status !== "pending")) { + await notifyChatResponseAvailability(tx, { companyId: receiptExisting.companyId, issueId: id }); + } + if (patch.status !== undefined && patch.status !== receiptExisting.status) { + await notifyChatInboundWakeNotices(tx, { companyId: receiptExisting.companyId, issueId: id }); + } + if (patch.hiddenAt === null && receiptExisting.hiddenAt !== null) { + await notifyChatRunMilestones(tx, { companyId: receiptExisting.companyId, issueId: id }); + } + if (patch.hiddenAt !== undefined || patch.harnessKind !== undefined || patch.conversationAgentId !== undefined) { + await notifyApprovalWork(tx, receiptExisting.companyId); + } + if ((patch.status !== undefined && patch.status !== receiptExisting.status) + || (patch.assigneeAgentId !== undefined && patch.assigneeAgentId !== receiptExisting.assigneeAgentId) + || (patch.executionRunId !== undefined && patch.executionRunId !== receiptExisting.executionRunId) + || (patch.checkoutRunId !== undefined && patch.checkoutRunId !== receiptExisting.checkoutRunId)) { + await notifyUnsafeWorkspaceExportRecovery(tx, { companyId: receiptExisting.companyId, issueId: id }); + } + if (patch.status !== undefined || patch.executionState !== undefined || patch.executionRunId !== undefined + || patch.monitorNextCheckAt !== undefined || patch.hiddenAt !== undefined || patch.originKind !== undefined + || patch.originId !== undefined || patch.conversationAgentId !== undefined || patch.assigneeAgentId !== undefined + || patch.assigneeUserId !== undefined || blockedByIssueIds !== undefined) { + + const affectsDependents = patch.status !== undefined && patch.status !== receiptExisting.status + && (patch.status === "done" || receiptExisting.status === "done"); + await notifyExternalConversationSettlement(tx, { companyId: receiptExisting.companyId, + issueId: affectsDependents ? undefined : id }); + } + if (patch.status !== undefined || patch.statusVersion !== undefined || patch.executionState !== undefined + || patch.executionRunId !== undefined || patch.checkoutRunId !== undefined + || patch.assigneeAgentId !== undefined || patch.assigneeUserId !== undefined || blockedByIssueIds !== undefined) { + await notifyNativeRecoveryNotices(tx, { companyId: receiptExisting.companyId, issueId: id }); + await notifyNativeRunProjection(tx, { companyId: receiptExisting.companyId, issueId: id }); + await notifyNativeResultCommit(tx, { companyId: receiptExisting.companyId, issueId: id }); + await notifyNativeWorkspaceFinalization(tx, { companyId: receiptExisting.companyId, issueId: id }); + await notifyNativeStatusReassessment(tx, { companyId: receiptExisting.companyId, issueId: id }); + } + const updated = await tx + .update(issues) + .set(patch) + .where(idPredicate) + .returning() + .then((rows: Array) => rows[0] ?? null); + if (!updated) return null; + if (updated.status === "done") { + await completeTerminalCaseHandoffsForIssue(tx, { companyId: updated.companyId, issueId: updated.id, + publications: activityPublications }); + } + if (workspaceSelectionActor && updated.workspaceBindingRevision !== receiptExisting.workspaceBindingRevision) { + await executionWorkspaceService(db).assertTaskWorkspaceUpdateAccess({ + task: updated, actor: workspaceSelectionActor, + }, tx); + } + if (issueData.description !== undefined) await attachOwnedDraftImages(tx, updated, updated.description, + { userId: actorUserId, agentId: actorAgentId }); + if (changesPrivacy && updated.visibility === "private") { + // Creation and tree changes hold the same company lock. No child can + // slip into the tree between discovering descendants and protecting them. + const protectedRows = await tx.execute(sql`with recursive descendants as ( + select i.id, i.parent_id, i.privacy_parent_issue_id from issues i + where i.id = ${updated.id} and i.company_id = ${updated.companyId} + union + select i.id, i.parent_id, i.privacy_parent_issue_id from issues i join descendants p + on i.parent_id = p.id or i.privacy_parent_issue_id = p.id + where i.company_id = ${updated.companyId} + ) update issues i set visibility = 'private', + privacy_root_issue_id = ${updated.privacyRootIssueId ?? updated.id}, + privacy_parent_issue_id = coalesce(i.privacy_parent_issue_id, i.parent_id), updated_at = now() + from descendants d where i.id = d.id and i.id <> ${updated.id} returning i.id`); + if (protectedRows.length) { + const descendants = await tx.select().from(issues).where(inArray(issues.id, protectedRows.map((row: { id: string }) => row.id))); + for (const descendant of descendants) await ensureAssignmentIssueAccessGrant(tx, descendant, null, { + agentId: actorAgentId ?? null, userId: actorUserId ?? null, + }); + } + } + await ensureAssignmentIssueAccessGrant(tx, updated, receiptExisting, { + agentId: actorAgentId ?? null, + userId: actorUserId ?? null, + }); + await recordChatCompletion(tx, receiptExisting, updated); + // An operator explicitly choosing a disposition owns that decision, + // including choosing In Review while the conversation is Idle. + if (actorUserId && issueData.status !== undefined) { + await tx.update(chatConversations).set({ state: "active", updatedAt: new Date() }) + .where(and(eq(chatConversations.companyId, updated.companyId), eq(chatConversations.issueId, updated.id), + eq(chatConversations.state, "waiting"), sql`exists (select 1 from chat_endpoints e + where e.id = ${chatConversations.endpointId} and e.company_id = ${chatConversations.companyId} and e.provider = 'slack')`)); + } + if (updated.assigneeAgentId !== existing.assigneeAgentId || updated.assigneeUserId !== existing.assigneeUserId) { + const { issueThreadInteractionService } = await import("./issue-thread-interactions.js"); + await issueThreadInteractionService(tx).expireConnectionIntentsForOwnershipChange(updated); + } + if (existing.status !== updated.status) { + if ( + (existing.status === "done" || existing.status === "cancelled") && + updated.status !== "done" && + updated.status !== "cancelled" + ) { + const terminalWorkspaces = await tx + .select({ id: executionWorkspaces.id }) + .from(executionWorkspaces) + .where( + and( + eq(executionWorkspaces.companyId, updated.companyId), + eq(executionWorkspaces.sourceIssueId, updated.id), + eq(executionWorkspaces.status, "archived"), + like(executionWorkspaces.cleanupReason, "issue_terminal%"), + ), + ); + for (const workspace of terminalWorkspaces) { + await logActivity(tx as unknown as Db, { + companyId: updated.companyId, + actorType: actorAgentId + ? "agent" + : actorUserId + ? "user" + : "system", + actorId: actorAgentId ?? actorUserId ?? "issue_service", + agentId: actorAgentId ?? null, + action: "execution_workspace.source_issue_reopened", + entityType: "execution_workspace", + entityId: workspace.id, + details: { + sourceIssueId: updated.id, + previousIssueStatus: existing.status, + nextIssueStatus: updated.status, + workspaceAction: "left_archived", + }, + }); + } + } + if (updated.status === "done" || updated.status === "cancelled") { + await finalizeSummarySlotsForTerminalIssue(tx, updated); + // Every terminal transition funnels through here, including direct + // service callers (tree control, recovery, status cards) + // that never touch the HTTP routes. Governed cards expire; ordinary + // historical questions remain answerable after completion. + // Dynamic import breaks the module + // cycle (issue-thread-interactions.js imports issueService). + const { issueThreadInteractionService } = + await import("./issue-thread-interactions.js"); + // Stop live question requests independently of card expiry. A + // historical question retained in the feed must not keep its + // source run waiting after the task closes. + const pendingQuestions = await tx + .select() + .from(issueThreadInteractions) + .where(and( + eq(issueThreadInteractions.companyId, updated.companyId), + eq(issueThreadInteractions.issueId, updated.id), + eq(issueThreadInteractions.kind, "ask_user_questions"), + eq(issueThreadInteractions.status, "pending"), + )); + const expiredInteractions = await issueThreadInteractionService( + tx, + ).expirePendingInteractionsForTerminalIssue(updated, { + agentId: actorAgentId ?? null, + userId: actorUserId ?? null, + }); + const { + nativeQuestionCancellationIdentity, + requestNativeQuestionRunCancellation, + } = await import("./native-runtime/native-question-bridge.js"); + for (const interaction of pendingQuestions) { + const nativeQuestion = + nativeQuestionCancellationIdentity(interaction); + if (nativeQuestion) { + if (dbOrTx !== db && !postCommitActions) { + throw new Error( + "Terminal native question updates in an external transaction require a post-commit action queue", + ); + } + const runId = await requestNativeQuestionRunCancellation( + tx, + nativeQuestion, + { kind: "issue_terminal", issueStatus: updated.status }, + ); + if (runId) { + queuedPostCommitActions.push({ + type: "cancel_native_question_run", + runId, + issueId: updated.id, + issueStatus: updated.status, + }); + } + } + } + for (const interaction of expiredInteractions) { + await logActivity(tx as unknown as Db, { + companyId: updated.companyId, + actorType: actorAgentId + ? "agent" + : actorUserId + ? "user" + : "system", + actorId: actorAgentId ?? actorUserId ?? "issue_service", + agentId: actorAgentId ?? null, + action: "issue.thread_interaction_expired", + entityType: "issue", + entityId: updated.id, + details: { + identifier: updated.identifier ?? null, + interactionId: interaction.id, + interactionKind: interaction.kind, + interactionStatus: interaction.status, + source: "issue.status_transition.issue_closed", + result: interaction.result ?? null, + }, + }); + } + } + // A status-card generation task that goes done/cancelled/blocked stops + // making progress; release the card's generation claim so the board tile + // stops spinning and offers "Run now" again (blocked = stuck on a human). + if ( + updated.status === "done" || + updated.status === "cancelled" || + updated.status === "blocked" + ) { + await finalizeStatusCardsForStalledGeneration(tx, updated); + } + } + if (nextLabelIds !== undefined) { + await syncIssueLabels( + updated.id, + existing.companyId, + nextLabelIds, + tx, + ); + } + if (blockedByIssueIds !== undefined) { + await syncBlockedByIssueIds( + updated.id, + existing.companyId, + blockedByIssueIds, + { + agentId: actorAgentId ?? null, + userId: actorUserId ?? null, + }, + tx, + ); + } + if ( + issueData.executionWorkspaceSettings !== undefined && + nextExecutionWorkspaceId && + nextExecutionWorkspacePreference === "reuse_existing" + ) { + const workspace = await tx + .select({ + id: executionWorkspaces.id, + metadata: executionWorkspaces.metadata, + }) + .from(executionWorkspaces) + .where( + and( + eq(executionWorkspaces.id, nextExecutionWorkspaceId), + eq(executionWorkspaces.companyId, existing.companyId), + ), + ) + .then( + (rows: Array<{ id: string; metadata: unknown }>) => + rows[0] ?? null, + ); + if (workspace) { + await tx + .update(executionWorkspaces) + .set({ + metadata: mergeExecutionWorkspaceConfig( + (workspace.metadata as Record | null) ?? + null, + buildReusedExecutionWorkspaceConfigPatchFromIssueSettings( + nextExecutionWorkspaceSettings, + ), + ), + updatedAt: new Date(), + }) + .where(eq(executionWorkspaces.id, workspace.id)); + } + } + const [enriched] = await withIssueLabels(tx, [updated]); + const nextBlockedByIssueIds = + blockedByIssueIds === undefined + ? undefined + : [...new Set(blockedByIssueIds)].sort(); + const changes = buildIssueChanges( + receiptExisting as unknown as Record, + updated as unknown as Record, + { + ...(nextLabelIds !== undefined + ? { + labelIds: { + from: (previousLabelsByIssueId.get(id) ?? []).map( + (label) => label.id, + ), + to: enriched.labelIds, + }, + } + : {}), + ...(nextBlockedByIssueIds !== undefined + ? { + blockedByIssueIds: { + from: ( + previousRelationSummaries.get(id)?.blockedBy ?? [] + ).map((relation) => relation.id), + to: nextBlockedByIssueIds, + }, + } + : {}), + }, + ); + if ( + (issueData.status === "done" || issueData.status === "cancelled") && + existing.status !== issueData.status && + existing.originKind === + RECOVERY_ORIGIN_KINDS.issueGraphLivenessEscalation + ) { + const parsedIncident = parseIssueGraphLivenessIncidentKey( + existing.originId, + ); + if ( + parsedIncident?.issueId && + parsedIncident.companyId === existing.companyId + ) { + await notifyExternalConversationSettlement(tx, { companyId: existing.companyId, issueId: parsedIncident.issueId }); + await tx + .delete(issueRelations) + .where( + and( + eq(issueRelations.companyId, existing.companyId), + eq(issueRelations.issueId, existing.id), + eq(issueRelations.relatedIssueId, parsedIncident.issueId), + eq(issueRelations.type, "blocks"), + ), + ); + } + } + if ( + actorUserId && + receiptExisting.status !== "done" && + updated.status === "done" + ) { + if (dbOrTx !== db && !postCommitActivityPublications) { + throw new Error( + "Human completion in an external transaction requires a post-commit activity queue", + ); + } + const now = new Date(); + const archiveState = await archiveInbox( + updated.companyId, + updated.id, + actorUserId, + now, + undefined, + tx, + ); + const { publication } = await persistActivity(tx as unknown as Db, { + companyId: updated.companyId, + actorType: "user", + actorId: actorUserId, + action: "issue.inbox_archived", + entityType: "issue", + entityId: updated.id, + details: { + userId: actorUserId, + archivedAt: archiveState.archivedAt, + targetResolvedFrom: "responsible_user", + source: "issue_status_done", + }, + }); + activityPublications.push(publication); + } + return { + ...enriched, + ...(nextBlockedByIssueIds !== undefined + ? { blockedByIssueIds: nextBlockedByIssueIds } + : {}), + changes, + }; + }; + + const result = await (dbOrTx === db + ? db.transaction(runUpdate) + : runUpdate(dbOrTx)); + if (dbOrTx === db && !postCommitActivityPublications) { + for (const publication of ownedActivityPublications) + publishActivity(publication); + } + if (dbOrTx === db && !postCommitActions) { + await executeIssuePostCommitActions(db, ownedPostCommitActions); + } + return result; + }, + + clearExecutionWorkspaceEnvironmentSelection: async ( + companyId: string, + environmentId: string, + ) => { + const rows = await db + .select({ + id: issues.id, + executionWorkspaceSettings: issues.executionWorkspaceSettings, + }) + .from(issues) + .where(eq(issues.companyId, companyId)); + + let cleared = 0; + for (const row of rows) { + const settings = parseIssueExecutionWorkspaceSettings( + row.executionWorkspaceSettings, + { includeEnvironmentId: true }, + ); + if (settings?.environmentId !== environmentId) continue; + + await db.transaction(async tx => { + await notifyStatusCardIssueChange(tx, companyId); + await tx + .update(issues) + .set({ + executionWorkspaceSettings: { + ...settings, + environmentId: null, + }, + updatedAt: new Date(), + }) + .where(eq(issues.id, row.id)); + }); + cleared += 1; + } + + return cleared; + }, + + remove: (id: string) => + db.transaction(async (tx) => { + const attachmentAssetIds = await tx + .select({ assetId: issueAttachments.assetId }) + .from(issueAttachments) + .where(eq(issueAttachments.issueId, id)); + const issueDocumentIds = await tx + .select({ documentId: issueDocuments.documentId }) + .from(issueDocuments) + .where(eq(issueDocuments.issueId, id)); + + let removedIssue; + try { + await notifyTerminalWorkspaceWork(tx, `issue:${id}`); + await requestRestorePointDeletion(tx, { sourceIssueId: id }); + const [owner] = await tx.select({ companyId: issues.companyId }).from(issues).where(eq(issues.id, id)); + if (owner) { + await notifyExternalConversationSettlement(tx, { companyId: owner.companyId }); + await notifyApprovalWork(tx, owner.companyId); + await notifyStatusCardIssueChange(tx, owner.companyId); + } + await notifyDecisionTargetIssueRemoval(tx, id); + await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors); + await notifyRunDispatch(tx, owner ? { companyId: owner.companyId } : undefined); + await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers); + removedIssue = await tx + .delete(issues) + .where(eq(issues.id, id)) + .returning() + .then((rows) => rows[0] ?? null); + } catch (err) { + // A foreign key to issues.id without a delete policy blocks the delete + // and raises SQLSTATE 23503. Map it to a clear 409 instead of a bare + // 500. This also covers the decisions table, whose NOT NULL references + // to issues.id stay restricted on purpose. + if (isForeignKeyViolation(err)) { + throw conflict( + "Issue cannot be deleted because another record still references it.", + ); + } + throw err; + } + + if (removedIssue && attachmentAssetIds.length > 0) { + await tx.delete(assets).where( + inArray( + assets.id, + attachmentAssetIds.map((row) => row.assetId), + ), + ); + } + + if (removedIssue && issueDocumentIds.length > 0) { + await tx.delete(documents).where( + inArray( + documents.id, + issueDocumentIds.map((row) => row.documentId), + ), + ); + } + + if (!removedIssue) return null; + const [enriched] = await withIssueLabels(tx, [removedIssue]); + return enriched; + }), + + checkout: async ( + id: string, + agentId: string, + expectedStatuses: string[], + checkoutRunId: string | null, + ) => { + const issueCompany = await db + .select({ companyId: issues.companyId }) + .from(issues) + .where(eq(issues.id, id)) + .then((rows) => rows[0] ?? null); + if (!issueCompany) throw notFound("Issue not found"); + await assertAssignableAgent(db, issueCompany.companyId, agentId, { + kind: "work", + }); + + const now = new Date(); + const activePauseHold = await treeControlSvc.getActivePauseHoldGate( + issueCompany.companyId, + id, + ); + if ( + activePauseHold && + !(await isTreeHoldInteractionCheckoutAllowed( + issueCompany.companyId, + checkoutRunId, + activePauseHold, + )) + ) { + throw conflict("Issue checkout blocked by active subtree pause hold", { + issueId: id, + holdId: activePauseHold.holdId, + rootIssueId: activePauseHold.rootIssueId, + mode: activePauseHold.mode, + securityPrinciples: [ + "Complete Mediation", + "Fail Securely", + "Secure Defaults", + ], + }); + } + + await clearExecutionRunIfTerminal(id); + await clearCheckoutRunIfTerminal(id); + + const dependencyReadiness = await listIssueDependencyReadinessMap( + db, + issueCompany.companyId, + [id], + ); + const readiness = dependencyReadiness.get(id); + const unresolvedBlockerIssueIds = + readiness?.unresolvedBlockerIssueIds ?? []; + if (unresolvedBlockerIssueIds.length > 0) { + const unresolvedBlockers = await listUnresolvedBlockerDetails( + db, + issueCompany.companyId, + unresolvedBlockerIssueIds, + readiness?.pendingFinalizeBlockerIssueIds, + ); + throw unprocessable("Issue is blocked by unresolved blockers", { + unresolvedBlockerIssueIds, + unresolvedBlockers, + }); + } + + const sameRunAssigneeCondition = checkoutRunId + ? and( + eq(issues.assigneeAgentId, agentId), + or( + isNull(issues.checkoutRunId), + eq(issues.checkoutRunId, checkoutRunId), + ), + ) + : and( + eq(issues.assigneeAgentId, agentId), + isNull(issues.checkoutRunId), + ); + const executionLockCondition = checkoutRunId + ? or( + isNull(issues.executionRunId), + eq(issues.executionRunId, checkoutRunId), + ) + : isNull(issues.executionRunId); + const updated = await db.transaction(async tx => { + await notifyStatusCardIssueChange(tx, issueCompany.companyId); + await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors); + await notifyRunDispatch(tx, { companyId: issueCompany.companyId }); + await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers); + return tx + .update(issues) + .set({ + assigneeAgentId: agentId, + assigneeUserId: null, + checkoutRunId, + executionRunId: checkoutRunId, + status: "in_progress", + startedAt: now, + updatedAt: now, + }) + .where( + and( + eq(issues.id, id), + inArray(issues.status, expectedStatuses), + or(isNull(issues.assigneeAgentId), sameRunAssigneeCondition), + executionLockCondition, + ), + ) + .returning() + .then((rows) => rows[0] ?? null); + }); + + if (updated) { + const [enriched] = await withIssueLabels(db, [updated]); + return enriched; + } + + const current = await db + .select({ + id: issues.id, + status: issues.status, + assigneeAgentId: issues.assigneeAgentId, + checkoutRunId: issues.checkoutRunId, + executionRunId: issues.executionRunId, + }) + .from(issues) + .where(eq(issues.id, id)) + .then((rows) => rows[0] ?? null); + + if (!current) throw notFound("Issue not found"); + + if ( + current.assigneeAgentId === agentId && + current.status === "in_progress" && + current.checkoutRunId == null && + (current.executionRunId == null || + current.executionRunId === checkoutRunId) && + checkoutRunId + ) { + const adopted = await db.transaction(async tx => { + await notifyStatusCardIssueChange(tx, issueCompany.companyId); + return tx + .update(issues) + .set({ + checkoutRunId, + executionRunId: checkoutRunId, + updatedAt: new Date(), + }) + .where( + and( + eq(issues.id, id), + eq(issues.status, "in_progress"), + eq(issues.assigneeAgentId, agentId), + isNull(issues.checkoutRunId), + or( + isNull(issues.executionRunId), + eq(issues.executionRunId, checkoutRunId), + ), + ), + ) + .returning() + .then((rows) => rows[0] ?? null); + }); + if (adopted) return adopted; + } + + if ( + checkoutRunId && + current.assigneeAgentId === agentId && + current.status === "in_progress" && + current.checkoutRunId && + current.checkoutRunId !== checkoutRunId + ) { + const staleAdoption = await adoptStaleCheckoutRun({ + issueId: id, + actorAgentId: agentId, + actorRunId: checkoutRunId, + expectedCheckoutRunId: current.checkoutRunId, + }); + if (staleAdoption.adopted) { + const row = await db + .select() + .from(issues) + .where(eq(issues.id, id)) + .then((rows) => rows[0] ?? null); + if (!row) throw notFound("Issue not found"); + const [enriched] = await withIssueLabels(db, [row]); + return enriched; + } + } + + // Adopt stale executionRunId — if the execution lock points to a terminal/missing run, clear it and proceed. + // Only adopts when the caller's expectedStatuses guard still holds; preserves any existing assigneeUserId + // and preserves the original startedAt when the issue is already in_progress. + if ( + checkoutRunId && + current.executionRunId && + current.executionRunId !== checkoutRunId && + (current.assigneeAgentId === agentId || current.assigneeAgentId == null) + ) { + const executionRun = await db.select().from(heartbeatRuns) + .where(eq(heartbeatRuns.id, current.executionRunId)).then(rows => rows[0] ?? null); + const stale = !executionRun || TERMINAL_HEARTBEAT_RUN_STATUSES.has(executionRun.status); + if (stale && !isExplicitContinuationRetryClaim({ ...current, companyId: issueCompany.companyId }, executionRun)) { + const now = new Date(); + const adoptionSet: Record = { + assigneeAgentId: agentId, + checkoutRunId, + executionRunId: checkoutRunId, + executionAgentNameKey: null, + executionLockedAt: now, + status: "in_progress", + updatedAt: now, + }; + if (current.status !== "in_progress") { + adoptionSet.startedAt = now; + } + const staleExecutionRunId = current.executionRunId; + const adopted = await db.transaction(async tx => { + await notifyStatusCardIssueChange(tx, issueCompany.companyId); + await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors); + await notifyRunDispatch(tx, { companyId: issueCompany.companyId }); + await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers); + return tx + .update(issues) + .set(adoptionSet) + .where( + and( + eq(issues.id, id), + inArray(issues.status, expectedStatuses), + eq(issues.executionRunId, staleExecutionRunId), + or( + isNull(issues.assigneeAgentId), + eq(issues.assigneeAgentId, agentId), + ), + ), + ) + .returning() + .then((rows) => rows[0] ?? null); + }); + if (adopted) { + const [enriched] = await withIssueLabels(db, [adopted]); + return enriched; + } + } + } + + // If this run already owns it and it's in_progress, return it (no self-409) + if ( + current.assigneeAgentId === agentId && + current.status === "in_progress" && + sameRunLock(current.checkoutRunId, checkoutRunId) + ) { + const row = await db + .select() + .from(issues) + .where(eq(issues.id, id)) + .then((rows) => rows[0] ?? null); + if (!row) throw notFound("Issue not found"); + const [enriched] = await withIssueLabels(db, [row]); + return enriched; + } + + throw conflict("Issue checkout conflict", { + issueId: current.id, + status: current.status, + assigneeAgentId: current.assigneeAgentId, + checkoutRunId: current.checkoutRunId, + executionRunId: current.executionRunId, + }); + }, + + assertCheckoutOwner: async ( + id: string, + actorAgentId: string, + actorRunId: string | null, + ) => { + await clearExecutionRunIfTerminal(id); + await clearCheckoutRunIfTerminal(id); + const loadCurrent = () => + db + .select({ + id: issues.id, + status: issues.status, + assigneeAgentId: issues.assigneeAgentId, + checkoutRunId: issues.checkoutRunId, + executionRunId: issues.executionRunId, + }) + .from(issues) + .where(eq(issues.id, id)) + .then((rows) => rows[0] ?? null); + const current = await loadCurrent(); + + if (!current) throw notFound("Issue not found"); + + const resolveSameRunOwnership = (candidate: { + id: string; + status: string; + assigneeAgentId: string | null; + checkoutRunId: string | null; + executionRunId: string | null; + }) => { + if ( + candidate.status === "in_progress" && + candidate.assigneeAgentId === actorAgentId && + sameRunLock(candidate.checkoutRunId, actorRunId) + ) { + return { ...candidate, adoptedFromRunId: null as string | null }; + } + return null; + }; + + const canAdoptUnownedCheckout = (candidate: { + status: string; + assigneeAgentId: string | null; + checkoutRunId: string | null; + executionRunId: string | null; + }) => + actorRunId && + candidate.status === "in_progress" && + candidate.assigneeAgentId === actorAgentId && + candidate.checkoutRunId == null && + (candidate.executionRunId == null || + candidate.executionRunId === actorRunId); + + const resolveOwnership = async (candidate: { + id: string; + status: string; + assigneeAgentId: string | null; + checkoutRunId: string | null; + executionRunId: string | null; + }) => { + const sameRunOwnership = resolveSameRunOwnership(candidate); + if (sameRunOwnership) + return { ownership: sameRunOwnership, latest: null }; + + if (canAdoptUnownedCheckout(candidate)) { + const adopted = await adoptUnownedCheckoutRun({ + issueId: id, + actorAgentId, + actorRunId: actorRunId!, + }); + + if (adopted) { + return { + ownership: { + ...adopted, + adoptedFromRunId: null as string | null, + }, + latest: null, + }; + } + } + + if ( + actorRunId && + candidate.status === "in_progress" && + candidate.assigneeAgentId === actorAgentId && + candidate.checkoutRunId && + candidate.checkoutRunId !== actorRunId + ) { + const previousCheckoutRunId = candidate.checkoutRunId; + const staleAdoption = await adoptStaleCheckoutRun({ + issueId: id, + actorAgentId, + actorRunId, + expectedCheckoutRunId: previousCheckoutRunId, + }); + + if (staleAdoption.adopted) { + return { + ownership: { + ...staleAdoption.adopted, + adoptedFromRunId: previousCheckoutRunId, + }, + latest: null, + }; + } + + if (staleAdoption.latest) { + const latestOwnership = resolveSameRunOwnership( + staleAdoption.latest, + ); + if (latestOwnership) + return { + ownership: latestOwnership, + latest: staleAdoption.latest, + }; + return { ownership: null, latest: staleAdoption.latest }; + } + } + + return { ownership: null, latest: null }; + }; + + const resolved = await resolveOwnership(current); + if (resolved.ownership) return resolved.ownership; + + const latest = resolved.latest ?? (await loadCurrent()); + if (!latest) throw notFound("Issue not found"); + const resolvedLatest = await resolveOwnership(latest); + if (resolvedLatest.ownership) return resolvedLatest.ownership; + if (resolvedLatest.latest) { + throw conflict("Issue run ownership conflict", { + issueId: resolvedLatest.latest.id, + status: resolvedLatest.latest.status, + assigneeAgentId: resolvedLatest.latest.assigneeAgentId, + checkoutRunId: resolvedLatest.latest.checkoutRunId, + executionRunId: resolvedLatest.latest.executionRunId, + actorAgentId, + actorRunId, + }); + } + + throw conflict("Issue run ownership conflict", { + issueId: latest.id, + status: latest.status, + assigneeAgentId: latest.assigneeAgentId, + checkoutRunId: latest.checkoutRunId, + executionRunId: latest.executionRunId, + actorAgentId, + actorRunId, + }); + }, + + release: async ( + id: string, + actorAgentId?: string, + actorRunId?: string | null, + ) => + db.transaction(async (tx) => { + await tx.execute( + sql`select ${issues.id} from ${issues} where ${issues.id} = ${id} for update`, + ); + const existing = await tx + .select() + .from(issues) + .where(eq(issues.id, id)) + .then((rows) => rows[0] ?? null); + + if (!existing) return null; + if ( + actorAgentId && + existing.assigneeAgentId && + existing.assigneeAgentId !== actorAgentId + ) { + throw conflict("Only assignee can release issue"); + } + if ( + actorAgentId && + existing.status === "in_progress" && + existing.assigneeAgentId === actorAgentId && + existing.checkoutRunId && + !sameRunLock(existing.checkoutRunId, actorRunId ?? null) + ) { + const stale = await isTerminalOrMissingHeartbeatRun( + existing.checkoutRunId, + tx, + ); + if (!stale) { + throw conflict("Only checkout run can release issue", { + issueId: existing.id, + assigneeAgentId: existing.assigneeAgentId, + checkoutRunId: existing.checkoutRunId, + actorRunId: actorRunId ?? null, + }); + } + } + + // Terminal assignment records who owned the work, not a live execution + // claim. Cleanup must preserve it; unfinished release still relinquishes it. + const isTerminal = existing.status === "done" || existing.status === "cancelled"; + const releaseStatus = + existing.status === "in_progress" ? "todo" : existing.status; + await notifyTerminalWorkspaceWork(tx, `issue:${id}`); + await notifyNativeStatusReassessment(tx, { companyId: existing.companyId, issueId: id }); + await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors); + await notifyRunDispatch(tx, { companyId: existing.companyId }); + await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers); + const updated = await tx + .update(issues) + .set({ + status: releaseStatus, + assigneeAgentId: isTerminal ? existing.assigneeAgentId : null, + checkoutRunId: null, + executionRunId: null, + executionAgentNameKey: null, + executionLockedAt: null, + updatedAt: new Date(), + }) + .where(eq(issues.id, id)) + .returning() + .then((rows) => rows[0] ?? null); + if (!updated) return null; + const [enriched] = await withIssueLabels(tx, [updated]); + return enriched; + }), + + adminForceRelease: async ( + id: string, + options: { clearAssignee?: boolean } = {}, + ) => + db.transaction(async (tx) => { + await tx.execute( + sql`select ${issues.id} from ${issues} where ${issues.id} = ${id} for update`, + ); + const existing = await tx + .select({ + id: issues.id, + companyId: issues.companyId, + checkoutRunId: issues.checkoutRunId, + executionRunId: issues.executionRunId, + }) + .from(issues) + .where(eq(issues.id, id)) + .then((rows) => rows[0] ?? null); + if (!existing) return null; + + const patch: Partial = { + checkoutRunId: null, + executionRunId: null, + executionAgentNameKey: null, + executionLockedAt: null, + updatedAt: new Date(), + }; + if (options.clearAssignee) { + patch.assigneeAgentId = null; + } + + await notifyStatusCardIssueChange(tx, existing.companyId); + await notifyTerminalWorkspaceWork(tx, `issue:${id}`); + if (existing.checkoutRunId || existing.executionRunId || options.clearAssignee) { + await notifyExternalConversationSettlement(tx, { companyId: existing.companyId, issueId: id }); + await notifyNativeStatusReassessment(tx, { companyId: existing.companyId, issueId: id }); + } + if (existing.checkoutRunId || existing.executionRunId) { + await notifyUnsafeWorkspaceExportRecovery(tx, { companyId: existing.companyId, issueId: id }); + } + await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors); + await notifyRunDispatch(tx, { companyId: existing.companyId }); + await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers); + const updated = await tx + .update(issues) + .set(patch) + .where(eq(issues.id, id)) + .returning() + .then((rows) => rows[0] ?? null); + if (!updated) return null; + + const [enriched] = await withIssueLabels(tx, [updated]); + return { + issue: enriched, + previous: { + checkoutRunId: existing.checkoutRunId, + executionRunId: existing.executionRunId, + }, + }; + }), + + listLabels: (companyId: string) => + db + .select() + .from(labels) + .where(eq(labels.companyId, companyId)) + .orderBy(asc(labels.name), asc(labels.id)), + + getLabelById: (id: string) => + db + .select() + .from(labels) + .where(eq(labels.id, id)) + .then((rows) => rows[0] ?? null), + + createLabel: async ( + companyId: string, + data: Pick, + ) => { + const [created] = await db + .insert(labels) + .values({ + companyId, + name: data.name.trim(), + color: data.color, + }) + .returning(); + return created; + }, + + deleteLabel: async (id: string) => db.transaction(async tx => { + const [label] = await tx.select({ companyId: labels.companyId }).from(labels).where(eq(labels.id, id)); + if (!label) return null; + await notifyStatusCardIssueChange(tx, label.companyId); + return tx.delete(labels).where(eq(labels.id, id)).returning().then(rows => rows[0] ?? null); + }), + + listComments: async ( + issueId: string, + opts?: { + afterCommentId?: string | null; + order?: "asc" | "desc"; + limit?: number | null; + humanOnly?: boolean; + companyId?: string; + }, + ) => { + const order = opts?.order === "asc" ? "asc" : "desc"; + const afterCommentId = opts?.afterCommentId?.trim() || null; + const limit = + opts?.limit && opts.limit > 0 + ? Math.min(Math.floor(opts.limit), MAX_ISSUE_COMMENT_PAGE_LIMIT) + : null; + + const conditions = [eq(issueComments.issueId, issueId)]; + if (opts?.companyId) conditions.push(eq(issueComments.companyId, opts.companyId)); + if (opts?.humanOnly) conditions.push(or( + eq(issueComments.authorType, "user"), + and(isNull(issueComments.authorType), isNotNull(issueComments.authorUserId)), + )!); + if (afterCommentId) { + // Guard: reject non-UUID cursors before hitting the DB to avoid Postgres type errors. + if (!isUuidLike(afterCommentId)) return []; + const anchor = await db + .select({ + id: issueComments.id, + createdAt: issueComments.createdAt, + }) + .from(issueComments) + .where( + and( + eq(issueComments.issueId, issueId), + eq(issueComments.id, afterCommentId), + ), + ) + .then((rows) => rows[0] ?? null); + + if (!anchor) return []; + const anchorCreatedAt = + anchor.createdAt instanceof Date + ? anchor.createdAt + : new Date(String(anchor.createdAt)); + conditions.push( + order === "asc" + ? or( + gt(issueComments.createdAt, anchorCreatedAt), + and( + eq(issueComments.createdAt, anchorCreatedAt), + gt(issueComments.id, anchor.id), + ), + )! + : or( + lt(issueComments.createdAt, anchorCreatedAt), + and( + eq(issueComments.createdAt, anchorCreatedAt), + lt(issueComments.id, anchor.id), + ), + )!, + ); + } + + const query = db + .select() + .from(issueComments) + .where(and(...conditions)) + .orderBy( + order === "asc" + ? asc(issueComments.createdAt) + : desc(issueComments.createdAt), + order === "asc" ? asc(issueComments.id) : desc(issueComments.id), + ); + + const comments = limit ? await query.limit(limit) : await query; + const { censorUsernameInLogs } = await instanceSettings.getGeneral(); + const projectedComments = await projectHistoricalRunComments(comments); + const enrichedComments = + await enrichCommentsWithDerivedAgentAttribution(projectedComments); + return enrichedComments.map((comment) => + redactIssueComment(comment, censorUsernameInLogs), + ); + }, + + getCommentCursor: async (issueId: string) => { + const [latest, countRow] = await Promise.all([ + db + .select({ + latestCommentId: issueComments.id, + latestCommentAt: issueComments.createdAt, + }) + .from(issueComments) + .where(eq(issueComments.issueId, issueId)) + .orderBy(desc(issueComments.createdAt), desc(issueComments.id)) + .limit(1) + .then((rows) => rows[0] ?? null), + db + .select({ + totalComments: sql`count(*)::int`, + }) + .from(issueComments) + .where(eq(issueComments.issueId, issueId)) + .then((rows) => rows[0] ?? null), + ]); + + return { + totalComments: Number(countRow?.totalComments ?? 0), + latestCommentId: latest?.latestCommentId ?? null, + latestCommentAt: latest?.latestCommentAt ?? null, + }; + }, + + getComment: async (commentId: string) => { + const { censorUsernameInLogs } = await instanceSettings.getGeneral(); + const comment = await db + .select() + .from(issueComments) + .where(eq(issueComments.id, commentId)) + .then((rows) => rows[0] ?? null); + if (!comment) return null; + const [projectedComment] = await projectHistoricalRunComments([comment]); + const [enrichedComment] = await enrichCommentsWithDerivedAgentAttribution( + [projectedComment ?? comment], + ); + return redactIssueComment( + enrichedComment ?? projectedComment ?? comment, + censorUsernameInLogs, + ); + }, + + removeComment: async (commentId: string) => { + const currentUserRedactionOptions = { + enabled: (await instanceSettings.getGeneral()).censorUsernameInLogs, + }; + + return db.transaction(async (tx) => { + if (!(await lockIssueForCommentMutation(tx, commentId))) return null; + const [source] = await tx.select({ companyId: issueComments.companyId, issueId: issueComments.issueId }) + .from(issueComments).where(eq(issueComments.id, commentId)); + if (!source) return null; + await notifyExternalConversationSettlement(tx, source); + await notifyStatusCardIssueChange(tx, source.companyId); + const [comment] = await tx + .delete(issueComments) + .where(eq(issueComments.id, commentId)) + .returning(); + + if (!comment) return null; + + await tx + .update(issues) + .set({ updatedAt: new Date() }) + .where(eq(issues.id, comment.issueId)); + + return redactIssueComment(comment, currentUserRedactionOptions.enabled); + }); + }, + + tombstoneComment: async ( + commentId: string, + actor: { + actorType: "agent" | "user"; + agentId?: string | null; + userId?: string | null; + runId?: string | null; + }, + options?: { + afterTombstone?: (comment: IssueComment, tx: any) => Promise; + }, + ) => { + const currentUserRedactionOptions = { + enabled: (await instanceSettings.getGeneral()).censorUsernameInLogs, + }; + + return db.transaction(async (tx) => { + if (!(await lockIssueForCommentMutation(tx, commentId))) return null; + const [source] = await tx.select({ companyId: issueComments.companyId, issueId: issueComments.issueId }) + .from(issueComments).where(and(eq(issueComments.id, commentId), isNull(issueComments.deletedAt))) + .for("update"); + if (!source) return null; + await notifyExternalConversationSettlement(tx, source); + await notifyStatusCardIssueChange(tx, source.companyId); + await notifyChatInboundWakeNotices(tx, source); + const now = new Date(); + const [comment] = await tx + .update(issueComments) + .set({ + body: DELETED_ISSUE_COMMENT_BODY, + presentation: null, + metadata: null, + deletedAt: now, + deletedByType: actor.actorType, + deletedByAgentId: + actor.actorType === "agent" ? (actor.agentId ?? null) : null, + deletedByUserId: + actor.actorType === "user" ? (actor.userId ?? null) : null, + deletedByRunId: actor.runId ?? null, + updatedAt: now, + }) + .where( + and( + eq(issueComments.id, commentId), + isNull(issueComments.deletedAt), + ), + ) + .returning(); + + if (!comment) return null; + + await tx + .update(issues) + .set({ updatedAt: now }) + .where(eq(issues.id, comment.issueId)); + + const redacted = redactIssueComment( + comment, + currentUserRedactionOptions.enabled, + ); + await options?.afterTombstone?.(redacted, tx); + + return redacted; + }); + }, + + addComment: async function addComment( + issueId: string, + body: string, + actor: { + agentId?: string; + userId?: string; + runId?: string | null; + onBehalfOfUserId?: string | null; + }, + options?: { + authorType?: IssueCommentAuthorType | null; + presentation?: IssueCommentPresentation | null; + metadata?: IssueCommentMetadata | null; + attachmentIds?: string[]; + authorizationReason?: string | null; + /** Server-only final assistant response, never a tool/progress comment. */ + completionReply?: boolean; + sourceTrust?: typeof issueComments.$inferInsert.sourceTrust; + createdAt?: Date | string | null; + clientRequestId?: string; + /** Server-only: authenticated Paperclip messages also belong in the Slack thread. */ + mirrorToSlack?: boolean; + }, + dbOrTx: any = db, + ): Promise { + if (dbOrTx === db) { + const append = () => + measureRunTransaction(db, "response", async (tx) => { + // Serialize run-authored comments on the issue so a provider retry + // cannot publish the same visible result twice. This needs no schema + // change: the issue row is the transaction fence, and the recursive + // call below performs the lookup and insert while holding it. + await measureRunPhase("response.row_lock", () => tx + .select({ id: issues.id }) + .from(issues) + .where(eq(issues.id, issueId)) + .for("update")); + return addComment(issueId, body, actor, options, tx); + }); + return options?.authorizationReason === + CHAT_RUN_PRESENTATION_AUTHORIZATION_REASON + ? retryNativeChatReviewPresentation(append) + : append(); + } + // The query below locks human comments on caller-owned transactions too, + // sharing the fence with both question creation and Slack settlement. + const issueQuery = dbOrTx + .select({ companyId: issues.companyId, conversationAgentId: issues.conversationAgentId }) + .from(issues) + .where(eq(issues.id, issueId)); + // Caller-owned transactions (including chat and review comments) must + // serialize with question creation before inserting the human comment. + const issue = await (actor.userId || (actor.runId && dbOrTx !== db) ? issueQuery.for("update") : issueQuery) + .then((rows: Array<{ companyId: string; conversationAgentId: string | null }>) => rows[0] ?? null); + + if (!issue) throw notFound("Issue not found"); + + if (issue.conversationAgentId && actor.userId && !(await instanceSettingsService(dbOrTx).getExperimental()).enableAgentChat) { + throw unprocessable("Agent Chat is disabled in Experimental settings"); + } + const currentUserRedactionOptions = { + // Keep every read on the caller's transaction connection. Re-entering + // the outer pool here can deadlock when concurrent transactions fill + // the pool while waiting on the same issue or delivery row. + enabled: (await instanceSettings.getGeneral({ db: dbOrTx })) + .censorUsernameInLogs, + }; + const redactedBody = redactCurrentUserText(body, currentUserRedactionOptions); + if (actor.userId && options?.clientRequestId) { + const [existing] = await dbOrTx.select().from(issueComments).where(and(eq(issueComments.issueId, issueId), + eq(issueComments.authorUserId, actor.userId), eq(issueComments.clientRequestId, options.clientRequestId))); + if (existing) { + if (existing.body !== redactedBody) throw conflict("Message request ID was already used for different content"); + return existing; + } + } + if (issue.conversationAgentId && actor.runId) { + const [run] = await dbOrTx.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, actor.runId)); + const [current] = await dbOrTx.select().from(issues).where(eq(issues.id, issueId)); + if (run?.status === "cancelled") throw conflict("This conversation turn was cancelled; it cannot post a reply"); + if (run?.contextSnapshot?.conversationSessionGeneration !== current.conversationSessionGeneration) { + throw conflict("Conversation session changed; this reply belongs to an earlier session"); + } + } + if (options?.completionReply && actor.agentId && actor.runId) { + const delivered = await existingChatCompletionReply(dbOrTx, actor.runId, issueId); + if (delivered) return redactIssueComment(delivered, currentUserRedactionOptions.enabled); + } + const authorType = issueCommentAuthorTypeSchema.parse( + options?.authorType ?? + (actor.agentId ? "agent" : actor.userId ? "user" : "system"), + ); + assertIssueCommentAuthorTypeAllowed(actor, authorType); + const presentation = issueCommentPresentationSchema + .nullable() + .parse(options?.presentation ?? null); + const createdAt = options?.createdAt ? new Date(options.createdAt) : null; + const validCreatedAt = + createdAt && !Number.isNaN(createdAt.getTime()) ? createdAt : null; + // Use one statement timestamp for both columns when the caller did not + // supply a valid historical timestamp. This keeps the comment's + // recency fields equal even when the surrounding transaction started + // earlier, while preserving imported timestamps and the normal default + // updatedAt behavior for those historical rows. + const currentInsertTimestamp = validCreatedAt + ? null + : sql`statement_timestamp()`; + // Invalid/stale run ids must not 500 the insert — null out unknowns. + const createdByRun = await resolveCommentCreatedByRun( + dbOrTx, + issue.companyId, + actor.runId, + ); + const createdByRunId = createdByRun?.id ?? null; + if ( + createdByRunId && + options?.authorizationReason === CHAT_RUN_PRESENTATION_AUTHORIZATION_REASON + ) { + const [presentationRun] = await dbOrTx + .select({ resultJson: heartbeatRuns.resultJson }) + .from(heartbeatRuns) + .where( + and( + eq(heartbeatRuns.id, createdByRunId), + eq(heartbeatRuns.companyId, issue.companyId), + ), + ) + .limit(1); + if ( + presentationRun?.resultJson?.finalizationReasonCode === + "governed_response_waiting" && + (parseObject(presentationRun.resultJson.nativeResult).summary !== body || + !(await authorizeNativeChatReviewPresentation( + dbOrTx, + { + companyId: issue.companyId, + issueId, + runId: createdByRunId, + resultJson: presentationRun.resultJson, + }, + "nonblocking", + ))) + ) { + throw conflict( + "This chat response is no longer authorized for external presentation", + { code: "chat_review_response_presentation_denied" }, + ); + } + } + if (actor.runId && !createdByRunId) { + logger.warn( + { issueId, companyId: issue.companyId, runId: actor.runId }, + "dropping invalid createdByRunId for issue comment insert", + ); + } + const attachmentIds = [...new Set(options?.attachmentIds ?? [])]; + if (actor.agentId && attachmentIds.length > 0) { + if (!createdByRunId) { + throw unprocessable( + "Agent comment attachments require the current registered run", + ); + } + // Routes commonly supply an existing transaction for attachment + // comments, so the outer addComment wrapper is not always responsible + // for serialization. Lock both stable parents here before selecting or + // binding files. Concurrent helper calls for the same run then observe + // one another's committed selection count instead of both admitting a + // twenty-first attachment. + const [lockedIssue] = await dbOrTx + .select({ id: issues.id }) + .from(issues) + .where( + and( + eq(issues.id, issueId), + eq(issues.companyId, issue.companyId), + ), + ) + .for("update"); + if (!lockedIssue) throw notFound("Issue not found"); + const [lockedRun] = await dbOrTx + .select({ id: heartbeatRuns.id }) + .from(heartbeatRuns) + .where( + and( + eq(heartbeatRuns.id, createdByRunId), + eq(heartbeatRuns.companyId, issue.companyId), + eq(heartbeatRuns.agentId, actor.agentId), + ), + ) + .for("update"); + if (!lockedRun) { + throw unprocessable( + "Agent comment attachments require the current registered run", + ); + } + } + const onBehalfOfUserId = actor.agentId + ? await resolveCommentResponsibleUserId( + dbOrTx, + issue.companyId, + createdByRunId, + actor.onBehalfOfUserId, + ) + : null; + const metadata = issueCommentMetadataSchema + .nullable() + .parse( + actor.agentId + ? withAgentCommentAuthorizationMetadata( + options?.metadata ?? null, + options?.authorizationReason, + ) + : (options?.metadata ?? null), + ); + let comment: typeof issueComments.$inferSelect | null = null; + if (createdByRunId) { + const existing = await dbOrTx + .select() + .from(issueComments) + .where( + and( + eq(issueComments.companyId, issue.companyId), + eq(issueComments.issueId, issueId), + eq(issueComments.createdByRunId, createdByRunId), + eq(issueComments.authorType, authorType), + actor.agentId + ? eq(issueComments.authorAgentId, actor.agentId) + : isNull(issueComments.authorAgentId), + eq(issueComments.body, redactedBody), + isNull(issueComments.deletedAt), + ), + ) + .orderBy(issueComments.createdAt, issueComments.id) + .limit(1) + .then( + (rows: Array) => rows[0] ?? null, + ); + if (existing) { + // Heartbeat's presentation resolver can intentionally select the + // exact same prose that the agent already wrote while handling an + // external-chat continuation. The first write is kept internal so + // lifecycle/tool chatter cannot escape; the later, narrowly + // authorized presentation must upgrade that one durable comment + // instead of returning early and letting the generic completion + // milestone win the provider reply slot. + const shouldUpgradeExternalAuthorization = + authorType === "agent" && + metadata?.authorizationReason === + CHAT_RUN_PRESENTATION_AUTHORIZATION_REASON && + existing.metadata?.authorizationReason !== + CHAT_RUN_PRESENTATION_AUTHORIZATION_REASON; + const shouldBindAttachments = + (options?.attachmentIds?.length ?? 0) > 0; + const shouldUpgradeAttachmentAuthorization = + shouldBindAttachments && + isExplicitExternalAgentComment(metadata) && + !isExplicitExternalAgentComment(existing.metadata); + if ( + !shouldUpgradeExternalAuthorization && + !shouldUpgradeAttachmentAuthorization && + !shouldBindAttachments + ) { + if (options?.completionReply && actor.agentId && createdByRunId) await acknowledgeChatCompletionReply(dbOrTx, createdByRunId, existing.id); + return redactIssueComment( + existing, + currentUserRedactionOptions.enabled, + ); + } + await notifyStatusCardIssueChange(dbOrTx, issue.companyId); + comment = + shouldUpgradeExternalAuthorization || + shouldUpgradeAttachmentAuthorization + ? await dbOrTx + .update(issueComments) + .set({ + // Preserve any structured provenance the provisional comment + // carried; the presentation pass only elevates its narrowly + // resolved external-publication authorization. + metadata: { + ...(existing.metadata ?? {}), + ...(metadata ?? {}), + }, + updatedAt: new Date(), + }) + .where( + and( + eq(issueComments.id, existing.id), + eq(issueComments.companyId, issue.companyId), + ), + ) + .returning() + .then( + (rows: Array) => + rows[0] ?? null, + ) + : existing; + } + } + if (!comment) { + [comment] = await measureRunPhase("response.write", () => insertIssueComment(dbOrTx, { + companyId: issue.companyId, + issueId, + authorAgentId: actor.agentId ?? null, + authorUserId: actor.userId ?? null, + onBehalfOfUserId, + authorType, + createdByRunId, + body: redactedBody, + clientRequestId: options?.clientRequestId ?? null, + presentation, + metadata, + sourceTrust: options?.sourceTrust ?? null, + createdAt: validCreatedAt ?? currentInsertTimestamp!, + ...(currentInsertTimestamp + ? { updatedAt: currentInsertTimestamp } + : {}), + })); + } + if (!comment) throw new Error("Failed to create issue comment"); + if (options?.completionReply && actor.agentId && createdByRunId) await acknowledgeChatCompletionReply(dbOrTx, createdByRunId, comment.id); + + const boundAttachments: Array<{ + id: string; + commentId: string; + originalFilename: string | null; + }> = []; + if (attachmentIds.length > 0) { + const attachmentRows = await dbOrTx + .select({ + id: issueAttachments.id, + issueCommentId: issueAttachments.issueCommentId, + originatingRunId: issueAttachments.originatingRunId, + createdByAgentId: assets.createdByAgentId, + originalFilename: assets.originalFilename, + }) + .from(issueAttachments) + .innerJoin(assets, eq(issueAttachments.assetId, assets.id)) + .where( + and( + eq(issueAttachments.companyId, issue.companyId), + eq(issueAttachments.issueId, issueId), + inArray(issueAttachments.id, attachmentIds), + ), + ) + .for("update"); + type CommentAttachmentRow = { + id: string; + issueCommentId: string | null; + originatingRunId: string | null; + createdByAgentId: string | null; + originalFilename: string | null; + }; + const attachmentById = new Map( + attachmentRows.map( + ( + attachment: CommentAttachmentRow, + ): [string, CommentAttachmentRow] => [attachment.id, attachment], + ), + ); + if (attachmentById.size !== attachmentIds.length) { + throw unprocessable( + "Comment attachments must belong to the same task and company", + ); + } + if ( + attachmentRows.some( + (attachment: { issueCommentId: string | null }) => + attachment.issueCommentId !== null && + attachment.issueCommentId !== comment.id, + ) + ) { + throw conflict( + "Comment attachments are already bound to another comment", + { + code: "issue_comment_attachments_already_bound", + companyId: issue.companyId, + issueId, + attachmentIds: attachmentRows + .filter( + (attachment: CommentAttachmentRow) => + attachment.issueCommentId !== null && + attachment.issueCommentId !== comment.id, + ) + .map((attachment: CommentAttachmentRow) => attachment.id), + }, + ); + } + if (actor.agentId && createdByRunId) { + if ( + attachmentRows.some( + (attachment: CommentAttachmentRow) => + attachment.createdByAgentId !== actor.agentId || + attachment.originatingRunId !== createdByRunId, + ) + ) { + throw unprocessable( + "Agent comment attachments must originate from the same agent and current run", + { code: "issue_attachment_run_origin_mismatch" }, + ); + } + const chatBindings = await resolveChatOriginPublicationBindings( + dbOrTx, + issue.companyId, + issueId, + createdByRunId, + ); + if (chatBindings.length > 0) { + const alreadySelected = + await listSelectedChatPresentationAttachments(dbOrTx, { + companyId: issue.companyId, + issueId, + agentId: actor.agentId, + runId: createdByRunId, + }); + const selectedIds = new Set( + alreadySelected.map((attachment) => attachment.id), + ); + for (const attachmentId of attachmentIds) { + selectedIds.add(attachmentId); + } + if (selectedIds.size > MAX_CHAT_PRESENTATION_ATTACHMENTS) { + throw unprocessable( + `An agent run can select at most ${MAX_CHAT_PRESENTATION_ATTACHMENTS} attachments for one chat response`, + { + code: "chat_attachment_selection_limit_exceeded", + limit: MAX_CHAT_PRESENTATION_ATTACHMENTS, + selectedCount: selectedIds.size, + }, + ); + } + } + } + const unboundAttachmentIds = attachmentRows.flatMap( + (attachment: CommentAttachmentRow) => + attachment.issueCommentId === null ? [attachment.id] : [], + ); + const attached = unboundAttachmentIds.length + ? await dbOrTx + .update(issueAttachments) + .set({ issueCommentId: comment.id, updatedAt: new Date() }) + .where( + and( + eq(issueAttachments.companyId, issue.companyId), + eq(issueAttachments.issueId, issueId), + inArray(issueAttachments.id, unboundAttachmentIds), + isNull(issueAttachments.issueCommentId), + ), + ) + .returning({ id: issueAttachments.id }) + : []; + if (attached.length !== unboundAttachmentIds.length) { + throw conflict( + "Comment attachments changed before they could be bound", + ); + } + for (const attachmentId of attachmentIds) { + const attachment = attachmentById.get(attachmentId)!; + boundAttachments.push({ + id: attachment.id, + commentId: comment.id, + originalFilename: attachment.originalFilename, + }); + } + } + + // A chat-origin run binds only the files it explicitly selects to an + // ordinary agent comment while it is still running. That comment stays + // internal until heartbeat chooses the final provider presentation. At + // that point, carry those already-authorized same-agent/same-run files + // forward without inferring any unbound artifact from the run. + if ( + authorType === "agent" && + actor.agentId && + createdByRunId && + metadata?.authorizationReason === + CHAT_RUN_PRESENTATION_AUTHORIZATION_REASON + ) { + const priorRows = await listSelectedChatPresentationAttachments( + dbOrTx, + { + companyId: issue.companyId, + issueId, + agentId: actor.agentId, + runId: createdByRunId, + }, + ); + const selectedIds = new Set( + boundAttachments.map((attachment) => attachment.id), + ); + for (const attachment of priorRows) { + if ( + selectedIds.has(attachment.id) || + !isExplicitExternalAgentComment(attachment.commentMetadata) + ) { + continue; + } + selectedIds.add(attachment.id); + boundAttachments.push({ + id: attachment.id, + commentId: attachment.commentId, + originalFilename: attachment.originalFilename, + }); + } + if (boundAttachments.length > MAX_CHAT_PRESENTATION_ATTACHMENTS) { + throw unprocessable( + `An agent run can select at most ${MAX_CHAT_PRESENTATION_ATTACHMENTS} attachments for one chat response`, + { + code: "chat_attachment_selection_limit_exceeded", + limit: MAX_CHAT_PRESENTATION_ATTACHMENTS, + selectedCount: boundAttachments.length, + }, + ); + } + } + + if (issue.conversationAgentId && actor.userId) { + await dbOrTx.update(issues).set({ conversationState: "active" }).where(eq(issues.id, issueId)); + } + if (options?.mirrorToSlack && actor.userId && authorType === "user") { + await mirrorSlackBoardComment(dbOrTx, comment, { attachmentIds: options.attachmentIds }); + } + if (authorType === "user" || actor.userId) { + await resumeSlackConversation(dbOrTx, issue.companyId, issueId); + } + // Update issue's updatedAt so comment activity is reflected in recency sorting + await dbOrTx + .update(issues) + .set({ updatedAt: new Date() }) + .where(eq(issues.id, issueId)); + + // Only an explicitly authored comment from the run causally woken by an + // inbound chat message is automatically publishable. Presentation, + // recovery, automation, and ordinary internal agent comments stay in + // Paperclip even while a bound conversation is active. + if (authorType === "agent" && isExplicitExternalAgentComment(metadata)) { + // An external-chat run may perform ordinary Paperclip lifecycle or + // bookkeeping writes before its adapter result is finalized. Those + // writes remain internal: only heartbeat's selected final presentation + // may consume this provider response slot. Explicit board "Send to + // channel" publications use the separate publication path. + const chatFinalOwnsProviderReply = + createdByRun !== null && + isExternalChatPresentationContext( + createdByRun.contextSnapshot, + readStringFromRecord(createdByRun.contextSnapshot, "source") === "tool_action_review" && + (await resolveChatOriginPublicationBindings( + dbOrTx, issue.companyId, issueId, createdByRunId, + )).length > 0, + ) && + metadata?.authorizationReason !== + CHAT_RUN_PRESENTATION_AUTHORIZATION_REASON; + const interactionOwnsProviderReply = createdByRunId + ? await hasChatRunOwnedProviderInteraction(dbOrTx, { + companyId: issue.companyId, + issueId, + runId: createdByRunId, + }) + : false; + const bindings = + interactionOwnsProviderReply || chatFinalOwnsProviderReply + ? [] + : await resolveChatOriginPublicationBindings( + dbOrTx, + issue.companyId, + issueId, + createdByRunId, + ); + const publicationCreatedAt = new Date(); + for (const binding of bindings) { + await measureRunPhase("response.publish", () => dbOrTx.transaction(async (tx: DbTransaction) => { + await notifyChatPublicationWork(tx); + return tx + .insert(chatPublications) + .values({ + companyId: binding.companyId, + endpointId: binding.endpointId, + conversationId: binding.conversationId, + issueId, + commentId: comment.id, + idempotencyKey: `comment:${comment.id}:${binding.endpointId}`, + payload: projectSafeChatPublication({ + classification: "external", + source: "agent_comment", + text: redactedBody, + }), + state: "pending", + createdAt: publicationCreatedAt, + updatedAt: publicationCreatedAt, + }) + .onConflictDoNothing(); + })); + for (const [index, attachment] of boundAttachments.entries()) { + const attachmentCreatedAt = new Date( + publicationCreatedAt.getTime() + index + 1, + ); + await measureRunPhase("response.publish", () => dbOrTx.transaction(async (tx: DbTransaction) => { + await notifyChatPublicationWork(tx); + return tx + .insert(chatPublications) + .values({ + companyId: binding.companyId, + endpointId: binding.endpointId, + conversationId: binding.conversationId, + issueId, + commentId: attachment.commentId, + idempotencyKey: `attachment:${attachment.id}:${binding.endpointId}`, + payload: projectSafeChatPublication({ + classification: "external", + source: "agent_comment", + text: `Shared ${attachment.originalFilename ?? "a file"}.`, + attachmentIds: [attachment.id], + }), + state: "pending", + createdAt: attachmentCreatedAt, + updatedAt: attachmentCreatedAt, + }) + .onConflictDoNothing(); + })); + } + } + } + + if ( + authorType === "user" && + actor.userId && + actor.userId !== "board-concierge" && + !createdByRunId + ) { + const { issueThreadInteractionService } = + await import("./issue-thread-interactions.js"); + const expiredInteractions = await issueThreadInteractionService( + dbOrTx, + ).expireRequestConfirmationsSupersededByComment( + { id: issueId, companyId: issue.companyId }, + comment, + { agentId: actor.agentId, userId: actor.userId }, + ); + for (const interaction of expiredInteractions) { + await logActivity(dbOrTx, { + companyId: issue.companyId, + actorType: "user", + actorId: actor.userId, + agentId: actor.agentId ?? null, + runId: createdByRunId, + action: "issue.thread_interaction_expired", + entityType: "issue", + entityId: issueId, + details: { + interactionId: interaction.id, + interactionKind: interaction.kind, + interactionStatus: interaction.status, + source: "issue.comment.service", + result: interaction.result ?? null, + }, + }); + } + } + + return redactIssueComment(comment, currentUserRedactionOptions.enabled); + }, + + createAttachment: async (input: { + issueId: string; + issueCommentId?: string | null; + provider: string; + objectKey: string; + contentType: string; + byteSize: number; + sha256: string; + originalFilename?: string | null; + createdByAgentId?: string | null; + createdByUserId?: string | null; + createdByRunId?: string | null; + }) => { + const issue = await db + .select({ id: issues.id, companyId: issues.companyId }) + .from(issues) + .where(eq(issues.id, input.issueId)) + .then((rows) => rows[0] ?? null); + if (!issue) throw notFound("Issue not found"); + + return db.transaction(async (tx) => { + if (input.createdByAgentId && input.issueCommentId) { + const [lockedIssue] = await tx + .select({ id: issues.id }) + .from(issues) + .where( + and( + eq(issues.id, issue.id), + eq(issues.companyId, issue.companyId), + ), + ) + .for("update"); + if (!lockedIssue) throw notFound("Issue not found"); + } + const registeredRun = + input.createdByRunId && isUuidLike(input.createdByRunId) + ? await tx + .select({ + id: heartbeatRuns.id, + status: heartbeatRuns.status, + }) + .from(heartbeatRuns) + .where( + and( + eq(heartbeatRuns.id, input.createdByRunId), + eq(heartbeatRuns.companyId, issue.companyId), + ...(input.createdByAgentId + ? [eq(heartbeatRuns.agentId, input.createdByAgentId)] + : []), + ), + ) + .for("update") + .then((rows) => rows[0] ?? null) + : null; + const registeredRunId = registeredRun?.id ?? null; + const parentComment = input.issueCommentId + ? await tx + .select({ + id: issueComments.id, + companyId: issueComments.companyId, + issueId: issueComments.issueId, + authorType: issueComments.authorType, + authorAgentId: issueComments.authorAgentId, + createdByRunId: issueComments.createdByRunId, + metadata: issueComments.metadata, + }) + .from(issueComments) + .where( + and( + eq(issueComments.id, input.issueCommentId), + isNull(issueComments.deletedAt), + ), + ) + .for("update") + .then((rows) => rows[0] ?? null) + : null; + if (input.issueCommentId && !parentComment) { + throw notFound("Issue comment not found"); + } + if ( + parentComment && + (parentComment.companyId !== issue.companyId || + parentComment.issueId !== issue.id) + ) { + throw unprocessable( + "Attachment comment must belong to same issue and company", + ); + } + if (input.createdByAgentId && parentComment) { + if ( + !registeredRunId || + parentComment.authorType !== "agent" || + parentComment.authorAgentId !== input.createdByAgentId || + parentComment.createdByRunId !== registeredRunId + ) { + throw unprocessable( + "Agent attachment comments must belong to the same agent and current run", + { code: "issue_attachment_parent_run_mismatch" }, + ); + } + const chatBindings = isExplicitExternalAgentComment( + parentComment.metadata, + ) + ? await resolveChatOriginPublicationBindings( + tx, + issue.companyId, + issue.id, + registeredRunId, + ) + : []; + if (chatBindings.length > 0) { + const alreadySelected = + await listSelectedChatPresentationAttachments(tx, { + companyId: issue.companyId, + issueId: issue.id, + agentId: input.createdByAgentId, + runId: registeredRunId, + }); + if (alreadySelected.length >= MAX_CHAT_PRESENTATION_ATTACHMENTS) { + throw unprocessable( + `An agent run can select at most ${MAX_CHAT_PRESENTATION_ATTACHMENTS} attachments for one chat response`, + { + code: "chat_attachment_selection_limit_exceeded", + limit: MAX_CHAT_PRESENTATION_ATTACHMENTS, + selectedCount: alreadySelected.length + 1, + }, + ); + } + } + } + await notifyNativeStatusReassessment(tx, { companyId: issue.companyId, issueId: issue.id }); + const [asset] = await tx + .insert(assets) + .values({ + companyId: issue.companyId, + provider: input.provider, + objectKey: input.objectKey, + contentType: input.contentType, + byteSize: input.byteSize, + sha256: input.sha256, + originalFilename: input.originalFilename ?? null, + createdByAgentId: input.createdByAgentId ?? null, + createdByUserId: input.createdByUserId ?? null, + }) + .returning(); + + const [attachment] = await tx + .insert(issueAttachments) + .values({ + companyId: issue.companyId, + issueId: issue.id, + assetId: asset.id, + issueCommentId: input.issueCommentId ?? null, + originatingRunId: registeredRunId, + }) + .returning(); + const contentPath = `/api/attachments/${attachment.id}/content`; + const [artifactWorkProduct] = registeredRunId + ? await tx + .insert(issueWorkProducts) + .values({ + companyId: issue.companyId, + issueId: issue.id, + type: "artifact", + provider: "paperclip", + externalId: attachment.id, + title: asset.originalFilename ?? "Attachment", + status: "active", + reviewState: "none", + isPrimary: false, + healthStatus: "unknown", + metadata: { + attachmentId: attachment.id, + contentType: asset.contentType, + byteSize: asset.byteSize, + contentPath, + openPath: contentPath, + downloadPath: `${contentPath}?download=1`, + originalFilename: asset.originalFilename, + }, + createdByRunId: registeredRunId, + }) + .returning({ id: issueWorkProducts.id }) + : []; + + if ( + input.createdByAgentId && + registeredRunId && + parentComment?.authorType === "agent" && + parentComment.authorAgentId === input.createdByAgentId && + parentComment.createdByRunId === registeredRunId && + parentComment.metadata?.authorizationReason === + CHAT_RUN_PRESENTATION_AUTHORIZATION_REASON + ) { + const bindings = await resolveChatOriginPublicationBindings( + tx, + issue.companyId, + issue.id, + registeredRunId, + ); + for (const binding of bindings) { + await notifyChatPublicationWork(tx); + await tx + .insert(chatPublications) + .values({ + companyId: issue.companyId, + endpointId: binding.endpointId, + conversationId: binding.conversationId, + issueId: issue.id, + commentId: parentComment.id, + idempotencyKey: `attachment:${attachment.id}:${binding.endpointId}`, + payload: projectSafeChatPublication({ + classification: "external", + source: "agent_comment", + text: `Shared ${asset.originalFilename ?? "a file"}.`, + attachmentIds: [attachment.id], + }), + state: "pending", + }) + .onConflictDoNothing(); + } + } + + return { + id: attachment.id, + companyId: attachment.companyId, + issueId: attachment.issueId, + issueCommentId: attachment.issueCommentId, + originatingRunId: attachment.originatingRunId, + assetId: attachment.assetId, + provider: asset.provider, + objectKey: asset.objectKey, + contentType: asset.contentType, + byteSize: asset.byteSize, + sha256: asset.sha256, + originalFilename: asset.originalFilename, + createdByAgentId: asset.createdByAgentId, + createdByUserId: asset.createdByUserId, + createdAt: attachment.createdAt, + updatedAt: attachment.updatedAt, + artifactWorkProductId: artifactWorkProduct?.id ?? null, + }; + }); + }, + + listAttachments: async (issueId: string) => + db + .select({ + id: issueAttachments.id, + companyId: issueAttachments.companyId, + issueId: issueAttachments.issueId, + issueCommentId: issueAttachments.issueCommentId, + originatingRunId: issueAttachments.originatingRunId, + assetId: issueAttachments.assetId, + provider: assets.provider, + objectKey: assets.objectKey, + contentType: assets.contentType, + byteSize: assets.byteSize, + sha256: assets.sha256, + originalFilename: assets.originalFilename, + createdByAgentId: assets.createdByAgentId, + createdByUserId: assets.createdByUserId, + createdAt: issueAttachments.createdAt, + updatedAt: issueAttachments.updatedAt, + }) + .from(issueAttachments) + .innerJoin(assets, eq(issueAttachments.assetId, assets.id)) + .where(eq(issueAttachments.issueId, issueId)) + .orderBy(desc(issueAttachments.createdAt)), + + getAttachmentById: async (id: string) => + db + .select({ + id: issueAttachments.id, + companyId: issueAttachments.companyId, + issueId: issueAttachments.issueId, + issueCommentId: issueAttachments.issueCommentId, + originatingRunId: issueAttachments.originatingRunId, + assetId: issueAttachments.assetId, + provider: assets.provider, + objectKey: assets.objectKey, + contentType: assets.contentType, + byteSize: assets.byteSize, + sha256: assets.sha256, + originalFilename: assets.originalFilename, + createdByAgentId: assets.createdByAgentId, + createdByUserId: assets.createdByUserId, + createdAt: issueAttachments.createdAt, + updatedAt: issueAttachments.updatedAt, + }) + .from(issueAttachments) + .innerJoin(assets, eq(issueAttachments.assetId, assets.id)) + .where(eq(issueAttachments.id, id)) + .then((rows) => rows[0] ?? null), + + removeAttachment: async (id: string) => + db.transaction(async (tx) => { + const existing = await tx + .select({ + id: issueAttachments.id, + companyId: issueAttachments.companyId, + issueId: issueAttachments.issueId, + issueCommentId: issueAttachments.issueCommentId, + originatingRunId: issueAttachments.originatingRunId, + assetId: issueAttachments.assetId, + provider: assets.provider, + objectKey: assets.objectKey, + contentType: assets.contentType, + byteSize: assets.byteSize, + sha256: assets.sha256, + originalFilename: assets.originalFilename, + createdByAgentId: assets.createdByAgentId, + createdByUserId: assets.createdByUserId, + createdAt: issueAttachments.createdAt, + updatedAt: issueAttachments.updatedAt, + }) + .from(issueAttachments) + .innerJoin(assets, eq(issueAttachments.assetId, assets.id)) + .where(eq(issueAttachments.id, id)) + .then((rows) => rows[0] ?? null); + if (!existing) return null; + + await tx.delete(issueAttachments).where(eq(issueAttachments.id, id)); + await tx.delete(assets).where(eq(assets.id, existing.assetId)); + return existing; + }), + + findMentionedAgents: async (companyId: string, body: string) => { + const explicitAgentMentionIds = extractAgentMentionIds(body); + if (explicitAgentMentionIds.length === 0) return []; + + const rows = await db + .select({ id: agents.id }) + .from(agents) + .where(eq(agents.companyId, companyId)); + const companyAgentIds = new Set(rows.map((agent) => agent.id)); + return explicitAgentMentionIds.filter((agentId) => + companyAgentIds.has(agentId), + ); + }, + + findMentionedProjectIds: async ( + issueId: string, + opts?: { includeCommentBodies?: boolean }, + ) => { + const issue = await db + .select({ + companyId: issues.companyId, + title: issues.title, + description: issues.description, + }) + .from(issues) + .where(eq(issues.id, issueId)) + .then((rows) => rows[0] ?? null); + if (!issue) return []; + + const mentionedIds = new Set(); + for (const source of [issue.title, issue.description ?? ""]) { + for (const projectId of extractProjectMentionIds(source)) { + mentionedIds.add(projectId); + } + } + + if (opts?.includeCommentBodies !== false) { + const comments = await db + .select({ body: issueComments.body }) + .from(issueComments) + .where( + and( + eq(issueComments.issueId, issueId), + isNull(issueComments.deletedAt), + ), + ); + + for (const comment of comments) { + for (const projectId of extractProjectMentionIds(comment.body)) { + mentionedIds.add(projectId); + } + } + } + + if (mentionedIds.size === 0) return []; + + const rows = await db + .select({ id: projects.id }) + .from(projects) + .where( + and( + eq(projects.companyId, issue.companyId), + inArray(projects.id, [...mentionedIds]), + ), + ); + const valid = new Set(rows.map((row) => row.id)); + return [...mentionedIds].filter((projectId) => valid.has(projectId)); + }, + + getAncestors: async (issueId: string) => { + const raw: Array<{ + id: string; + identifier: string | null; + title: string; + description: string | null; + status: string; + priority: string; + assigneeAgentId: string | null; + projectId: string | null; + goalId: string | null; + }> = []; + const visited = new Set([issueId]); + const start = await db + .select() + .from(issues) + .where(eq(issues.id, issueId)) + .then((r) => r[0] ?? null); + let currentId = start?.parentId ?? null; + while (currentId && !visited.has(currentId) && raw.length < 50) { + visited.add(currentId); + const parent = await db + .select({ + id: issues.id, + identifier: issues.identifier, + title: issues.title, + description: issues.description, + status: issues.status, + priority: issues.priority, + assigneeAgentId: issues.assigneeAgentId, + projectId: issues.projectId, + goalId: issues.goalId, + parentId: issues.parentId, + }) + .from(issues) + .where(eq(issues.id, currentId)) + .then((r) => r[0] ?? null); + if (!parent) break; + raw.push({ + id: parent.id, + identifier: parent.identifier ?? null, + title: parent.title, + description: parent.description ?? null, + status: parent.status, + priority: parent.priority, + assigneeAgentId: parent.assigneeAgentId ?? null, + projectId: parent.projectId ?? null, + goalId: parent.goalId ?? null, + }); + currentId = parent.parentId ?? null; + } + + // Batch-fetch referenced projects and goals + const projectIds = [ + ...new Set( + raw.map((a) => a.projectId).filter((id): id is string => id != null), + ), + ]; + const goalIds = [ + ...new Set( + raw.map((a) => a.goalId).filter((id): id is string => id != null), + ), + ]; + + const projectMap = new Map< + string, + { + id: string; + name: string; + description: string | null; + status: string; + goalId: string | null; + workspaces: Array<{ + id: string; + companyId: string; + projectId: string; + name: string; + cwd: string | null; + repoUrl: string | null; + repoRef: string | null; + metadata: Record | null; + isPrimary: boolean; + createdAt: Date; + updatedAt: Date; + }>; + primaryWorkspace: { + id: string; + companyId: string; + projectId: string; + name: string; + cwd: string | null; + repoUrl: string | null; + repoRef: string | null; + metadata: Record | null; + isPrimary: boolean; + createdAt: Date; + updatedAt: Date; + } | null; + } + >(); + const goalMap = new Map< + string, + { + id: string; + title: string; + description: string | null; + level: string; + status: string; + } + >(); + + if (projectIds.length > 0) { + const workspaceRows = await db + .select() + .from(projectWorkspaces) + .where(inArray(projectWorkspaces.projectId, projectIds)) + .orderBy( + desc(projectWorkspaces.isPrimary), + asc(projectWorkspaces.createdAt), + asc(projectWorkspaces.id), + ); + const workspaceMap = new Map< + string, + Array<(typeof workspaceRows)[number]> + >(); + for (const workspace of workspaceRows) { + const existing = workspaceMap.get(workspace.projectId); + if (existing) existing.push(workspace); + else workspaceMap.set(workspace.projectId, [workspace]); + } + + const rows = await db + .select({ + id: projects.id, + name: projects.name, + description: projects.description, + status: projects.status, + goalId: projects.goalId, + }) + .from(projects) + .where(inArray(projects.id, projectIds)); + for (const r of rows) { + const projectWorkspaceRows = workspaceMap.get(r.id) ?? []; + const workspaces = projectWorkspaceRows.map((workspace) => ({ + id: workspace.id, + companyId: workspace.companyId, + projectId: workspace.projectId, + name: workspace.name, + cwd: workspace.cwd, + repoUrl: workspace.repoUrl ?? null, + repoRef: workspace.repoRef ?? null, + metadata: + (workspace.metadata as Record | null) ?? null, + isPrimary: workspace.isPrimary, + createdAt: workspace.createdAt, + updatedAt: workspace.updatedAt, + })); + const primaryWorkspace = + workspaces.find((workspace) => workspace.isPrimary) ?? + workspaces[0] ?? + null; + projectMap.set(r.id, { + ...r, + workspaces, + primaryWorkspace, + }); + // Also collect goalIds from projects + if (r.goalId && !goalIds.includes(r.goalId)) goalIds.push(r.goalId); + } + } + + if (goalIds.length > 0) { + const rows = await db + .select({ + id: goals.id, + title: goals.title, + description: goals.description, + level: goals.level, + status: goals.status, + }) + .from(goals) + .where(inArray(goals.id, goalIds)); + for (const r of rows) goalMap.set(r.id, r); + } + + return raw.map((a) => ({ + ...a, + project: a.projectId ? (projectMap.get(a.projectId) ?? null) : null, + goal: a.goalId ? (goalMap.get(a.goalId) ?? null) : null, + })); + }, + }; + + type IssueServiceApi = typeof service & { + updateForCompany: ( + id: string, + companyId: string, + data: Parameters[1], + dbOrTx?: any, + postCommitActivityPublications?: ActivityPublication[], + postCommitActions?: IssuePostCommitAction[], + ) => ReturnType; + }; + const serviceApi = service as IssueServiceApi; + + // A company-scoped wrapper around `update`. It passes the company as a + // guard on every read, lock, and write predicate, so a caller with only + // a company id and an issue id cannot update an issue in another company. + serviceApi.updateForCompany = async ( + id, + companyId, + data, + dbOrTx = db, + postCommitActivityPublications, + postCommitActions, + ) => { + return service.update( + id, + { ...data, companyGuard: companyId }, + dbOrTx, + postCommitActivityPublications, + postCommitActions, + ); + }; + + return serviceApi; +} From a5d31c7f1fc61765a8514eab0115a1af09316dac Mon Sep 17 00:00:00 2001 From: Dotta Date: Fri, 30 Oct 2026 18:21:05 -0500 Subject: [PATCH 47/74] fix(github): retain transaction ownership and scoped credential debt Preserve webhook metadata across OAuth rotation and lease cleanup; scope and coalesce GitHub work activation, retaining explicit producer coverage. Co-Authored-By: Paperclip --- .../2026-10-30-github-connection-sleep.md | 15 +- .../__tests__/github-catalog-timing.test.ts | 14 +- .../github-connection-events.test.ts | 16 +- .../__tests__/github-connection-work.test.ts | 144 +++++++++++++++++- server/src/__tests__/openapi-routes.test.ts | 4 + .../server-startup-feedback-export.test.ts | 5 +- .../configuration-work-notifications.ts | 6 +- .../services/chat-channels/configuration.ts | 8 + server/src/services/chat-github-wizard.ts | 5 +- server/src/services/companies.ts | 7 +- .../src/services/github-connection-events.ts | 16 +- .../github-connection-notifications.ts | 27 +++- server/src/services/github-connection-work.ts | 31 ++-- .../src/services/issue-thread-interactions.ts | 8 +- server/src/services/tool-access.ts | 10 +- .../tool-access/connections/removal.ts | 2 + .../tool-access/github-grant-continuity.ts | 23 ++- .../tool-access/oauth/access-finalization.ts | 2 + .../tool-access/oauth/cloud-callback.ts | 2 + .../tool-access/oauth/grant-refresh.ts | 29 +--- .../tool-access/oauth/refresh-leases.ts | 19 +-- 21 files changed, 307 insertions(+), 86 deletions(-) diff --git a/doc/plans/2026-10-30-github-connection-sleep.md b/doc/plans/2026-10-30-github-connection-sleep.md index 11271110a7a..40f83708a71 100644 --- a/doc/plans/2026-10-30-github-connection-sleep.md +++ b/doc/plans/2026-10-30-github-connection-sleep.md @@ -14,7 +14,11 @@ events, validates their bindings, persists company-scoped receipts, applies the existing installation or pull-request behavior, and acknowledges completed work. A receipt row lock prevents overlapping receivers from applying an installation delta twice. Completed receipts remain passive history. The existing normalized -merged-pull-request callback remains unchanged. +merged-pull-request callback keeps its normalized key. Snapshot and confirmation +reads/writes share the receipt transaction, including at pool size one; existing +confirmation publications and direct wakes flush only after the outer commit. +The pre-existing non-chat direct-wake crash gap remains: chat-bound continuations +have their existing durable outbox, while an unbound direct wake is best effort. A new tenant sends signed event-lease claim `ewv=1`. A compatible Cloud broker records delivery version 1 on the existing instance registration before returning @@ -51,6 +55,15 @@ Changed credential metadata invalidates the old retry revision, so an earlier ne expiry can preempt it. Disabled connections and archived companies do not perform credential work. Reactivation uses the normal committed configuration signal. +GitHub owners receive only relevant credential/configuration signals, not generic +health or receiver notifications. Continuity reconstruction is company-scoped +after the startup scan. Event draining retains instance-wide binding matching +because Cloud leases an instance queue. Event hints coalesce with a five-second +minimum pass spacing; an unrelated connection PATCH cannot trigger that drain. +OAuth rotation merges only its owned OAuth fields, lease acquisition compares +current metadata, and cleanup atomically removes only the matching lease path, +preserving concurrent GitHub installation changes and binding-removal debt. + Supported local producers are the owning connection, OAuth, grant and company services in one tenant process. Their notifications belong to the outer database transaction. A savepoint does not commit a wake, and rollback cancels the write. diff --git a/server/src/__tests__/github-catalog-timing.test.ts b/server/src/__tests__/github-catalog-timing.test.ts index 4ee15c6db85..5d0c4835d84 100644 --- a/server/src/__tests__/github-catalog-timing.test.ts +++ b/server/src/__tests__/github-catalog-timing.test.ts @@ -1,9 +1,10 @@ +import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js"; import { randomUUID } from "node:crypto"; import { context, trace } from "@opentelemetry/api"; import { AsyncLocalStorageContextManager } from "@opentelemetry/context-async-hooks"; import { InMemorySpanExporter, SimpleSpanProcessor, TracerProvider } from "@opentelemetry/sdk-trace"; import { - authUsers, companies, companyMemberships, connectionGrants, createDb, + applyPendingMigrations, authUsers, companies, companySecrets, companyMemberships, connectionGrants, createDb, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS, toolApplications, toolConnections, } from "@paperclipai/db"; import { eq, sql } from "drizzle-orm"; @@ -15,15 +16,18 @@ import type { PaperclipCloudConnector } from "../services/paperclip-cloud-connec import { toolAccessService } from "../services/tool-access.js"; import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; -const support = await getEmbeddedPostgresTestSupport(); +const databaseUrl = resolveExternalTestDatabaseUrl("PAPERCLIP_GITHUB_CATALOG_TEST_DATABASE_URL"); +const support = databaseUrl ? { supported: true } : await getEmbeddedPostgresTestSupport(); (support.supported ? describe : describe.skip)("GitHub repository catalog execution timing", () => { let temporary: Awaited>; let db: ReturnType; + const ownedCompanies: string[] = []; let exporter: InMemorySpanExporter, provider: TracerProvider, manager: AsyncLocalStorageContextManager; beforeAll(async () => { - temporary = await startEmbeddedPostgresTestDatabase("paperclip-github-catalog-timing-"); - db = createDb(temporary.connectionString, { maxConnections: 3 }); + if (databaseUrl) await applyPendingMigrations(databaseUrl); + else temporary = await startEmbeddedPostgresTestDatabase("paperclip-github-catalog-timing-"); + db = createDb(databaseUrl ?? temporary.connectionString, { maxConnections: 3 }); }, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS); beforeEach(() => { vi.stubEnv("PAPERCLIP_SECRETS_MASTER_KEY", "01234567890123456789012345678901"); @@ -38,6 +42,7 @@ const support = await getEmbeddedPostgresTestSupport(); vi.restoreAllMocks(); vi.unstubAllEnvs(); trace.disable(); context.disable(); manager.disable(); await provider.shutdown(); + for (const id of ownedCompanies.splice(0)) { await db.delete(companyMemberships).where(eq(companyMemberships.companyId, id)); await db.delete(companySecrets).where(eq(companySecrets.companyId, id)); await db.delete(companies).where(eq(companies.id, id)); } }); afterAll(async () => { await db?.$client.end({ timeout: 0 }); @@ -47,6 +52,7 @@ const support = await getEmbeddedPostgresTestSupport(); async function fixture() { const userId = `catalog-user-${randomUUID()}`; const [company] = await db.insert(companies).values({ name: "Catalog timing", issuePrefix: `GH${randomUUID().slice(0, 6)}` }).returning(); + ownedCompanies.push(company.id); await db.insert(authUsers).values({ id: userId, name: "Catalog owner", email: `${userId}@example.test`, createdAt: new Date(), updatedAt: new Date() }); await db.insert(companyMemberships).values({ companyId: company.id, principalType: "user", principalId: userId, status: "active", membershipRole: "owner" }); const [application] = await db.insert(toolApplications).values({ companyId: company.id, name: "GitHub catalog", type: "mcp_http" }).returning(); diff --git a/server/src/__tests__/github-connection-events.test.ts b/server/src/__tests__/github-connection-events.test.ts index 52cc835a444..aa6b1545078 100644 --- a/server/src/__tests__/github-connection-events.test.ts +++ b/server/src/__tests__/github-connection-events.test.ts @@ -10,6 +10,8 @@ import { connectionGrants, createDb, externalObjects, + issues, + issueThreadInteractions, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS, toolApplications, toolConnections, @@ -35,12 +37,14 @@ describeEmbeddedPostgres("GitHub connection event delivery", () => { beforeAll(async () => { if (!externalTestDatabaseUrl) tempDb = await startEmbeddedPostgresTestDatabase("paperclip-github-events-"); if (externalTestDatabaseUrl) await applyPendingMigrations(externalTestDatabaseUrl); - db = createDb(externalTestDatabaseUrl ?? tempDb!.connectionString, { maxConnections: 4 }); + db = createDb(externalTestDatabaseUrl ?? tempDb!.connectionString, { maxConnections: 1 }); }, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS); afterEach(async () => { await db.delete(activityLog); await db.delete(connectionEventDeliveries); + await db.delete(issueThreadInteractions); + await db.delete(issues); await db.delete(externalObjects); await db.delete(connectionGrants); await db.delete(toolConnections); @@ -105,6 +109,10 @@ describeEmbeddedPostgres("GitHub connection event delivery", () => { }, }, }); + const [issue] = await db.insert(issues).values({ companyId, title: "Merge release", status: "in_progress" }).returning(); + const [confirmation] = await db.insert(issueThreadInteractions).values({ companyId, issueId: issue!.id, + kind: "request_confirmation", title: "Confirm pull request merged", continuationPolicy: "none", + payload: { version: 1, prompt: "Confirm https://github.com/paperclipai/paperclip/pull/123 is merged" } }).returning(); const externalObjectId = randomUUID(); await db.insert(externalObjects).values({ id: externalObjectId, @@ -172,11 +180,15 @@ describeEmbeddedPostgres("GitHub connection event delivery", () => { } const warning = vi.spyOn(logger, "warn").mockImplementation(() => undefined); - await expect(service.pollOnce({ forSetup: activeSetup })).resolves.toMatchObject({ leased: 2, processed: 1, duplicate: 0, failed: 1 }); + const outcome = await service.pollOnce({ forSetup: activeSetup }); + const [processingReceipt] = await db.select().from(connectionEventDeliveries).where(eq(connectionEventDeliveries.providerDeliveryId, leasedEvent.id)); + expect(processingReceipt?.lastError).toBeNull(); + expect(outcome).toMatchObject({ leased: 2, processed: 1, duplicate: 0, failed: 1 }); expect(warning.mock.calls).toEqual([[{ deliveryId: "undecryptable-delivery", reason: "inner_event_authentication_failed", }, "GitHub connector event retained for retry"]]); warning.mockRestore(); + expect((await db.select().from(issueThreadInteractions).where(eq(issueThreadInteractions.id, confirmation!.id)))[0]!.status).toBe("accepted"); const [snapshot] = await db.select().from(externalObjects).where(eq(externalObjects.id, externalObjectId)); expect(snapshot).toMatchObject({ statusKey: "merged", diff --git a/server/src/__tests__/github-connection-work.test.ts b/server/src/__tests__/github-connection-work.test.ts index f92cefd5ac1..a079c130f02 100644 --- a/server/src/__tests__/github-connection-work.test.ts +++ b/server/src/__tests__/github-connection-work.test.ts @@ -1,10 +1,12 @@ +import { notifyChatEndpointConfiguration } from "../services/chat-channels/configuration-work-notifications.js"; +import { toolAccessService } from "../services/tool-access.js"; import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; -import { randomUUID, generateKeyPairSync, verify, createHash } from "node:crypto"; +import { randomUUID, generateKeyPairSync, verify, createHash, diffieHellman, hkdfSync, randomBytes, createCipheriv } from "node:crypto"; import { createServer, type Server } from "node:http"; import { once } from "node:events"; import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest"; -import { eq } from "drizzle-orm"; -import { applyPendingMigrations, companies, connectionGrants, createDb, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS, toolApplications, toolConnections } from "@paperclipai/db"; +import { eq, sql } from "drizzle-orm"; +import { applyPendingMigrations, companies, activityLog, companySecrets, connectionGrants, createDb, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS, toolApplications, toolConnections } from "@paperclipai/db"; import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js"; import { createGitHubGrantContinuity, githubGrantContinuityDeadline } from "../services/tool-access/github-grant-continuity.js"; import { createPaperclipCloudConnector } from "../services/paperclip-cloud-connector.js"; @@ -12,6 +14,13 @@ import { githubConnectionEventService } from "../services/github-connection-even import { createDeliveryWorkCoordinator } from "../services/delivery-work-coordinator.js"; import { registerGitHubConnectionContinuityWork, registerGitHubConnectionEventWork } from "../services/github-connection-work.js"; import { notifyGitHubCredentialChange } from "../services/github-connection-notifications.js"; +import { createOAuthCredentialStore } from "../services/tool-access/oauth/credential-store.js"; +import { createOAuthRefreshLeases } from "../services/tool-access/oauth/refresh-leases.js"; +import { createOAuthGrantRefresh } from "../services/tool-access/oauth/grant-refresh.js"; +import { secretService } from "../services/secrets.js"; +import { writeConnectionCredential } from "../services/connection-credentials.js"; +import { GITHUB_CONNECTOR_PROFILES } from "@paperclipai/shared"; +import { notifyToolConnectionHealth } from "../services/tool-connection-health-notifications.js"; import { idleWorkSnapshot } from "../services/task-admission.js"; const databaseUrl = resolveExternalTestDatabaseUrl("PAPERCLIP_GITHUB_WORK_TEST_DATABASE_URL"); @@ -24,12 +33,16 @@ describeDatabase("GitHub coordinator with PostgreSQL and controlled Cloud HTTP ( let server: Server, origin: string; let bindings: Array>, leases: number, failBindings: boolean, githubRequests: number; const signing = generateKeyPairSync("ed25519"), sealing = generateKeyPairSync("x25519"); + let releaseRefresh: (() => void) | undefined, refreshArrived: (() => void) | undefined; + let refreshGate: Promise | undefined; const realFetch = globalThis.fetch; let companyIds: string[] = []; beforeAll(async () => { if (databaseUrl) await applyPendingMigrations(databaseUrl); else temporaryDatabase = await startEmbeddedPostgresTestDatabase("paperclip-github-work-"); db = createDb(databaseUrl ?? temporaryDatabase!.connectionString, { maxConnections: 4 }); + // This explicitly assigned external test database is exclusive to this suite. + await db.transaction(async tx => { await tx.execute(sql`SET LOCAL client_min_messages = warning`); await tx.execute(sql`TRUNCATE companies CASCADE`); }); server = createServer(async (req, res) => { const chunks: Buffer[] = []; for await (const chunk of req) chunks.push(Buffer.from(chunk)); const raw = Buffer.concat(chunks).toString(); @@ -39,6 +52,23 @@ describeDatabase("GitHub coordinator with PostgreSQL and controlled Cloud HTTP ( const claims = JSON.parse(Buffer.from(payload, "base64url").toString()); expect(claims.iss).toBe("instance"); expect(claims.env).toBe("test"); expect(claims.aud).toBe(`${origin}${req.url}`); + if (req.url!.endsWith("/refresh")) { + refreshArrived?.(); await refreshGate; + const scopes = [...GITHUB_CONNECTOR_PROFILES["github.code"].scopes]; + const algorithm = "X25519-HKDF-SHA256-A256GCM"; + const purpose = "access"; + const aad = Buffer.from([1, algorithm, purpose, "instance", "test", "github", "github.code", scopes.sort().join(" ")].join("\n")); + const ephemeral = generateKeyPairSync("x25519"); + const epk = Buffer.from(ephemeral.publicKey.export({ format: "jwk" }).x!, "base64url"); + const recipient = Buffer.from(sealing.publicKey.export({ format: "jwk" }).x!, "base64url"); + const key = Buffer.from(hkdfSync("sha256", diffieHellman({ privateKey: ephemeral.privateKey, publicKey: sealing.publicKey }), Buffer.concat([epk, recipient]), aad, 32)); + const iv = randomBytes(12), cipher = createCipheriv("aes-256-gcm", key, iv); cipher.setAAD(aad); + const credentials = { v: 1, accessToken: "rotated-access", refreshToken: "rotated-refresh", tokenType: "Bearer", scopes, + accessTokenExpiresAt: new Date(Date.now() + 8 * 3600_000).toISOString(), refreshTokenExpiresAt: null, + subject: claims.sub, companyId: claims.cid, instanceId: "instance", environment: "test", provider: "github", profile: "github.code" }; + const ct = Buffer.concat([cipher.update(JSON.stringify(credentials)), cipher.final(), cipher.getAuthTag()]); + res.setHeader("content-type", "application/json"); res.end(JSON.stringify({ sealed: { v: 1, alg: algorithm, purpose, provider: "github", profile: "github.code", epk: epk.toString("base64url"), iv: iv.toString("base64url"), ct: ct.toString("base64url") } })); return; + } if (req.url!.endsWith("/events/lease")) { leases++; res.setHeader("content-type", "application/json"); res.end(JSON.stringify({ events: [], ...(claims.ewv === 1 ? { eventWakeVersion: 1 } : {}) })); return; @@ -59,7 +89,7 @@ describeDatabase("GitHub coordinator with PostgreSQL and controlled Cloud HTTP ( }, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS); afterEach(async () => { vi.restoreAllMocks(); vi.useRealTimers(); - for (const id of companyIds) await db.delete(companies).where(eq(companies.id, id)); companyIds = []; + for (const id of companyIds) { await db.delete(activityLog).where(eq(activityLog.companyId, id)); await db.delete(companySecrets).where(eq(companySecrets.companyId, id)); await db.delete(companies).where(eq(companies.id, id)); } companyIds = []; }); afterAll(async () => { server.close(); await once(server, "close"); await db.$client.end({ timeout: 1 }); await temporaryDatabase?.cleanup(); }); @@ -106,6 +136,7 @@ describeDatabase("GitHub coordinator with PostgreSQL and controlled Cloud HTTP ( await Promise.all([ew.ready, cw.ready]); expect(leases).toBe(1); const startupLeases = leases; await f.service.refreshManagedGitHubGrantAccess(f.connection, f.grant, { actorType: "system", actorId: "test" }); + await vi.advanceTimersByTimeAsync(6_000); await vi.waitFor(() => expect(leases).toBeGreaterThan(startupLeases)); await vi.advanceTimersByTimeAsync(31_000); await settled(coordinator); const [saved] = await db.select().from(connectionGrants).where(eq(connectionGrants.id, f.grant.id)); @@ -148,6 +179,111 @@ describeDatabase("GitHub coordinator with PostgreSQL and controlled Cloud HTTP ( expect(await restarted.nextGitHubConnectionContinuityAt()).toBeNull(); }); + it("preserves webhook removal debt across an actual signed, held Cloud token rotation and lease cleanup", async () => { + const f = await fixture(["101"]); + vi.stubEnv("PAPERCLIP_SECRETS_MASTER_KEY", "01234567890123456789012345678901"); + const refs = []; + for (const path of ["oauth.access_token", "oauth.refresh_token"]) { + const saved = await writeConnectionCredential(db, { companyId: f.company.id, connectionName: "GitHub", configPath: path, label: path, value: path, ownerUserId: "owner" }); + refs.push({ configPath: path, secretId: saved.secret.id, versionSelector: "latest" as const }); + } + const [connection] = await db.update(toolConnections).set({ credentialSource: "paperclip_vault", config: { ...f.connection.config, + oauth: { strategy: "paperclip_cloud_connector", connectorProfile: "github.code" } } }).where(eq(toolConnections.id, f.connection.id)).returning(); + const [grant] = await db.update(connectionGrants).set({ credentialSecretRefs: refs }).where(eq(connectionGrants.id, f.grant.id)).returning(); + const getConnection = async () => (await db.select().from(toolConnections).where(eq(toolConnections.id, connection!.id)))[0]!; + const store = createOAuthCredentialStore(db, secretService(db)); + await store.syncCredentialBindings(connection!, refs); + const leases = createOAuthRefreshLeases(db, () => new Date(), getConnection, store.getOAuthGrantRow); + const rotation = createOAuthGrantRefresh(db, { now: () => new Date(), store, leases, getConnectionRow: getConnection, + currentCloudConnector: () => f.connector, exchangeOAuthToken: vi.fn() }); + const arrived = new Promise(resolve => { refreshArrived = resolve; }); + refreshGate = new Promise(resolve => { releaseRefresh = resolve; }); + const pending = rotation({ companyId: f.company.id, connectionId: connection!.id, grantId: grant!.id, forceRefresh: true }); + const result = Promise.allSettled([pending]); + try { + await Promise.race([arrived, pending.then(() => { throw new Error("Rotation returned before the Cloud HTTP boundary"); })]); + const connector = { ...f.connector, acknowledgeEvents: async () => 1, leaseEvents: async () => ({ leaseId: "removal", events: [{ id: randomUUID(), provider: "github" as const, + event: "installation", action: "deleted", installationId: "101", repositoryId: null, createdAt: new Date(Date.now() + 1000).toISOString(), bindingIds: [`${grant!.id}_101`], payload: { installationId: "101" } }] }) }; + expect(await githubConnectionEventService(db, { connector }).pollOnce()).toMatchObject({ processed: 1 }); + const [before] = await db.select().from(connectionGrants).where(eq(connectionGrants.id, grant!.id)); + expect(before!.providerTenant!.github!.bindingSync!.removeInstallationIds).toEqual(["101"]); + releaseRefresh!(); + const [completion] = await result; if (completion.status === "rejected") throw completion.reason; + const [saved] = await db.select().from(connectionGrants).where(eq(connectionGrants.id, grant!.id)); + expect(saved!.providerTenant!.github).toEqual(before!.providerTenant!.github); + expect(saved!.providerTenant!.oauth).toMatchObject({ tokenType: "Bearer" }); + expect(saved!.providerTenant!.oauth!.refreshLease).toBeUndefined(); + await f.service.syncGitHubGrantBindings(connection!, saved!); + expect(bindings).toContainEqual(expect.objectContaining({ installationId: "101", active: false })); + expect((await db.select().from(connectionGrants).where(eq(connectionGrants.id, grant!.id)))[0]!.providerTenant!.github!.bindingSync).toBeUndefined(); + } finally { releaseRefresh?.(); await result; refreshGate = undefined; refreshArrived = undefined; vi.unstubAllEnvs(); } + }); + + it("lease acquire and cleanup cannot replace a concurrent webhook metadata revision", async () => { + const f = await fixture(["101"]); + let interleave = true; + const removal = { id: randomUUID(), removeInstallationIds: ["202"] }; + const getGrant = async () => { + const [snapshot] = await db.select().from(connectionGrants).where(eq(connectionGrants.id, f.grant.id)); + if (interleave) { + interleave = false; + await db.update(connectionGrants).set({ providerTenant: sql`jsonb_set(${connectionGrants.providerTenant}, '{github,bindingSync}', ${JSON.stringify(removal)}::jsonb)` }).where(eq(connectionGrants.id, f.grant.id)); + } + return snapshot!; + }; + const leaseOwner = createOAuthRefreshLeases(db, () => new Date(), async () => f.connection, getGrant); + const lease = await leaseOwner.acquireOAuthGrantRefreshLease(f.connection, f.grant, true); + expect(lease.leaseId).toBeTruthy(); + expect(lease.grant.providerTenant!.github!.bindingSync).toEqual(removal); + // Cleanup deliberately reads a stale snapshot immediately before a new delta. + const second = { ...removal, id: randomUUID(), removeInstallationIds: ["202", "303"] }; + const cleanup = createOAuthRefreshLeases(db, () => new Date(), async () => f.connection, async () => { + const snapshot = await getGrant(); + await db.update(connectionGrants).set({ providerTenant: sql`jsonb_set(${connectionGrants.providerTenant}, '{github,bindingSync}', ${JSON.stringify(second)}::jsonb)` }).where(eq(connectionGrants.id, f.grant.id)); + return snapshot; + }); + const cleared = await cleanup.clearOAuthGrantRefreshLease(lease.grant, lease.leaseId!); + expect(cleared.providerTenant!.github!.bindingSync).toEqual(second); + expect(cleared.providerTenant!.oauth!.refreshLease).toBeUndefined(); + }); + + it("ignores unrelated health write bursts and scopes credential reconstruction to its owning company", async () => { + const f = await fixture(["101"]), other = await fixture(["202"]); fakeClock(); + // A large unrelated company set must not enter a targeted credential reload. + for (let index = 0; index < 50; index++) { + const id = randomUUID(); companyIds.push(id); + await db.insert(companies).values({ id, name: "Unrelated credential owner", issuePrefix: `U${index}${id.slice(0, 6)}` }); + } + const [unrelatedConnection] = await db.update(toolConnections).set({ config: {}, transportConfig: {}, transport: "rest_api", authKind: "none", credentialRefs: [], credentialSecretRefs: [] }).where(eq(toolConnections.id, other.connection.id)).returning(); + await db.update(connectionGrants).set({ credentialSecretRefs: [] }).where(eq(connectionGrants.id, other.grant.id)); + const api = toolAccessService(db); + const load = vi.spyOn(f.service, "githubConnectionContinuitySchedule"); + const sweep = vi.spyOn(f.service, "sweepGitHubConnectionContinuity"); + const coordinator = createDeliveryWorkCoordinator({ owner: db, canRun: () => true }); + const continuity = registerGitHubConnectionContinuityWork(db, coordinator, f.service, () => true); + const events = registerGitHubConnectionEventWork(db, coordinator, githubConnectionEventService(db, { connector: f.connector }), () => true); + try { + await Promise.all([continuity.ready, events.ready]); + await vi.waitFor(() => expect(leases).toBe(1)); + await vi.waitFor(() => expect(load).toHaveBeenCalledTimes(1)); + load.mockClear(); sweep.mockClear(); + for (let i = 0; i < 30; i++) { + await db.transaction(async tx => { await notifyToolConnectionHealth(tx, other.connection); await notifyChatEndpointConfiguration(tx, { id: randomUUID(), provider: "discord" }); }); + await api.updateConnection(unrelatedConnection!.id, { name: `Unrelated patch ${i}` }); + } + await vi.advanceTimersByTimeAsync(10_000); + expect(load).not.toHaveBeenCalled(); expect(sweep).not.toHaveBeenCalled(); expect(leases).toBe(1); + for (let index = 0; index < 30; index++) await db.transaction(async tx => { await notifyGitHubCredentialChange(tx, f.connection); }); + expect(leases).toBe(1); + await vi.advanceTimersByTimeAsync(6_000); + await vi.waitFor(() => expect(load).toHaveBeenCalledWith(f.company.id)); + expect(load.mock.calls.every(([companyId]) => companyId === f.company.id)).toBe(true); + expect(sweep).not.toHaveBeenCalled(); + await vi.waitFor(() => expect(leases).toBe(2)); + expect(load).toHaveBeenCalledTimes(1); + } finally { await events.stop(); await continuity.stop(); await coordinator.stop(); } + }); + it("settles only the company and grant that own saved binding debt", async () => { const f = await fixture(["101"]), other = await fixture(["202"]); await db.update(connectionGrants).set({ providerTenant: { ...f.grant.providerTenant, github: { ...f.grant.providerTenant!.github!, diff --git a/server/src/__tests__/openapi-routes.test.ts b/server/src/__tests__/openapi-routes.test.ts index 0a1a220b8fd..e763cc85608 100644 --- a/server/src/__tests__/openapi-routes.test.ts +++ b/server/src/__tests__/openapi-routes.test.ts @@ -47,6 +47,7 @@ const apiPrefixes: Record = { "company-skills.ts": "/api", "company-skill-policy.ts": "/api", "connection-intents.ts": "/api", + "connection-event-wake.ts": "/", "costs.ts": "/api", "dashboard.ts": "/api", "dot-runner.ts": "/api", @@ -113,6 +114,8 @@ const HTTP_METHODS = new Set([ const explicitOpenApiCoverageExclusions = new Set(); const explicitOpenApiOperationCoverageExclusions = new Set([ + // Exact Cloud-control signed action; ordinary board/agent credentials cannot invoke it. + "POST /api/instance/connector-events/wake", // Muse uses its closed, versioned receiver protocol and dedicated credentials, // documented in doc/muse-personal-agent.md and the shared protocol contract. // Board connection/qualification management remains in this OpenAPI document. @@ -180,6 +183,7 @@ function normalizeExpressPath(routePath: string) { } function resolveMountedPath(file: string, prefix: string, routePath: string) { + if (file === "connection-event-wake.ts") return routePath; if (file === "public-mcp.ts" && (routePath.startsWith("/mcp/oauth/") || routePath.startsWith("/.well-known/"))) return routePath; if ( (file === "chat-channels.ts" || file === "email.ts") && diff --git a/server/src/__tests__/server-startup-feedback-export.test.ts b/server/src/__tests__/server-startup-feedback-export.test.ts index d73175ee556..29ff63b985b 100644 --- a/server/src/__tests__/server-startup-feedback-export.test.ts +++ b/server/src/__tests__/server-startup-feedback-export.test.ts @@ -492,6 +492,7 @@ vi.mock("../services/index.js", () => ({ failed: 0, })), nextGitHubConnectionContinuityAt: vi.fn(async () => null), + githubConnectionContinuitySchedule: vi.fn(async () => []), sweepGitHubConnectionContinuity: vi.fn(async () => ({ checked: 0, due: 0, @@ -1303,7 +1304,9 @@ describe("startServer feedback export wiring", () => { } const polls = vi.mocked(githubConnectionEventService).mock.results.map(result => result.value.pollOnce); const sweeps = vi.mocked(toolAccessService).mock.results.map(result => result.value.sweepGitHubConnectionContinuity); - const count = () => [...polls, ...sweeps].reduce((sum, call) => sum + call.mock.calls.length, 0); + const schedules = vi.mocked(toolAccessService).mock.results.map(result => result.value.githubConnectionContinuitySchedule); + expect(sweeps.reduce((sum, call) => sum + call.mock.calls.length, 0)).toBe(0); + const count = () => [...polls, ...sweeps, ...schedules].reduce((sum, call) => sum + call.mock.calls.length, 0); const startup = count(); expect(startup).toBe(2); for (let i = 0; i < 3; i++) for (const tick of ticks) tick(); diff --git a/server/src/services/chat-channels/configuration-work-notifications.ts b/server/src/services/chat-channels/configuration-work-notifications.ts index 9d3240aee12..8a301b7cd0f 100644 --- a/server/src/services/chat-channels/configuration-work-notifications.ts +++ b/server/src/services/chat-channels/configuration-work-notifications.ts @@ -1,3 +1,4 @@ +import { DELIVERY_QUEUES, notifyDeliveryWork } from "../delivery-work-notifications.js"; import { notifyGitHubWebhookRecovery } from "./github/webhook-recovery-notifications.js"; import { notifyVoiceChange } from "../voice/voice-recovery-notifications.js"; import { notifyChatReceiver, notifyChatConnectionReceiver, notifyChatCompanyReceivers } from "./runtime/receiver-work-notifications.js"; @@ -7,7 +8,10 @@ import { notifyDiscordCommandRegistration } from "./discord/command-work-notific export async function notifyChatEndpointConfiguration(transaction: object, endpoint: { id: string; provider: ChatProvider }) { if (endpoint.provider === "speko") await notifyVoiceChange(transaction, { kind: "endpoint", id: endpoint.id }); await notifyChatReceiver(transaction, endpoint); - if (endpoint.provider === "github") await notifyGitHubWebhookRecovery(transaction, { kind: "endpoint", id: endpoint.id }); + if (endpoint.provider === "github") { + await notifyDeliveryWork(transaction, DELIVERY_QUEUES.connectionEvents); + await notifyGitHubWebhookRecovery(transaction, { kind: "endpoint", id: endpoint.id }); + } if (endpoint.provider === "discord") { await notifyDiscordCommandRegistration(transaction, { kind: "endpoint", id: endpoint.id }); } diff --git a/server/src/services/chat-channels/configuration.ts b/server/src/services/chat-channels/configuration.ts index fa1b1d5b1ce..8ded7cbb96e 100644 --- a/server/src/services/chat-channels/configuration.ts +++ b/server/src/services/chat-channels/configuration.ts @@ -1,3 +1,4 @@ +import { notifyChatEndpointConfiguration } from "./configuration-work-notifications.js"; import { notifyChatConnectionHealth } from "./connection-health-notifications.js"; import type { ConnectionManagementContext } from "../connection-management.js"; import { CHAT_HUMAN_CREDENTIAL_FIELDS } from "./credentials.js"; @@ -196,6 +197,7 @@ export function createChatConfiguration(db: Db, options: ChatConfigurationOption code: "chat_endpoint_not_active", }); } + await notifyChatEndpointConfiguration(tx, endpoint); await notifyChatEndpointWork(tx, endpoint.connectionId, endpoint.companyId); await tx .update(chatEndpoints) @@ -334,6 +336,7 @@ export function createChatConfiguration(db: Db, options: ChatConfigurationOption inArray(chatDeliveries.state, ["received", "retry"]), ), ); + await notifyChatEndpointConfiguration(tx, endpoint); await notifyChatEndpointWork(tx, endpoint.connectionId, endpoint.companyId); [activatedEndpoint] = await tx .update(chatEndpoints) @@ -383,6 +386,7 @@ export function createChatConfiguration(db: Db, options: ChatConfigurationOption await managementContext?.reauthorize(); await db.transaction(async (tx) => { await credentialLease.assertOwned(tx); + await notifyChatEndpointConfiguration(tx, endpoint); await notifyChatEndpointWork(tx, endpoint.connectionId, endpoint.companyId); await tx .update(chatEndpoints) @@ -487,6 +491,7 @@ export function createChatConfiguration(db: Db, options: ChatConfigurationOption code: "chat_endpoint_already_removed", }); } + await notifyChatEndpointConfiguration(tx, endpoint); await notifyChatEndpointWork(tx, endpoint.connectionId, endpoint.companyId); await tx .update(chatEndpoints) @@ -721,6 +726,7 @@ export function createChatConfiguration(db: Db, options: ChatConfigurationOption await managementContext?.reauthorize(); const claimed = await db.transaction(async (tx) => { await credentialLease.assertOwned(tx); + await notifyChatEndpointConfiguration(tx, endpoint); await notifyChatEndpointWork(tx, endpoint.connectionId, endpoint.companyId); const rows = await tx .update(chatEndpoints) @@ -800,6 +806,7 @@ export function createChatConfiguration(db: Db, options: ChatConfigurationOption await installSpekoVoiceTools(tx, endpoint, actorUserId); } const preserveSlackVerification = preservesProvisionedSlackVerification(current.setup, credentials.signingSecret ?? "", `${getWebhookPublicBaseUrl()}/api/chat-webhooks/${endpoint.publicId}/slack`); + await notifyChatEndpointConfiguration(tx, endpoint); await notifyChatEndpointWork(tx, endpoint.connectionId, endpoint.companyId); await tx .update(chatEndpoints) @@ -887,6 +894,7 @@ export function createChatConfiguration(db: Db, options: ChatConfigurationOption await managementContext?.reauthorize(); await db.transaction(async (tx) => { await credentialLease.assertOwned(tx); + await notifyChatEndpointConfiguration(tx, endpoint); await notifyChatEndpointWork(tx, endpoint.connectionId, endpoint.companyId); await tx .update(chatEndpoints) diff --git a/server/src/services/chat-github-wizard.ts b/server/src/services/chat-github-wizard.ts index cd5f38a1c75..abde5141afd 100644 --- a/server/src/services/chat-github-wizard.ts +++ b/server/src/services/chat-github-wizard.ts @@ -1,3 +1,4 @@ +import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js"; import { notifyGitHubWebhookRecovery } from "./chat-channels/github/webhook-recovery-notifications.js"; import type { AuthorizationActor } from "./authorization.js"; import { connectionManagementService, type ConnectionManagementContext } from "./connection-management.js"; @@ -173,8 +174,10 @@ export function githubChatWizardService( values: Partial>, ) { await db.transaction(async tx => { - if (values.cloudRegistrationId !== undefined) + if (values.cloudRegistrationId !== undefined) { + await notifyDeliveryWork(tx, DELIVERY_QUEUES.connectionEvents, bot.companyId); await notifyGitHubWebhookRecovery(tx, { kind: "endpoint", id: bot.id }); + } await tx .update(chatEndpoints) .set({ diff --git a/server/src/services/companies.ts b/server/src/services/companies.ts index cce62d9c934..d26f41c761f 100644 --- a/server/src/services/companies.ts +++ b/server/src/services/companies.ts @@ -1,3 +1,4 @@ +import { notifyGitHubCompanyChange } from "./github-connection-notifications.js"; import { notifyChatInboundWakeNotices } from "./chat-inbound-wake-notice-notifications.js"; import { notifyCompanyRetentionWork } from "./retention-work-notifications.js"; import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js"; @@ -332,8 +333,7 @@ export function companyService(db: Db, _budgetHooks: BudgetServiceHooks = {}) { const renamedPrefix = await resolveRenamedIssuePrefix(tx, id, companyPatch); if (companyPatch.status !== undefined && companyPatch.status !== existing.status) { - await notifyDeliveryWork(tx, DELIVERY_QUEUES.connectionEvents); - await notifyDeliveryWork(tx, DELIVERY_QUEUES.connectionContinuity); + await notifyGitHubCompanyChange(tx, id); await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers); await notifyDeliveryWork(tx, DELIVERY_QUEUES.skillSourceSync); await notifyDeliveryWork(tx, DELIVERY_QUEUES.routineSchedules); @@ -490,8 +490,7 @@ export function companyService(db: Db, _budgetHooks: BudgetServiceHooks = {}) { const wasAlreadyArchived = existing.status === "archived"; if (!wasAlreadyArchived) { - await notifyDeliveryWork(tx, DELIVERY_QUEUES.connectionEvents); - await notifyDeliveryWork(tx, DELIVERY_QUEUES.connectionContinuity); + await notifyGitHubCompanyChange(tx, id); await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers); await notifyDeliveryWork(tx, DELIVERY_QUEUES.skillSourceSync); await notifyDeliveryWork(tx, DELIVERY_QUEUES.routineSchedules); diff --git a/server/src/services/github-connection-events.ts b/server/src/services/github-connection-events.ts index 53834d6c4f5..5db8585ffcf 100644 --- a/server/src/services/github-connection-events.ts +++ b/server/src/services/github-connection-events.ts @@ -237,6 +237,8 @@ export function githubConnectionEventService( eq(toolConnections.companyId, connectionGrants.companyId), )) .where(and( + sql`${toolConnections.config}->>'sourceTemplateKey' = 'github'`, + sql`${toolConnections.config}->'oauth'->>'connectorProfile' = 'github.code'`, eq(connectionGrants.status, "active"), eq(toolConnections.status, "active"), eq(toolConnections.enabled, true), @@ -245,11 +247,11 @@ export function githubConnectionEventService( return bindingRows(rows); } - async function applyPullRequestEvent(companyId: string, event: LeasedEvent) { + async function applyPullRequestEvent(database: Db, companyId: string, event: LeasedEvent, effects: Array<(committedDb: Db) => Promise>) { const snapshot = githubSnapshotUpdate(event.payload); if (!snapshot) return; const appliedAt = now(); - await db.update(externalObjects).set({ + await database.update(externalObjects).set({ statusKey: snapshot.statusKey, statusLabel: snapshot.statusLabel, statusCategory: snapshot.statusCategory, @@ -269,13 +271,13 @@ export function githubConnectionEventService( sql`lower(${externalObjects.externalId}) = lower(${snapshot.externalId})`, )); if (snapshot.merged && event.action === "closed") { - await issueThreadInteractionService(db, { wakeup: options.wakeup }) + await issueThreadInteractionService(database, { wakeup: options.wakeup }) .sweepMergedPullRequestConfirmations([{ companyId, owner: snapshot.owner, repo: snapshot.repo, number: snapshot.number, - }]); + }], { deferConfirmationCommitEffects: effect => effects.push(effect) }); } } @@ -337,7 +339,7 @@ export function githubConnectionEventService( webhookHealth: unavailable ? "unhealthy" as const : "healthy" as const, }, }; - await notifyGitHubCredentialChange(database, { config: { sourceTemplateKey: "github" } }); + await notifyGitHubCredentialChange(database, { companyId: binding.companyId, config: { sourceTemplateKey: "github" } }); await database.update(connectionGrants).set({ providerTenant, updatedAt: now() }) .where(and(eq(connectionGrants.id, binding.grantId), eq(connectionGrants.companyId, binding.companyId))); await notifyToolConnectionHealth(database, { companyId: binding.companyId, id: binding.connectionId }); @@ -380,6 +382,7 @@ export function githubConnectionEventService( } const postCommitPublications: ActivityPublication[] = []; + const postCommitEffects: Array<(committedDb: Db) => Promise> = []; let applied = true; try { const applyAndFinalize = async (database: Db) => { @@ -391,7 +394,7 @@ export function githubConnectionEventService( if (!receipt) await database.update(connectionEventDeliveries).set({ status: "received", attempts: sql`${connectionEventDeliveries.attempts} + 1`, lastError: null, updatedAt: receiptAt, }).where(eq(connectionEventDeliveries.id, locked!.id)); - if (event.event === "pull_request") await applyPullRequestEvent(companyId, normalizedEvent); + if (event.event === "pull_request") await applyPullRequestEvent(database, companyId, normalizedEvent, postCommitEffects); if (event.event === "installation" || event.event === "installation_repositories") { for (const binding of bindings) await applyInstallationEvent(database, binding, normalizedEvent); } else { @@ -470,6 +473,7 @@ export function githubConnectionEventService( }, "GitHub webhook activity publication failed after commit"); } } + for (const effect of postCommitEffects) await effect(db); return applied ? "processed" as const : "duplicate" as const; } diff --git a/server/src/services/github-connection-notifications.ts b/server/src/services/github-connection-notifications.ts index 79f3c067aba..55811a3a64d 100644 --- a/server/src/services/github-connection-notifications.ts +++ b/server/src/services/github-connection-notifications.ts @@ -1,7 +1,26 @@ -import type { toolConnections } from "@paperclipai/db"; +import { toolConnections, chatEndpoints, type Db } from "@paperclipai/db"; +import { and, eq, sql } from "drizzle-orm"; import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js"; -export async function notifyGitHubCredentialChange(transaction: object, connection: Pick) { +export async function notifyGitHubCredentialChange(transaction: object, connection: Pick) { if (connection.config.sourceTemplateKey !== "github") return; - await notifyDeliveryWork(transaction, DELIVERY_QUEUES.connectionContinuity); - await notifyDeliveryWork(transaction, DELIVERY_QUEUES.connectionEvents); + await notifyDeliveryWork(transaction, DELIVERY_QUEUES.connectionContinuity, connection.companyId); + await notifyDeliveryWork(transaction, DELIVERY_QUEUES.connectionEvents, connection.companyId); +} + +/** Company lifecycle changes only activate owners with a supported registration. */ +export async function notifyGitHubCompanyChange(transaction: object, companyId: string) { + const db = transaction as Db; + const [connection] = await db.select({ id: toolConnections.id }).from(toolConnections).where(and( + eq(toolConnections.companyId, companyId), sql`${toolConnections.config}->>'sourceTemplateKey' = 'github'`, + )).limit(1); + if (connection) { + await notifyDeliveryWork(transaction, DELIVERY_QUEUES.connectionContinuity, companyId); + await notifyDeliveryWork(transaction, DELIVERY_QUEUES.connectionEvents, companyId); + return; + } + const [bot] = await db.select({ id: chatEndpoints.id }).from(chatEndpoints).where(and( + eq(chatEndpoints.companyId, companyId), eq(chatEndpoints.provider, "github"), + sql`${chatEndpoints.setup}->'github'->>'cloudRegistrationId' is not null`, + )).limit(1); + if (bot) await notifyDeliveryWork(transaction, DELIVERY_QUEUES.connectionEvents, companyId); } diff --git a/server/src/services/github-connection-work.ts b/server/src/services/github-connection-work.ts index 7849cf3bda8..45d2d0ee6e2 100644 --- a/server/src/services/github-connection-work.ts +++ b/server/src/services/github-connection-work.ts @@ -1,8 +1,9 @@ import { logger } from "../middleware/logger.js"; +import { registerScopedDeadlineWork } from "./scoped-deadline-work.js"; import type { Db } from "@paperclipai/db"; import type { createDeliveryWorkCoordinator } from "./delivery-work-coordinator.js"; import { registerDeadlineWork } from "./deadline-work.js"; -import { DELIVERY_QUEUES, subscribeDeliveryWork } from "./delivery-work-notifications.js"; +import { DELIVERY_QUEUES } from "./delivery-work-notifications.js"; import { registerConnectionEventWake } from "./connection-event-wake.js"; import type { githubConnectionEventService } from "./github-connection-events.js"; import type { createGitHubGrantContinuity } from "./tool-access/github-grant-continuity.js"; @@ -11,26 +12,28 @@ type Coordinator = ReturnType; export function registerGitHubConnectionEventWork(db: Db, coordinator: Coordinator, service: ReturnType, canRun: () => boolean) { const worker = registerDeadlineWork(coordinator, DELIVERY_QUEUES.connectionEvents, { - retryMs: 5_000, canRun, + retryMs: 5_000, minimumSpacingMs: 5_000, notificationCoalesceMs: 1_000, canRun, run: () => service.pollOnce({ forDelivery: true }), next: async () => service.nextPollAt(), }); - const stops = [registerConnectionEventWake(db, worker.wake), - subscribeDeliveryWork(db, DELIVERY_QUEUES.toolConnectionHealth, worker.wake), - subscribeDeliveryWork(db, DELIVERY_QUEUES.chatReceivers, worker.wake)]; + const stops = [registerConnectionEventWake(db, worker.wake)]; return { ...worker, stop: async () => { for (const stop of stops) stop(); await worker.stop(); } }; } export function registerGitHubConnectionContinuityWork(db: Db, coordinator: Coordinator, - service: Pick, "sweepGitHubConnectionContinuity" | "nextGitHubConnectionContinuityAt">, + service: Pick, "sweepGitHubConnectionContinuity" | "nextGitHubConnectionContinuityAt" | "githubConnectionContinuitySchedule">, canRun: () => boolean) { - const worker = registerDeadlineWork(coordinator, DELIVERY_QUEUES.connectionContinuity, { - retryMs: 30_000, canRun, - run: async () => { - const result = await service.sweepGitHubConnectionContinuity(); + const worker = registerScopedDeadlineWork(coordinator, { + queue: DELIVERY_QUEUES.connectionContinuity, + minimumPassIntervalMs: 5_000, + decodeScope: hint => hint === "*" ? undefined : hint, + matches: (entry, companyId?: string) => !companyId || entry.companyId === companyId, + load: companyId => service.githubConnectionContinuitySchedule(companyId), + check: async entry => { + const result = await service.sweepGitHubConnectionContinuity(entry.companyId); if (result.failed) logger.warn(result, "GitHub credential continuity retained retry obligations"); + const dueAt = await service.nextGitHubConnectionContinuityAt(entry.companyId); + return dueAt === null ? null : { ...entry, dueAt }; }, - next: () => service.nextGitHubConnectionContinuityAt(), - }); - const unsubscribe = subscribeDeliveryWork(db, DELIVERY_QUEUES.toolConnectionHealth, worker.wake); - return { ...worker, stop: async () => { unsubscribe(); await worker.stop(); } }; + }, canRun); + return worker; } diff --git a/server/src/services/issue-thread-interactions.ts b/server/src/services/issue-thread-interactions.ts index 59c14618342..730a82db3a6 100644 --- a/server/src/services/issue-thread-interactions.ts +++ b/server/src/services/issue-thread-interactions.ts @@ -2986,6 +2986,7 @@ export function issueThreadInteractionService( repo: string; number: number; }> = [], + mutationOptions: InteractionResolutionMutationOptions = {}, ) => { const rows = await db .select({ @@ -3005,6 +3006,7 @@ export function issueThreadInteractionService( and( eq(issueThreadInteractions.kind, "request_confirmation"), eq(issueThreadInteractions.status, "pending"), + mergedHints.length ? inArray(issues.companyId, [...new Set(mergedHints.map(hint => hint.companyId))]) : undefined, ), ); @@ -3087,6 +3089,7 @@ export function issueThreadInteractionService( issue: candidate.issue, current: candidate.interaction, input: {}, + mutationOptions, actor: { systemId: "system:pr-merged", resolutionDetails: { @@ -3123,7 +3126,7 @@ export function issueThreadInteractionService( ) { continue; } - await opts.wakeup(wakeIssue.assigneeAgentId, { + const wake = async () => { await opts.wakeup!(wakeIssue.assigneeAgentId!, { source: "automation", triggerDetail: "system", reason: "issue_commented", @@ -3154,6 +3157,9 @@ export function issueThreadInteractionService( source: "merged_pull_request_sweep", }, }); + }; + if (mutationOptions.deferConfirmationCommitEffects) mutationOptions.deferConfirmationCommitEffects(wake); + else await wake(); woken += 1; } diff --git a/server/src/services/tool-access.ts b/server/src/services/tool-access.ts index 7d4cfe8cf0d..23932fafbf9 100644 --- a/server/src/services/tool-access.ts +++ b/server/src/services/tool-access.ts @@ -1,3 +1,4 @@ +import { notifyGitHubCredentialChange } from "./github-connection-notifications.js"; import { notifyVoiceChange } from "./voice/voice-recovery-notifications.js"; import { notifyToolConnectionHealth, writeToolConnectionHealthChange } from "./tool-connection-health-notifications.js"; import { connectionHealthSchedule } from "./tool-access/catalog/health-schedule.js"; @@ -787,7 +788,7 @@ export function toolAccessService( const refreshLeases = createOAuthRefreshLeases(db, now, getConnectionRow, getOAuthGrantRow); const maybeRefreshOAuthCredentials = createLegacyOAuthRefresh(db, { now, secrets, store: credentialStore, leases: refreshLeases, getConnectionRow, exchangeOAuthToken }); const refreshOAuthGrantCredentials = createOAuthGrantRefresh(db, { now, store: credentialStore, leases: refreshLeases, currentCloudConnector, getConnectionRow, exchangeOAuthToken }); - const { refreshManagedGitHubGrantAccess, syncGitHubGrantBindings, sweepGitHubConnectionContinuity, nextGitHubConnectionContinuityAt } = createGitHubGrantContinuity(db, { now, currentCloudConnector, refreshOAuthGrantCredentials, resolveOAuthGrantSecret }); + const { refreshManagedGitHubGrantAccess, syncGitHubGrantBindings, sweepGitHubConnectionContinuity, nextGitHubConnectionContinuityAt, githubConnectionContinuitySchedule } = createGitHubGrantContinuity(db, { now, currentCloudConnector, refreshOAuthGrantCredentials, resolveOAuthGrantSecret }); const vercelCredentials = createVercelCredentialService(db, now, vercelConnect); const { resolveVercelCredentialHeaders, vercelGrantForConnection } = vercelCredentials; const credentialAccess = createConnectionCredentialAccess(db, { secrets, audit, vercel: vercelCredentials, resolveOAuthGrantSecret, maybeRefreshOAuthCredentials, refreshOAuthGrantCredentials }); @@ -3152,6 +3153,7 @@ export function toolAccessService( await tx.select({ id: toolApplications.id }).from(toolApplications) .where(and(eq(toolApplications.id, connection.applicationId), eq(toolApplications.companyId, companyId))) .for("update"); + await notifyGitHubCredentialChange(tx, connection); await notifyToolConnectionHealth(tx, connection); const [row] = await tx.update(toolConnections).set({ config, transportConfig: config, status: "active", enabled: true, ...(input.agentInstructions !== undefined ? { agentInstructions: input.agentInstructions } : {}), updatedAt: new Date() }) .where(and(eq(toolConnections.id, connection.id), eq(toolConnections.companyId, companyId))).returning(); @@ -3510,6 +3512,7 @@ export function toolAccessService( }, tx, ); + await notifyGitHubCredentialChange(tx, connection); await notifyToolConnectionHealth(tx, connection); const [updatedConnection] = await tx .update(toolConnections) @@ -4673,6 +4676,10 @@ export function toolAccessService( return rows; }) : await writeToolConnectionHealthChange(db, existing, async tx => { + if (input.enabled !== undefined || input.status !== undefined || input.config !== undefined || input.transportConfig !== undefined || input.credentialSecretRefs !== undefined || input.credentialRefs !== undefined) { + await notifyGitHubCredentialChange(tx, existing); + await notifyGitHubCredentialChange(tx, { companyId: existing.companyId, config }); + } if (existing.transport === "voice") await notifyVoiceChange(tx, { kind: "connection", id: existing.id }); const rows = await write(tx); await syncCredentialBindings(rows[0]!, [], tx); @@ -4733,6 +4740,7 @@ export function toolAccessService( sweepGitHubConnectionContinuity, nextGitHubConnectionContinuityAt, + githubConnectionContinuitySchedule, ...catalogReader, ...toolActionReviewQueue, ...profileService, diff --git a/server/src/services/tool-access/connections/removal.ts b/server/src/services/tool-access/connections/removal.ts index a9c201f9984..be9bea0dd78 100644 --- a/server/src/services/tool-access/connections/removal.ts +++ b/server/src/services/tool-access/connections/removal.ts @@ -1,3 +1,4 @@ +import { notifyGitHubCredentialChange } from "../../github-connection-notifications.js"; import { notifyVoiceChange } from "../../voice/voice-recovery-notifications.js"; import { notifyChatEndpointWork } from "../../chat-work-notifications.js"; import { notifyToolConnectionHealth } from "../../tool-connection-health-notifications.js"; @@ -543,6 +544,7 @@ async function removeConnection( const archived = await db.transaction(async (tx) => { if (connection.transport === "chat_sdk") await notifyChatEndpointWork(tx, connection.id, connection.companyId); if (connection.transport === "voice") await notifyVoiceChange(tx, { kind: "connection", id: connection.id }); + await notifyGitHubCredentialChange(tx, connection); await notifyToolConnectionHealth(tx, connection); await tx.select({ id: toolApplications.id }).from(toolApplications) .where(and(eq(toolApplications.id, connection.applicationId), eq(toolApplications.companyId, connection.companyId))) diff --git a/server/src/services/tool-access/github-grant-continuity.ts b/server/src/services/tool-access/github-grant-continuity.ts index 1096d5f08e6..e3095597d4b 100644 --- a/server/src/services/tool-access/github-grant-continuity.ts +++ b/server/src/services/tool-access/github-grant-continuity.ts @@ -204,7 +204,7 @@ export function createGitHubGrantContinuity(db: Db, { now, currentCloudConnector return { updated, previousGitHub }; }); - return syncGitHubGrantBindings(connection, updated, actor, accessToken); + return measureRunPhase("connectors.github.webhooks", () => syncGitHubGrantBindings(connection, updated, actor, accessToken)); } async function syncGitHubGrantBindings( @@ -287,7 +287,7 @@ export function createGitHubGrantContinuity(db: Db, { now, currentCloudConnector }); } - async function continuityRows() { + async function continuityRows(companyId?: string) { return db .select({ grant: connectionGrants, connection: toolConnections }) .from(connectionGrants) @@ -296,6 +296,7 @@ export function createGitHubGrantContinuity(db: Db, { now, currentCloudConnector eq(toolConnections.companyId, connectionGrants.companyId), )) .where(and( + companyId ? eq(connectionGrants.companyId, companyId) : undefined, eq(connectionGrants.status, "active"), eq(toolConnections.status, "active"), eq(toolConnections.enabled, true), @@ -306,16 +307,16 @@ export function createGitHubGrantContinuity(db: Db, { now, currentCloudConnector )); } - async function nextGitHubConnectionContinuityAt(): Promise { - const deadlines = (await continuityRows()).flatMap(({ grant, connection }) => { + async function nextGitHubConnectionContinuityAt(companyId?: string): Promise { + const deadlines = (await continuityRows(companyId)).flatMap(({ grant, connection }) => { const deadline = githubGrantContinuityDeadline(connection, grant); return deadline === null ? [] : [deadline]; }); return deadlines.length ? Math.min(...deadlines) : null; } - async function sweepGitHubConnectionContinuity() { - const rows = await continuityRows(); + async function sweepGitHubConnectionContinuity(companyId?: string) { + const rows = await continuityRows(companyId); const due = rows.filter(({ grant, connection }) => { const deadline = githubGrantContinuityDeadline(connection, grant); return deadline !== null && deadline <= now().getTime(); @@ -340,7 +341,15 @@ export function createGitHubGrantContinuity(db: Db, { now, currentCloudConnector } return { checked: rows.length, due: due.length, refreshed, failed }; } - return { refreshManagedGitHubGrantAccess, syncGitHubGrantBindings, sweepGitHubConnectionContinuity, nextGitHubConnectionContinuityAt }; + async function githubConnectionContinuitySchedule(companyId?: string) { + const schedule = new Map(); + for (const { grant, connection } of await continuityRows(companyId)) { + const dueAt = githubGrantContinuityDeadline(connection, grant); + if (dueAt !== null) schedule.set(connection.companyId, Math.min(schedule.get(connection.companyId) ?? Infinity, dueAt)); + } + return [...schedule].map(([companyId, dueAt]) => ({ id: companyId, companyId, dueAt })); + } + return { githubConnectionContinuitySchedule, refreshManagedGitHubGrantAccess, syncGitHubGrantBindings, sweepGitHubConnectionContinuity, nextGitHubConnectionContinuityAt }; } /** Credential expiry and provider continuity are obligations, not a scan cadence. */ diff --git a/server/src/services/tool-access/oauth/access-finalization.ts b/server/src/services/tool-access/oauth/access-finalization.ts index a6ad361dde8..4d0ab9ed010 100644 --- a/server/src/services/tool-access/oauth/access-finalization.ts +++ b/server/src/services/tool-access/oauth/access-finalization.ts @@ -1,3 +1,4 @@ +import { notifyGitHubCredentialChange } from "../../github-connection-notifications.js"; import { notifyToolConnectionHealth, writeToolConnectionHealthChange } from "../../tool-connection-health-notifications.js"; import { connectionActorUserId } from "../actor.js"; import type { ActorInfo } from "../types.js"; @@ -116,6 +117,7 @@ async function finalizeOAuthAccess( throw conflict("This connection already uses a company identity"); } connection = await writeToolConnectionHealthChange(db, connection, async tx => { + await notifyGitHubCredentialChange(tx, connection); const [updated] = await tx .update(toolConnections) .set({ diff --git a/server/src/services/tool-access/oauth/cloud-callback.ts b/server/src/services/tool-access/oauth/cloud-callback.ts index 7381c1687a4..ebba51b0b92 100644 --- a/server/src/services/tool-access/oauth/cloud-callback.ts +++ b/server/src/services/tool-access/oauth/cloud-callback.ts @@ -1,3 +1,4 @@ +import { notifyGitHubCredentialChange } from "../../github-connection-notifications.js"; import type { createGitHubGrantContinuity } from "../github-grant-continuity.js"; import { randomUUID } from "node:crypto"; import { notifyIssueReviewState } from "../../issue-review-state-notifications.js"; @@ -304,6 +305,7 @@ export function createCloudOAuthCallback(db: Db, dependencies: Dependencies) { accessRef, ...(refreshRef ? [refreshRef] : []), ]; + await notifyGitHubCredentialChange(tx, connection); await notifyToolConnectionHealth(tx, connection); const grantValues = { providerTenant: { diff --git a/server/src/services/tool-access/oauth/grant-refresh.ts b/server/src/services/tool-access/oauth/grant-refresh.ts index ce0ca102352..096b5d4f9c9 100644 --- a/server/src/services/tool-access/oauth/grant-refresh.ts +++ b/server/src/services/tool-access/oauth/grant-refresh.ts @@ -14,7 +14,7 @@ import { secretService } from "../../secrets.js"; import { isPaperclipCloudConnectorStrategy, PaperclipCloudConnectorError, type PaperclipCloudConnector } from "../../paperclip-cloud-connector.js"; import { actorForSecret, type OAuthCredentialStore } from "./credential-store.js"; -import { withoutOAuthRefreshLease, withoutOAuthGrantRefreshLease, type OAuthRefreshLeases } from "./refresh-leases.js"; +import { withoutOAuthRefreshLease, type OAuthRefreshLeases } from "./refresh-leases.js"; import { managedConnectorProfile, oauthGrantConfig, oauthGrantExpiresAtMs } from "./client-settings.js"; import { connectionSetupUrl, connectionReconnectUrl } from "../identifiers.js"; import { singleFlight } from "../single-flight.js"; @@ -179,10 +179,7 @@ export function createOAuthGrantRefresh(db: Db, { now, store, leases, currentClo grant.kind === "user" && grant.subjectUserId ? { actorType: "user", actorId: grant.subjectUserId } : input.actor; - const providerTenant = { - ...(grant.providerTenant ?? {}), - oauth: { - ...withoutOAuthGrantRefreshLease(oauthGrantConfig(grant)), + const oauthUpdates = { strategy: "paperclip_cloud_connector", accessTokenExpiresAt: credentials.accessTokenExpiresAt, scopes: credentials.scopes, @@ -191,7 +188,6 @@ export function createOAuthGrantRefresh(db: Db, { now, store, leases, currentClo ...(credentials.refreshTokenExpiresAt ? { refreshTokenExpiresAt: credentials.refreshTokenExpiresAt } : {}), - }, }; const updated = await db.transaction(async (tx) => { await notifyGitHubCredentialChange(tx, connection); @@ -219,7 +215,7 @@ export function createOAuthGrantRefresh(db: Db, { now, store, leases, currentClo const [committed] = await tx .update(connectionGrants) .set({ - providerTenant, + providerTenant: sql`jsonb_set(coalesce(${connectionGrants.providerTenant}, '{}'::jsonb), '{oauth}', (coalesce(${connectionGrants.providerTenant}->'oauth', '{}'::jsonb) || ${JSON.stringify(oauthUpdates)}::jsonb) - 'refreshLease')`, status: "active", updatedAt: now(), }) @@ -344,18 +340,11 @@ export function createOAuthGrantRefresh(db: Db, { now, store, leases, currentClo ref.configPath !== "oauth.access_token" && ref.configPath !== "oauth.refresh_token", ); - const providerTenant = { - ...(grant.providerTenant ?? {}), - oauth: { - ...withoutOAuthGrantRefreshLease(grantOauth), - accessTokenExpiresAt: undefined, - }, - }; const [marked] = await db .update(connectionGrants) .set({ status: "needs_reauthorization", - providerTenant, + providerTenant: sql`${connectionGrants.providerTenant} #- '{oauth,refreshLease}' #- '{oauth,accessTokenExpiresAt}'`, credentialSecretRefs: retainedCredentialSecretRefs, updatedAt: now(), }) @@ -525,26 +514,22 @@ export function createOAuthGrantRefresh(db: Db, { now, store, leases, currentClo if (connection.config.sourceTemplateKey === "google-workspace") { refreshed.scopes = googleWorkspaceEffectiveScopes(normalizeOauthScopes(grantOauth.requestedScopes ?? grantOauth.scopes), refreshed.scopes); } - const providerTenant = { - ...(grant.providerTenant ?? {}), - oauth: { - ...withoutOAuthGrantRefreshLease(grantOauth), + const oauthUpdates = { strategy: typeof grantOauth.strategy === "string" ? grantOauth.strategy : "direct_oauth", - accessTokenExpiresAt: expiresAt ?? undefined, + accessTokenExpiresAt: expiresAt ?? null, scopes: refreshed.scopes, scopeSource: refreshed.scopeSource, unrequestedScopes: refreshed.unrequestedScopes, tokenType: token.tokenType, refreshedAt: now().toISOString(), - }, }; const [updated] = await writeToolConnectionHealthChange(db, connection, tx => tx .update(connectionGrants) .set({ - providerTenant, + providerTenant: sql`jsonb_set(coalesce(${connectionGrants.providerTenant}, '{}'::jsonb), '{oauth}', (coalesce(${connectionGrants.providerTenant}->'oauth', '{}'::jsonb) || ${JSON.stringify(oauthUpdates)}::jsonb) - 'refreshLease')`, credentialSecretRefs: nextCredentialSecretRefs, status: "active", updatedAt: now(), diff --git a/server/src/services/tool-access/oauth/refresh-leases.ts b/server/src/services/tool-access/oauth/refresh-leases.ts index 8250da26e65..44dd257129d 100644 --- a/server/src/services/tool-access/oauth/refresh-leases.ts +++ b/server/src/services/tool-access/oauth/refresh-leases.ts @@ -43,15 +43,11 @@ export function createOAuthRefreshLeases(db: Db, now: () => Date, getConnectionR ) { const latest = await getConnectionRow(connection.id, connection.companyId); if (oauthRefreshLeaseId(latest) !== leaseId) return latest; - const nextConfig = { - ...latest.config, - oauth: withoutOAuthRefreshLease(oauthConfig(latest)), - }; const [updated] = await db .update(toolConnections) .set({ - config: nextConfig, - transportConfig: nextConfig, + config: sql`${toolConnections.config} #- '{oauth,refreshLease}'`, + transportConfig: sql`${toolConnections.transportConfig} #- '{oauth,refreshLease}'`, updatedAt: now(), }) .where( @@ -165,14 +161,10 @@ export function createOAuthRefreshLeases(db: Db, now: () => Date, getConnectionR }); const oauth = oauthGrantConfig(latest); if (asRecord(oauth.refreshLease).id !== leaseId) return latest; - const providerTenant = { - ...(latest.providerTenant ?? {}), - oauth: withoutOAuthGrantRefreshLease(oauth), - }; const [updated] = await db .update(connectionGrants) .set({ - providerTenant, + providerTenant: sql`${connectionGrants.providerTenant} #- '{oauth,refreshLease}'`, updatedAt: now(), }) .where( @@ -202,7 +194,7 @@ export function createOAuthRefreshLeases(db: Db, now: () => Date, getConnectionR leaseId: string | null; }> { const waitDeadline = Date.now() + OAUTH_REFRESH_LEASE_WAIT_MS; - const initialUpdatedAt = grant.updatedAt.getTime(); + const initialOAuth = JSON.stringify(withoutOAuthGrantRefreshLease(oauthGrantConfig(grant))); while (true) { const latest = await getOAuthGrantRow({ companyId: connection.companyId, @@ -221,7 +213,7 @@ export function createOAuthRefreshLeases(db: Db, now: () => Date, getConnectionR (!forceRefresh && (expiresAtMs === null || expiresAtMs > Date.now() + 60_000)) || (forceRefresh && - latest.updatedAt.getTime() > initialUpdatedAt && + JSON.stringify(withoutOAuthGrantRefreshLease(oauthGrantConfig(latest))) !== initialOAuth && !asRecord(oauthGrantConfig(latest).refreshLease).id) ) { return { grant: latest, leaseId: null }; @@ -266,6 +258,7 @@ export function createOAuthRefreshLeases(db: Db, now: () => Date, getConnectionR eq(connectionGrants.id, latest.id), eq(connectionGrants.companyId, latest.companyId), eq(connectionGrants.status, "active"), + sql`${connectionGrants.providerTenant} = ${JSON.stringify(latest.providerTenant)}::jsonb`, sql`${connectionGrants.providerTenant} #>> '{oauth,refreshLease,id}' is null`, ), ) From efbe3fa67d8dee2fb2037d9b097ed6ae1876a982 Mon Sep 17 00:00:00 2001 From: Dotta Date: Fri, 30 Oct 2026 18:25:01 -0500 Subject: [PATCH 48/74] fix(server): isolate blocker traversal budgets per root Share queried graph facts across roots while retaining independent node/depth limits, expanded reachability and truncation for classification. Unrelated limited graphs must not change terminal subjects or delay archive deadlines. Prove depth/node overflow and overlapping-root liveness red/green, preserve the100-root SQL bound, and qualify existing owner assertions on isolated PostgreSQL. Co-Authored-By: Paperclip --- .../2026-10-30-attention-retention-sleep.md | 2 +- .../attention-retention-work.test.ts | 57 ++++++++++++++++++- .../__tests__/issue-blocker-attention.test.ts | 12 +++- server/src/services/issues.ts | 56 +++++++++++------- 4 files changed, 101 insertions(+), 26 deletions(-) diff --git a/doc/plans/2026-10-30-attention-retention-sleep.md b/doc/plans/2026-10-30-attention-retention-sleep.md index 2e0373fc975..463fbe94188 100644 --- a/doc/plans/2026-10-30-attention-retention-sleep.md +++ b/doc/plans/2026-10-30-attention-retention-sleep.md @@ -25,7 +25,7 @@ The feed and worker share confirmation collapse, newer-run probes, issue visibil ## Existing mutation boundaries reactivate work -Committed hints identify a company, source kind, and source ID where those values are known. Retention's own writes use exact source hints; blocker retention changes invalidate the company/source because multiple blocked roots can project one terminal subject. Reconstruction pages and due batches remain bounded. Review attention and interaction invokability are computed in company batches per candidate page; a full 100-candidate review page uses at most 20 statements and 100 confirmations use at most six. Decisions, recovery source links, runs, eligible agents, issue visibility, newer-run probes and blocker graphs also load in company batches per page. A 100-source decision or recovery page uses at most three statements; 100 failed runs use at most six; 100 blocker roots use at most 30 graph/source statements plus retention-state reads for their emitted entries. These evidence maps exist only for one load. Archive checks still lock and freshly re-read sources, terminal subjects and related eligibility. Ordinary execution, decision and budget mutations pass their affected company; exact decision sources are included. +Committed hints identify a company, source kind, and source ID where those values are known. Retention's own writes use exact source hints; blocker retention changes invalidate the company/source because multiple blocked roots can project one terminal subject. Reconstruction pages and due batches remain bounded. Review attention and interaction invokability are computed in company batches per candidate page; a full 100-candidate review page uses at most 20 statements and 100 confirmations use at most six. Decisions, recovery source links, runs, eligible agents, issue visibility, newer-run probes and blocker graphs also load in company batches per page. A 100-source decision or recovery page uses at most three statements; 100 failed runs use at most six; 100 blocker roots use at most 30 graph/source statements plus retention-state reads for their emitted entries. These evidence maps exist only for one load. The blocker owner shares fetched node and edge facts while tracking depth, node count, expanded nodes and live-work reachability independently for each root. An unrelated graph reaching its limit cannot change a shallow root’s terminal subject or deadline. Archive checks still lock and freshly re-read sources, terminal subjects and related eligibility. Ordinary execution, decision and budget mutations pass their affected company; exact decision sources are included. Producer coverage includes decision creation and changes; interaction creation, resolution, expiry and supersession; issue creation, updates, blocker relations, checkout, adoption and release; agent hire, configuration, lifecycle, error clearing and deletion; budget incidents and company pause/resume; exhausted-run events and retry-run creation; native status, review, finalization and reconstruction; external conversation transitions; and imported blocker relations. Existing service transactions retain outer commit ownership. Savepoint release does not publish work before outer commit. diff --git a/server/src/__tests__/attention-retention-work.test.ts b/server/src/__tests__/attention-retention-work.test.ts index fd1a01e8582..29a82f866a5 100644 --- a/server/src/__tests__/attention-retention-work.test.ts +++ b/server/src/__tests__/attention-retention-work.test.ts @@ -15,7 +15,7 @@ import { decisionRetentionService, DEFAULT_DECISION_ARCHIVE_DAYS } from "../serv import { appendHeartbeatRunEvent } from "../services/heartbeat-run-events.js"; import { issueThreadInteractionService } from "../services/issue-thread-interactions.js"; import { createAgentLifecycle } from "../services/agent-lifecycle.js"; -import { issueService } from "../services/issues.js"; +import { BLOCKER_ATTENTION_MAX_DEPTH, BLOCKER_ATTENTION_MAX_NODES, issueService } from "../services/issues.js"; import { notifyAttentionRetentionWork } from "../services/retention-work-notifications.js"; import { idleWorkSnapshot } from "../services/task-admission.js"; import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; @@ -256,6 +256,61 @@ const delay = DEFAULT_DECISION_ARCHIVE_DAYS * 86_400_000; expect(refreshed.rows!.some(entry => entry.sourceId === terminals[99].id)).toBe(false); }); + it.each(["depth", "nodes"] as const)("preserves shallow terminal selection when an unrelated graph hits its %s limit", async limit => { + const root = await insertIssue("blocked", { updatedAt: new Date() }); + const middle = await insertIssue("todo", { updatedAt: new Date() }); + const terminal = await insertIssue("backlog", { assigneeAgentId: agentId }); + const heavy = await insertIssue("blocked", { updatedAt: new Date() }); + const count = limit === "depth" ? BLOCKER_ATTENTION_MAX_DEPTH + 1 : BLOCKER_ATTENTION_MAX_NODES; + const heavyNodes = await db.insert(issues).values(Array.from({ length: count }, () => ({ companyId, + title: randomUUID(), status: "todo", updatedAt: new Date() }))).returning(); + await db.insert(issueRelations).values([ + { companyId, issueId: middle.id, relatedIssueId: root.id, type: "blocks" }, + { companyId, issueId: terminal.id, relatedIssueId: middle.id, type: "blocks" }, + ...heavyNodes.map((node, index) => ({ companyId, issueId: node.id, + relatedIssueId: limit === "nodes" || index === 0 ? heavy.id : heavyNodes[index - 1].id, type: "blocks" })), + ]); + const owner = issueService(db); + const expected = (await owner.listBlockerAttention(companyId, [root])).get(root.id)!; + const limited = (await owner.listBlockerAttention(companyId, [heavy])).get(heavy.id)!; + expect(expected.terminalBlockerIssueId).toBe(terminal.id); + expect(limited.terminalBlockerIssueId).toBe(limited.directBlockerIssueId); + for (const input of [[root, heavy], [heavy, root]]) { + const batch = await owner.listBlockerAttention(companyId, input); + expect(batch.get(root.id)).toEqual(expected); expect(batch.get(heavy.id)).toEqual(limited); + } + const page = await attentionRetentionService(db).load({ companyId, sourceKind: "blocker_attention" }); + const entry = page.rows!.find(item => item.rootId === root.id && item.variant === "terminal")!; + expect(entry.sourceId).toBe(terminal.id); expect(entry.dueAt).toBeLessThan(Date.now()); + await attentionRetentionService(db).check(entry); + expect((await state("blocker_attention", terminal.id))?.archivedAt).toBeInstanceOf(Date); + expect(await db.select().from(decisionArchiveNotificationOutbox).where(eq(decisionArchiveNotificationOutbox.sourceId, terminal.id))).toHaveLength(1); + }); + + it("keeps root-specific reachability and cycle budgets when another root explores farther", async () => { + const root = await insertIssue("blocked"); + const chain = await db.insert(issues).values(Array.from({ length: BLOCKER_ATTENTION_MAX_DEPTH + 1 }, (_, index) => ({ + companyId, title: randomUUID(), status: index === BLOCKER_ATTENTION_MAX_DEPTH ? "in_progress" + : index === BLOCKER_ATTENTION_MAX_DEPTH - 2 ? "blocked" : "todo", updatedAt: old() }))).returning(); + const overlapping = chain[BLOCKER_ATTENTION_MAX_DEPTH - 2]; + const cycle = await insertIssue("blocked"); + const leaf = await insertIssue("backlog", { assigneeAgentId: agentId }); + await db.insert(issueRelations).values([ + ...chain.map((node, index) => ({ companyId, issueId: node.id, relatedIssueId: index ? chain[index - 1].id : root.id, type: "blocks" })), + { companyId, issueId: leaf.id, relatedIssueId: cycle.id, type: "blocks" }, + { companyId, issueId: cycle.id, relatedIssueId: leaf.id, type: "blocks" }, + ]); + const owner = issueService(db), roots = [root, overlapping, cycle]; + const singles = new Map(); + for (const item of roots) singles.set(item.id, (await owner.listBlockerAttention(companyId, [item])).get(item.id)); + expect(singles.get(root.id).blockingTreeLive).toBe(false); + expect(singles.get(overlapping.id).blockingTreeLive).toBe(true); + for (const input of [roots, [...roots].reverse()]) { + const batch = await owner.listBlockerAttention(companyId, input); + for (const item of roots) expect(batch.get(item.id)).toEqual(singles.get(item.id)); + } + }); + it("scopes ordinary execution-control writes to their company", async () => { const otherCompany = randomUUID(); await db.insert(companies).values({ id: otherCompany, name: "Other", issuePrefix: "OTHER" }); diff --git a/server/src/__tests__/issue-blocker-attention.test.ts b/server/src/__tests__/issue-blocker-attention.test.ts index 1e6c00631e7..4f1a0e71573 100644 --- a/server/src/__tests__/issue-blocker-attention.test.ts +++ b/server/src/__tests__/issue-blocker-attention.test.ts @@ -8,6 +8,7 @@ import { approvals, companies, createDb, + applyPendingMigrations, heartbeatRuns, issueApprovals, issueRelations, @@ -21,7 +22,10 @@ import { import { issueService } from "../services/issues.js"; import { buildIssueGraphLivenessIncidentKey } from "../services/recovery/origins.js"; -const embeddedPostgresSupport = await getEmbeddedPostgresTestSupport(); +import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js"; + +const externalUrl = resolveExternalTestDatabaseUrl("PAPERCLIP_BLOCKER_ATTENTION_TEST_DATABASE_URL"); +const embeddedPostgresSupport = externalUrl ? { supported: true } : await getEmbeddedPostgresTestSupport(); const describeEmbeddedPostgres = embeddedPostgresSupport.supported ? describe : describe.skip; if (!embeddedPostgresSupport.supported) { @@ -36,8 +40,9 @@ describeEmbeddedPostgres("issue blocker attention", () => { let tempDb: Awaited> | null = null; beforeAll(async () => { - tempDb = await startEmbeddedPostgresTestDatabase("paperclip-issue-blocker-attention-"); - db = createDb(tempDb.connectionString); + if (externalUrl) await applyPendingMigrations(externalUrl); + else tempDb = await startEmbeddedPostgresTestDatabase("paperclip-issue-blocker-attention-"); + db = createDb(externalUrl ?? tempDb!.connectionString, { maxConnections: 6 }); svc = issueService(db); }, 20_000); @@ -55,6 +60,7 @@ describeEmbeddedPostgres("issue blocker attention", () => { }); afterAll(async () => { + await db?.$client.end({ timeout: 1 }); await tempDb?.cleanup(); }); diff --git a/server/src/services/issues.ts b/server/src/services/issues.ts index 7239075de99..fd1fd9ba819 100644 --- a/server/src/services/issues.ts +++ b/server/src/services/issues.ts @@ -3840,18 +3840,19 @@ async function listIssueBlockerAttentionMap( const edgesByIssueId = new Map(); for (const root of roots) nodesById.set(root.id, { ...root }); - let frontier = roots.map((root) => root.id); - let truncated = false; + type Traversal = { nodes: Set; expanded: Set; frontier: string[]; truncated: boolean }; + const traversals = new Map(roots.map(root => [root.id, { + nodes: new Set([root.id]), expanded: new Set(), frontier: [root.id], truncated: false, + }])); + const expanded = new Set(); const pendingFinalizeBlockerIssueIds = new Set(); - for ( - let depth = 0; - frontier.length > 0 && depth < BLOCKER_ATTENTION_MAX_DEPTH; - depth += 1 - ) { - const nextFrontier = new Set(); + for (let depth = 0; depth < BLOCKER_ATTENTION_MAX_DEPTH; depth += 1) { + const active = [...traversals.values()].filter(traversal => !traversal.truncated && traversal.frontier.length > 0); + if (active.length === 0) break; + const frontier = [...new Set(active.flatMap(traversal => traversal.frontier))].filter(id => !expanded.has(id)); for (const chunk of chunkList( - [...new Set(frontier)], + frontier, ISSUE_LIST_RELATED_QUERY_CHUNK_SIZE, )) { const readinessByIssueId = await listIssueDependencyReadinessMap( @@ -3959,17 +3960,27 @@ async function listIssueBlockerAttentionMap( assigneeAgentId: row.assigneeAgentId, assigneeUserId: row.assigneeUserId, }); - nextFrontier.add(row.blockerIssueId); } + for (const id of chunk) expanded.add(id); } - if (nodesById.size > BLOCKER_ATTENTION_MAX_NODES) { - truncated = true; - break; + for (const traversal of active) { + const nextFrontier = new Set(); + for (const id of traversal.frontier) { + traversal.expanded.add(id); + for (const edge of edgesByIssueId.get(id) ?? []) { + if (traversal.nodes.has(edge.blockerIssueId) || !nodesById.has(edge.blockerIssueId)) continue; + traversal.nodes.add(edge.blockerIssueId); + nextFrontier.add(edge.blockerIssueId); + } + } + traversal.truncated = traversal.nodes.size > BLOCKER_ATTENTION_MAX_NODES; + traversal.frontier = traversal.truncated ? [] : [...nextFrontier]; } - frontier = [...nextFrontier]; } - if (frontier.length > 0) truncated = true; + for (const traversal of traversals.values()) { + if (traversal.frontier.length > 0) traversal.truncated = true; + } const nodeIds = [...nodesById.keys()]; const activeIssueIds = new Set(); @@ -4215,6 +4226,7 @@ async function listIssueBlockerAttentionMap( const classifyPath = ( nodeId: string, seen: Set, + truncated: boolean, ): PathClassification => { const sample = blockerSampleIdentifier(nodesById.get(nodeId)); if (truncated || seen.has(nodeId)) { @@ -4319,7 +4331,7 @@ async function listIssueBlockerAttentionMap( const nextSeen = new Set(seen); nextSeen.add(nodeId); const classified = downstream.map((edge) => - classifyPath(edge.blockerIssueId, nextSeen), + classifyPath(edge.blockerIssueId, nextSeen, truncated), ); const stalledChild = classified.find( (result) => result.stalled || result.sampleStalledBlockerIdentifier, @@ -4385,8 +4397,8 @@ async function listIssueBlockerAttentionMap( }; }; - const pathHasLiveWork = (nodeId: string, seen: Set): boolean => { - if (seen.has(nodeId)) return false; + const pathHasLiveWork = (nodeId: string, seen: Set, traversal: Traversal): boolean => { + if (seen.has(nodeId) || !traversal.nodes.has(nodeId)) return false; const node = nodesById.get(nodeId); if (!node || node.companyId !== companyId) return false; if ( @@ -4397,6 +4409,7 @@ async function listIssueBlockerAttentionMap( ) return true; + if (!traversal.expanded.has(nodeId)) return false; const nextSeen = new Set(seen); nextSeen.add(nodeId); return (edgesByIssueId.get(node.id) ?? []).some((edge) => { @@ -4406,7 +4419,7 @@ async function listIssueBlockerAttentionMap( !pendingFinalizeBlockerIssueIds.has(edge.blockerIssueId) ) return false; - return pathHasLiveWork(edge.blockerIssueId, nextSeen); + return pathHasLiveWork(edge.blockerIssueId, nextSeen, traversal); }); }; @@ -4419,6 +4432,7 @@ async function listIssueBlockerAttentionMap( }; for (const root of roots) { + const traversal = traversals.get(root.id)!; const topLevelEdges = (edgesByIssueId.get(root.id) ?? []).filter((edge) => { const blocker = nodesById.get(edge.blockerIssueId); return ( @@ -4440,7 +4454,7 @@ async function listIssueBlockerAttentionMap( const classified = topLevelEdges.map((edge) => ({ edge, - result: classifyPath(edge.blockerIssueId, new Set([root.id])), + result: classifyPath(edge.blockerIssueId, new Set([root.id]), traversal.truncated), })); const coveredBlockerCount = classified.filter( (entry) => entry.result.covered, @@ -4523,7 +4537,7 @@ async function listIssueBlockerAttentionMap( sampleStalledFromChain ?? null, blockingTreeLive: topLevelEdges.some((edge) => - pathHasLiveWork(edge.blockerIssueId, new Set([root.id])), + pathHasLiveWork(edge.blockerIssueId, new Set([root.id]), traversal), ), directBlockerIssueId: sampleEntry?.edge.blockerIssueId ?? null, terminalBlockerIssueId, From 2ee5c7d6dc4a6cbf4ee9211c71dec3b75496509b Mon Sep 17 00:00:00 2001 From: Dotta Date: Fri, 30 Oct 2026 18:28:40 -0500 Subject: [PATCH 49/74] fix: preserve status-card deadline ownership through restart Use a dedicated advisory-lock connection so a one-slot write pool can finish evaluation and result writeback. Record internal deadline provenance so startup recovers lost membership hints without advancing saved policy or retry debt. Co-Authored-By: Paperclip --- doc/plans/2026-10-30-status-cards-sleep.md | 23 + .../src/migrations/0419_moaning_vulture.sql | 1 + ...{0414_snapshot.json => 0419_snapshot.json} | 516 +++++++++++++++++- packages/db/src/migrations/meta/_journal.json | 7 + packages/db/src/schema/status_cards.ts | 3 + .../status-card-evaluation-work.test.ts | 88 ++- server/src/__tests__/status-cards.test.ts | 2 + server/src/routes/status-cards.ts | 26 +- .../src/services/status-card-finalization.ts | 1 + server/src/services/status-cards.ts | 47 +- 10 files changed, 684 insertions(+), 30 deletions(-) create mode 100644 packages/db/src/migrations/0419_moaning_vulture.sql rename packages/db/src/migrations/meta/{0414_snapshot.json => 0419_snapshot.json} (99%) diff --git a/doc/plans/2026-10-30-status-cards-sleep.md b/doc/plans/2026-10-30-status-cards-sleep.md index 2af51ff65cf..dbf229cdace 100644 --- a/doc/plans/2026-10-30-status-cards-sleep.md +++ b/doc/plans/2026-10-30-status-cards-sleep.md @@ -100,3 +100,26 @@ An expired saved deadline cannot schedule an immediate skipped-attempt loop. Existing startup, status-card, company-search, execution and monitor suites cover the production wiring and unchanged consumers. Exact commands and results belong in the PR verification record; this document does not claim deployed savings. + +### Review corrections: pool capacity and restart deadline ownership + +Scheduled evaluation holds its advisory transaction on `withDedicatedDbConnection`, +not on the ordinary write pool. The helper ends that physical connection in +`finally`, including contention and failed evaluation. Its lock still spans the +actual refresh settlement. Summary writeback runs watched-query reads on its +existing transaction, so a one-connection main pool completes the full flow. + +Migration0419 adds nullable internal `next_eval_reason` to the existing card row. +Membership deadlines may be reconstructed at startup after a committed source +hint was lost. Policy eligibility and the atomically recorded five-minute retry +lease retain their saved deadline. Explicit refresh deadlines remain due work. +The HTTP card response removes this internal provenance; the public schema stays +unchanged. + +For legacy rows, a pending change hash preserves policy eligibility. An untagged +active deadline farther than five minutes away cannot be an outstanding +five-minute claim, so startup rechecks it. An ambiguous untagged deadline within +five minutes keeps its remaining time without extension. This bounds migration +compatibility delay while preserving prior crash/failure recovery. After the +first successful evaluation all newly saved deadlines have explicit ownership. +No recurring compatibility scan is added. diff --git a/packages/db/src/migrations/0419_moaning_vulture.sql b/packages/db/src/migrations/0419_moaning_vulture.sql new file mode 100644 index 00000000000..03ea349aec1 --- /dev/null +++ b/packages/db/src/migrations/0419_moaning_vulture.sql @@ -0,0 +1 @@ +ALTER TABLE "status_cards" ADD COLUMN "next_eval_reason" text; \ No newline at end of file diff --git a/packages/db/src/migrations/meta/0414_snapshot.json b/packages/db/src/migrations/meta/0419_snapshot.json similarity index 99% rename from packages/db/src/migrations/meta/0414_snapshot.json rename to packages/db/src/migrations/meta/0419_snapshot.json index 2679e7d98bf..d03afbb279d 100644 --- a/packages/db/src/migrations/meta/0414_snapshot.json +++ b/packages/db/src/migrations/meta/0419_snapshot.json @@ -1,6 +1,6 @@ { - "id": "27c9008c-60e3-449b-9630-9e4977fad444", - "prevId": "5690a14a-f9cc-41a6-af51-c27f688f4ef6", + "id": "cce00822-b7a4-4acd-9c59-0f6f5822c0cf", + "prevId": "c777fa84-85a6-418f-bea1-0d60b56cf071", "version": "7", "dialect": "postgresql", "tables": { @@ -19052,6 +19052,98 @@ "checkConstraints": {}, "isRLSEnabled": false }, + "public.company_skill_mutation_receipts": { + "name": "company_skill_mutation_receipts", + "schema": "", + "columns": { + "company_id": { + "name": "company_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "key": { + "name": "key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "fingerprint": { + "name": "fingerprint", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "skill_id": { + "name": "skill_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "version_id": { + "name": "version_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "response": { + "name": "response", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "company_skill_mutation_receipts_key_idx": { + "name": "company_skill_mutation_receipts_key_idx", + "columns": [ + { + "expression": "company_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "company_skill_mutation_receipts_company_id_companies_id_fk": { + "name": "company_skill_mutation_receipts_company_id_companies_id_fk", + "tableFrom": "company_skill_mutation_receipts", + "tableTo": "companies", + "columnsFrom": [ + "company_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, "public.company_skill_repository_snapshots": { "name": "company_skill_repository_snapshots", "schema": "", @@ -19079,7 +19171,46 @@ "name": "file_inventory", "type": "jsonb", "primaryKey": false, - "notNull": true + "notNull": true, + "default": "'[]'::jsonb" + }, + "storage": { + "name": "storage", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "upload": { + "name": "upload", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "restore": { + "name": "restore", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "migration_at": { + "name": "migration_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false, + "default": "now()" + }, + "maintenance_at": { + "name": "maintenance_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false, + "default": "now()" + }, + "restore_retry_at": { + "name": "restore_retry_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false }, "created_at": { "name": "created_at", @@ -19110,6 +19241,38 @@ "concurrently": false, "method": "btree", "with": {} + }, + "company_skill_snapshots_migration_idx": { + "name": "company_skill_snapshots_migration_idx", + "columns": [ + { + "expression": "migration_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"company_skill_repository_snapshots\".\"migration_at\" is not null", + "concurrently": false, + "method": "btree", + "with": {} + }, + "company_skill_snapshots_maintenance_idx": { + "name": "company_skill_snapshots_maintenance_idx", + "columns": [ + { + "expression": "maintenance_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"company_skill_repository_snapshots\".\"maintenance_at\" is not null", + "concurrently": false, + "method": "btree", + "with": {} } }, "foreignKeys": { @@ -19999,6 +20162,25 @@ "primaryKey": false, "notNull": false }, + "package_snapshot_id": { + "name": "package_snapshot_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "package_file_map": { + "name": "package_file_map", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "migration_at": { + "name": "migration_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false, + "default": "now()" + }, "revision_number": { "name": "revision_number", "type": "integer", @@ -20057,6 +20239,52 @@ } }, "indexes": { + "company_skill_versions_migration_idx": { + "name": "company_skill_versions_migration_idx", + "columns": [ + { + "expression": "migration_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"company_skill_versions\".\"migration_at\" is not null", + "concurrently": false, + "method": "btree", + "with": {} + }, + "company_skill_versions_package_snapshot_idx": { + "name": "company_skill_versions_package_snapshot_idx", + "columns": [ + { + "expression": "package_snapshot_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "company_skill_versions_repository_snapshot_idx": { + "name": "company_skill_versions_repository_snapshot_idx", + "columns": [ + { + "expression": "repository_snapshot_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, "company_skill_versions_skill_revision_idx": { "name": "company_skill_versions_skill_revision_idx", "columns": [ @@ -20168,6 +20396,19 @@ "onDelete": "restrict", "onUpdate": "no action" }, + "company_skill_versions_package_snapshot_id_company_skill_repository_snapshots_id_fk": { + "name": "company_skill_versions_package_snapshot_id_company_skill_repository_snapshots_id_fk", + "tableFrom": "company_skill_versions", + "tableTo": "company_skill_repository_snapshots", + "columnsFrom": [ + "package_snapshot_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "restrict", + "onUpdate": "no action" + }, "company_skill_versions_author_agent_id_agents_id_fk": { "name": "company_skill_versions_author_agent_id_agents_id_fk", "tableFrom": "company_skill_versions", @@ -20613,6 +20854,78 @@ "checkConstraints": {}, "isRLSEnabled": false }, + "public.skill_package_deletions": { + "name": "skill_package_deletions", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "company_id": { + "name": "company_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "provider": { + "name": "provider", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "archive_key": { + "name": "archive_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "manifest_key": { + "name": "manifest_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "delete_after": { + "name": "delete_after", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "skill_package_deletions_due_idx": { + "name": "skill_package_deletions_due_idx", + "columns": [ + { + "expression": "delete_after", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, "public.company_transfer_runs": { "name": "company_transfer_runs", "schema": "", @@ -33849,7 +34162,7 @@ } ], "isUnique": false, - "where": "\"heartbeat_runs\".\"runtime_mode\" = 'native' AND \"heartbeat_runs\".\"status\" = 'cancelled'\n AND \"heartbeat_runs\".\"runner_profile_json\"->'nativeWorkspaceSync'->>'state' = 'prepared'\n AND \"heartbeat_runs\".\"result_json\"->'nativeCancellation'->>'dispatchState' = 'acknowledged'\n AND (\"heartbeat_runs\".\"result_json\"->'workspaceRestoreRecovery'->>'schema' = 'paperclip.workspace-restore-recovery.v1'\n AND \"heartbeat_runs\".\"result_json\"->>'workspaceRestoreFailure' IN ('restore_permission_denied', 'restore_lock_timeout', 'restore_unsafe_archive', 'restore_failed')) IS NOT TRUE", + "where": "\"heartbeat_runs\".\"runtime_mode\" = 'native' AND \"heartbeat_runs\".\"status\" = 'cancelled'\n AND \"heartbeat_runs\".\"runner_profile_json\"->'nativeWorkspaceSync'->>'state' = 'prepared'\n AND \"heartbeat_runs\".\"result_json\"->'nativeCancellation'->>'dispatchState' = 'acknowledged'\n AND (\"heartbeat_runs\".\"result_json\"->'workspaceRestoreRecovery'->>'schema' = 'paperclip.workspace-restore-recovery.v1'\n AND \"heartbeat_runs\".\"result_json\"->>'workspaceRestoreFailure' IN ('restore_permission_denied', 'restore_lock_timeout', 'restore_unsafe_archive', 'restore_invalid_metadata', 'restore_failed')) IS NOT TRUE", "concurrently": false, "method": "btree", "with": {} @@ -51261,6 +51574,195 @@ "checkConstraints": {}, "isRLSEnabled": false }, + "public.sandbox_creation_operations": { + "name": "sandbox_creation_operations", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true + }, + "company_id": { + "name": "company_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "environment_id": { + "name": "environment_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "original_environment_id": { + "name": "original_environment_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "run_id": { + "name": "run_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "request_key": { + "name": "request_key", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'run'" + }, + "issue_id": { + "name": "issue_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "execution_workspace_id": { + "name": "execution_workspace_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "provider": { + "name": "provider", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "plugin_id": { + "name": "plugin_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "driver_key": { + "name": "driver_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "state": { + "name": "state", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'pending'" + }, + "config": { + "name": "config", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "metadata": { + "name": "metadata", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "lease_id": { + "name": "lease_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "next_attempt_at": { + "name": "next_attempt_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "sandbox_creation_operations_pending_deadline_idx": { + "name": "sandbox_creation_operations_pending_deadline_idx", + "columns": [ + { + "expression": "next_attempt_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"sandbox_creation_operations\".\"state\" = 'pending'", + "concurrently": false, + "method": "btree", + "with": {} + }, + "sandbox_creation_operations_owner_idx": { + "name": "sandbox_creation_operations_owner_idx", + "columns": [ + { + "expression": "company_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "original_environment_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "run_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "request_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "sandbox_creation_operations_environment_id_environments_id_fk": { + "name": "sandbox_creation_operations_environment_id_environments_id_fk", + "tableFrom": "sandbox_creation_operations", + "tableTo": "environments", + "columnsFrom": [ + "environment_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, "public.secret_access_events": { "name": "secret_access_events", "schema": "", @@ -52268,6 +52770,12 @@ "primaryKey": false, "notNull": false }, + "next_eval_reason": { + "name": "next_eval_reason", + "type": "text", + "primaryKey": false, + "notNull": false + }, "next_eval_at": { "name": "next_eval_at", "type": "timestamp with time zone", diff --git a/packages/db/src/migrations/meta/_journal.json b/packages/db/src/migrations/meta/_journal.json index 47d9d449f1a..3cc42512d8c 100644 --- a/packages/db/src/migrations/meta/_journal.json +++ b/packages/db/src/migrations/meta/_journal.json @@ -2920,6 +2920,13 @@ "when": 1793395364937, "tag": "0418_curved_lady_mastermind", "breakpoints": true + }, + { + "idx": 419, + "version": "7", + "when": 1793402408013, + "tag": "0419_moaning_vulture", + "breakpoints": true } ] } \ No newline at end of file diff --git a/packages/db/src/schema/status_cards.ts b/packages/db/src/schema/status_cards.ts index 06efaa0368b..fd18d22fb8b 100644 --- a/packages/db/src/schema/status_cards.ts +++ b/packages/db/src/schema/status_cards.ts @@ -56,6 +56,9 @@ export const statusCards = pgTable( lastModel: text("last_model"), generatingIssueId: uuid("generating_issue_id").references(() => issues.id, { onDelete: "set null" }), failureReason: text("failure_reason"), + // Internal deadline ownership; null denotes a row written before provenance + // was recorded. Membership may be rechecked at startup, retry/policy may not. + nextEvalReason: text("next_eval_reason").$type<"membership" | "policy" | "retry" | "refresh">(), nextEvalAt: timestamp("next_eval_at", { withTimezone: true }), archivedAt: timestamp("archived_at", { withTimezone: true }), archivedByUserId: text("archived_by_user_id"), diff --git a/server/src/__tests__/status-card-evaluation-work.test.ts b/server/src/__tests__/status-card-evaluation-work.test.ts index 4f369cd76e9..4e00e326855 100644 --- a/server/src/__tests__/status-card-evaluation-work.test.ts +++ b/server/src/__tests__/status-card-evaluation-work.test.ts @@ -3,7 +3,7 @@ import { documentService } from "../services/documents.js"; import { randomUUID } from "node:crypto"; import { eq, sql } from "drizzle-orm"; import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; -import { activityLog, agentWakeupRequests, agents, approvals, budgetIncidents, budgetPolicies, costEvents, companies, companyMemberships, createDb, documents, +import { activityLog, agentWakeupRequests, agents, approvals, budgetIncidents, budgetPolicies, costEvents, companies, companyMemberships, createDb, documents, withDedicatedDbConnection, heartbeatRuns, instanceSettings, issueComments, issues, statusCards, statusCardUpdates, documentRevisions } from "@paperclipai/db"; import { companySearchQuerySchema, statusCardRefreshPolicySchema } from "@paperclipai/shared"; import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js"; @@ -188,6 +188,86 @@ const support = external ? { supported: true } : await getEmbeddedPostgresTestSu .toHaveLength(1); }); + it("finishes generation and writeback with a one-connection pool and releases dedicated locks", async () => { + vi.useRealTimers(); + const previousDb = db; + db = createDb(external ?? temporary!.connectionString, { maxConnections: 1, idleTimeoutSeconds: 0, maxLifetimeSeconds: 0 }); + try { + const created = await card("reactive", { ...policy(), debounceSeconds: 1 }); + const countLocks = async () => Number((await db.execute<{ count: string }>(sql` + select count(*) from pg_stat_activity where datname = current_database() + and application_name = 'paperclip-workspace-finalization-lock'`))[0]!.count); + const baseline = await countLocks(); + await issueService(db).create(companyId, { title: "Pool-one blocker", status: "blocked", priority: "medium" }); + const service = statusCardEvaluationService(db, { wakeup }); + const acquired = Promise.withResolvers(), release = Promise.withResolvers(); + const holder = withDedicatedDbConnection(db, connection => connection.transaction(async tx => { + await tx.execute(sql`select pg_advisory_xact_lock(hashtextextended(${created.id}, 0))`); + acquired.resolve(); await release.promise; + })); + await acquired.promise; + try { + await service.run(new AbortController().signal); + expect(wakeup).not.toHaveBeenCalled(); + expect(await service.next()).toBeGreaterThan(Date.now()); + } finally { release.resolve(); await holder; } + await service.run(new AbortController().signal); + expect((await statusCardService(db).getById(created.id))!.pendingChangeCount).toBeGreaterThan(0); + await vi.waitFor(async () => { + await service.run(new AbortController().signal); + expect(wakeup).toHaveBeenCalledOnce(); + }, { timeout: 5000, interval: 100 }); + await writeback(created.id, (await statusCardService(db).getById(created.id))!.generatingIssueId!); + await service.run(new AbortController().signal); + expect(await service.next()).toBeNull(); + expect(idleWorkSnapshot().active).toBe(0); + await vi.waitFor(async () => expect(await countLocks()).toBe(baseline)); + // Failure exits also release the physical connection and advisory lock. + await db.update(statusCards).set({ queries: [{ scope: "invalid" } as never], nextEvalAt: new Date() }).where(eq(statusCards.id, created.id)); + await service.run(new AbortController().signal); + expect((await statusCardService(db).getById(created.id))!.nextEvalReason).toBe("retry"); + await vi.waitFor(async () => expect(await countLocks()).toBe(baseline)); + await withDedicatedDbConnection(db, connection => connection.transaction(async tx => { + expect((await tx.execute<{ acquired: boolean }>(sql`select pg_try_advisory_xact_lock(hashtextextended(${created.id}, 0)) as acquired`))[0]!.acquired).toBe(true); + })); + } finally { await db.$client.end({ timeout: 1 }); db = previousDb; } + }, 15_000); + + it.each([false, true])("recovers a lost committed hint before a future membership deadline (legacy=%s)", async legacy => { + await issueService(db).create(companyId, { title: "Original blocker", status: "blocked", priority: "medium" }); + compiledQuery = companySearchQuerySchema.parse({ scope: "issues", status: ["blocked"], updatedWithin: "1d" }); + const created = await card(); + expect(created.nextEvalReason).toBe("membership"); + const oldExpiry = created.nextEvalAt!.getTime(); + // No coordinator exists to receive this committed source hint. A fresh + // service has no in-memory hint and must reconstruct from durable truth. + await issueService(db).create(companyId, { title: "Committed before crash", status: "blocked", priority: "medium" }); + if (legacy) await db.update(statusCards).set({ nextEvalReason: null }).where(eq(statusCards.id, created.id)); + const restarted = statusCardEvaluationService(db, { wakeup }); + await restarted.run(new AbortController().signal); + const pending = (await statusCardService(db).getById(created.id))!; + expect(pending.pendingChangeCount).toBe(1); + expect(pending.nextEvalReason).toBe("policy"); + expect(pending.nextEvalAt!.getTime()).toBeLessThan(oldExpiry); + expect(wakeup).not.toHaveBeenCalled(); + vi.setSystemTime(pending.nextEvalAt!); await restarted.run(new AbortController().signal); + expect(wakeup).toHaveBeenCalledOnce(); + await writeback(created.id, (await statusCardService(db).getById(created.id))!.generatingIssueId!); + expect((await statusCardService(db).getById(created.id))!.nextEvalReason).toBe("membership"); + }); + + it("preserves a legacy ambiguous five-minute claim without extending it on restart", async () => { + const created = await card(); + const retry = new Date(Date.now() + 240_000); + await db.update(statusCards).set({ nextEvalAt: retry, nextEvalReason: null }).where(eq(statusCards.id, created.id)); + await issueService(db).create(companyId, { title: "Unconsumed source", status: "blocked", priority: "medium" }); + const restarted = statusCardEvaluationService(db, { wakeup }); + await restarted.run(new AbortController().signal); + expect(await restarted.next()).toBe(retry.getTime()); + vi.setSystemTime(retry); await restarted.run(new AbortController().signal); + expect((await statusCardService(db).getById(created.id))!.pendingChangeCount).toBeGreaterThan(0); + }); + it.each(["debounce", "hours", "rate", "budget"] as const)("restores pending %s eligibility after restart without advancing it or acting early", async gate => { vi.setSystemTime(new Date("2026-11-02T18:00:00Z")); const refreshPolicy = { ...policy(), ...(gate === "hours" ? { activeHours: { start: "09:00", end: "17:00", timezone: "UTC" } } : {}), @@ -335,8 +415,12 @@ const support = external ? { supported: true } : await getEmbeddedPostgresTestSu await issueService(db).create(companyId, { title: "Assignment blocker", status: "blocked", priority: "medium" }); await vi.waitFor(async () => expect((await statusCardService(db).getById(created.id))?.pendingChangeCount).toBeGreaterThan(0)); await vi.advanceTimersByTimeAsync(60_000); await settled(); + await vi.waitFor(async () => { + await vi.advanceTimersByTimeAsync(1000); + expect((await statusCardService(db).getById(created.id))!.state).toBe("error"); + }); const failed = (await statusCardService(db).getById(created.id))!; - expect(failed).toMatchObject({ state: "error", generatingIssueId: null, nextEvalAt: null }); + expect(failed).toMatchObject({ state: "error", generatingIssueId: null, nextEvalAt: null, nextEvalReason: null }); expect(errors.length).toBeGreaterThan(0); const history = await db.select().from(statusCardUpdates).where(eq(statusCardUpdates.cardId, created.id)); const ledger = history.find(row => row.status === "failed")!; diff --git a/server/src/__tests__/status-cards.test.ts b/server/src/__tests__/status-cards.test.ts index 0c24b05dd70..05e1bba48e5 100644 --- a/server/src/__tests__/status-cards.test.ts +++ b/server/src/__tests__/status-cards.test.ts @@ -192,6 +192,7 @@ describeEmbeddedPostgres("status card routes", () => { .send({ title: "Launch health", titlePinned: true }); expect(patched.status).toBe(200); expect(patched.body).toMatchObject({ title: "Launch health", titlePinned: true }); + expect(patched.body).not.toHaveProperty("nextEvalReason"); const scheduled = await request(app) .patch(`/api/status-cards/${created.body.id}`) @@ -1152,6 +1153,7 @@ describeEmbeddedPostgres("status card routes", () => { }); expect(summaryWrite.status).toBe(200); expect(summaryWrite.body.card).toMatchObject({ state: "active", queryVersion: 1, generatingIssueId: null }); + expect(summaryWrite.body.card).not.toHaveProperty("nextEvalReason"); expect(summaryWrite.body.document.latestBody).toContain("**Decide:**"); expect(await db.select().from(statusCardUpdates).then((rows) => rows.find((row) => row.kind === "full"))).toMatchObject({ status: "ok", finishedAt: expect.any(Date), inputTokens: 5200, outputTokens: 980 }); diff --git a/server/src/routes/status-cards.ts b/server/src/routes/status-cards.ts index da4f91680bb..d21616b7cf2 100644 --- a/server/src/routes/status-cards.ts +++ b/server/src/routes/status-cards.ts @@ -16,6 +16,12 @@ import { accessService, heartbeatService, instanceSettingsService, issueService, import { queueIssueAssignmentWakeup, type IssueAssignmentWakeupDeps } from "../services/issue-assignment-wakeup.js"; import { assertCompanyAccess, getAccessibleResource, getActorInfo, hasCompanyAccess } from "./authz.js"; +// Deadline provenance belongs to scheduling, not the public card contract. +function publicCard(card: T) { + const { nextEvalReason: _reason, ...result } = card; + return result; +} + export function statusCardRoutes(db: Db, opts: { heartbeat?: IssueAssignmentWakeupDeps } = {}) { const router = Router(); const access = accessService(db); @@ -144,9 +150,9 @@ export function statusCardRoutes(db: Db, opts: { heartbeat?: IssueAssignmentWake assertCompanyAccess(req, companyId); await assertStatusCardsEnabled(); const query = listStatusCardsQuerySchema.parse(req.query); - res.json(await service.list(companyId, query.archived, { + res.json((await service.list(companyId, query.archived, { issueReadCondition: await issueReadSqlCondition(db, req.actor), - })); + })).map(publicCard)); }); router.post("/companies/:companyId/status-cards", validate(createStatusCardSchema), async (req, res) => { @@ -162,7 +168,7 @@ export function statusCardRoutes(db: Db, opts: { heartbeat?: IssueAssignmentWake try { const compile = await enqueueCompile(req, card.id); await logMutation(req, companyId, "status_card.created", card.id, { state: card.state }); - res.status(201).json(compile.card); + res.status(201).json(publicCard(compile.card)); } catch (error) { await service.remove(card.id); throw error; @@ -173,9 +179,9 @@ export function statusCardRoutes(db: Db, opts: { heartbeat?: IssueAssignmentWake await assertStatusCardsEnabled(); const card = await getAccessibleResource(req, res, service.getById(req.params.id as string), "Status card not found"); if (!card) return; - res.json(await service.hydrate(card, { + res.json(publicCard(await service.hydrate(card, { issueReadCondition: await issueReadSqlCondition(db, req.actor), - })); + }))); }); router.patch("/status-cards/:id", validate(patchStatusCardSchema), async (req, res) => { @@ -197,7 +203,7 @@ export function statusCardRoutes(db: Db, opts: { heartbeat?: IssueAssignmentWake fields: Object.keys(req.body), archived: Boolean(updated.archivedAt), }); - res.json(compile?.card ?? restore?.card ?? updated); + res.json(publicCard(compile?.card ?? restore?.card ?? updated)); }); router.delete("/status-cards/:id", async (req, res) => { @@ -234,7 +240,7 @@ export function statusCardRoutes(db: Db, opts: { heartbeat?: IssueAssignmentWake generatingIssueId: result.generatingIssue.id, alreadyGenerating: result.alreadyGenerating, }); - res.status(result.alreadyGenerating ? 200 : 202).json(result); + res.status(result.alreadyGenerating ? 200 : 202).json({ ...result, card: publicCard(result.card) }); }); router.post("/status-cards/:id/refresh", validate(refreshStatusCardSchema), async (req, res) => { @@ -249,7 +255,7 @@ export function statusCardRoutes(db: Db, opts: { heartbeat?: IssueAssignmentWake alreadyGenerating: result.alreadyGenerating, enqueued: result.enqueued, }); - res.status(result.enqueued && !result.alreadyGenerating ? 202 : 200).json(result); + res.status(result.enqueued && !result.alreadyGenerating ? 202 : 200).json({ ...result, card: publicCard(result.card) }); }); router.get("/status-cards/:id/dry-run", async (req, res) => { @@ -289,7 +295,7 @@ export function statusCardRoutes(db: Db, opts: { heartbeat?: IssueAssignmentWake generationIssueId: req.body.generationIssueId, changeSummary: req.body.changeSummary, }); - res.json(updated); + res.json(publicCard(updated)); }); router.put("/status-cards/:id/summary", validate(writeStatusCardSummarySchema), async (req, res) => { @@ -309,7 +315,7 @@ export function statusCardRoutes(db: Db, opts: { heartbeat?: IssueAssignmentWake documentId: result.document.id, changeSummary: req.body.changeSummary, }); - res.json(result); + res.json({ ...result, card: publicCard(result.card) }); }); return router; diff --git a/server/src/services/status-card-finalization.ts b/server/src/services/status-card-finalization.ts index 5d7ee62b523..702509be8b4 100644 --- a/server/src/services/status-card-finalization.ts +++ b/server/src/services/status-card-finalization.ts @@ -47,6 +47,7 @@ export async function finalizeStatusCardsForStalledGeneration( failureReason, generatingIssueId: null, nextEvalAt: null, + nextEvalReason: null, updatedAt: now, }) .where( diff --git a/server/src/services/status-cards.ts b/server/src/services/status-cards.ts index 19e7ac75140..f9e6372ea2e 100644 --- a/server/src/services/status-cards.ts +++ b/server/src/services/status-cards.ts @@ -1,6 +1,6 @@ import { notifyStatusCardEvaluation } from "./status-card-notifications.js"; import { createHash } from "node:crypto"; -import { and, desc, eq, gte, inArray, isNotNull, isNull, lte, ne, or, sql, type SQL } from "drizzle-orm"; +import { and, desc, eq, gt, gte, inArray, isNotNull, isNull, lte, ne, or, sql, type SQL } from "drizzle-orm"; import { agents, companies, @@ -11,6 +11,7 @@ import { issueComments, statusCards, statusCardUpdates, + withDedicatedDbConnection, type Db, } from "@paperclipai/db"; import type { @@ -301,11 +302,12 @@ export function statusCardService( ...(input.refreshPolicy !== undefined ? { refreshPolicy: input.refreshPolicy, + nextEvalReason: "refresh", nextEvalAt: card.archivedAt ? null : input.refreshPolicy.mode === "reactive" && card.queries.length > 0 ? now : nextStatusCardEvaluationAt(input.refreshPolicy, now), } : {}), ...(archiveChanged && input.archived - ? { archivedAt: now, archivedByAgentId: actor.agentId, archivedByUserId: actor.userId, nextEvalAt: null } + ? { archivedAt: now, archivedByAgentId: actor.agentId, archivedByUserId: actor.userId, nextEvalAt: null, nextEvalReason: null } : {}), ...(archiveChanged && !input.archived ? { @@ -315,6 +317,7 @@ export function statusCardService( lastChangeAt: now, lastUpdateRunKind: null, nextEvalAt: card.queries.length > 0 ? now : null, + nextEvalReason: "refresh", } : {}), }; @@ -537,9 +540,10 @@ export function statusCardService( companyId: string, issueIds: string[], options: StatusCardReadOptions = {}, + queryDb: Db = db, ): Promise { if (issueIds.length === 0) return []; - const rows = await db + const rows = await queryDb .select({ id: issues.id, identifier: issues.identifier, @@ -589,11 +593,11 @@ export function statusCardService( return loadIssueSummaries(card.companyId, card.mentionedIssueIds ?? [], options); } - async function executeQueries(card: StatusCardRow, options: StatusCardReadOptions = {}, onNextMembershipChange?: (at: Date | null) => void) { + async function executeQueries(card: StatusCardRow, options: StatusCardReadOptions = {}, onNextMembershipChange?: (at: Date | null) => void, queryDb: Db = db) { const issueMap = new Map(); for (const storedQuery of card.queries) { const query = companySearchQuerySchema.parse(storedQuery); - const response = await searchSvc.search(card.companyId, query, { + const response = await companySearchService(queryDb).search(card.companyId, query, { issueReadCondition: options.issueReadCondition, onNextMembershipChange, }); for (const result of response.results) { @@ -605,12 +609,12 @@ export function statusCardService( const mentioned = await loadIssueSummaries( card.companyId, (card.mentionedIssueIds ?? []).filter((issueId) => !issueMap.has(issueId)), - options, + options, queryDb, ); for (const issue of mentioned) issueMap.set(issue.id, issue); const snapshot = [...issueMap.values()]; if (snapshot.length === 0) return snapshot; - const latestHumanComments = await db + const latestHumanComments = await queryDb .select({ issueId: issueComments.issueId, // The postgres-js driver returns the `max()` aggregate over a timestamp @@ -672,6 +676,7 @@ export function statusCardService( lastChangeAt: null, state: "active", nextEvalAt, + nextEvalReason: nextEvalAt ? card.refreshPolicy.mode === "reactive" ? "membership" : "refresh" : null, }).where(eq(statusCards.id, card.id)).returning()); return { card: next!, generatingIssue: null, alreadyGenerating: false, enqueued: false }; } @@ -699,6 +704,7 @@ export function statusCardService( pendingChangeHash, lastChangeAt, state: decision.action === "pause_budget" ? "paused_budget" : decision.action === "pause_hours" ? "paused_hours" : "active", + nextEvalReason: "policy", nextEvalAt: nextStatusCardPolicyEligibility({ policy: card.refreshPolicy, now, lastChangeAt, completedUpdateTimes: recent.filter(row => row.kind !== "compile" && row.finishedAt).map(row => row.startedAt), @@ -755,6 +761,7 @@ export function statusCardService( lastChangeAt, state: "active", nextEvalAt, + nextEvalReason: nextEvalAt ? card.refreshPolicy.mode === "reactive" ? "membership" : "refresh" : null, failureReason: null, }).where(and(eq(statusCards.id, card.id), isNull(statusCards.archivedAt), or(isNull(statusCards.generatingIssueId), priorGenerationPredicate))).returning()); if (!next) { @@ -788,9 +795,16 @@ export function statusCardService( async function tickDueStatusCards(now = new Date(), reactiveCompanies?: ReadonlySet | "changed", reconstruct = false) { const changedScope = reactiveCompanies === "changed" ? sql`true` : reactiveCompanies?.size ? inArray(statusCards.companyId, [...reactiveCompanies]) : sql`false`; - // Startup restores every disarmed reactive scope. A concurrent committed - // source hint can additionally preempt its company's saved future deadline. - const reactiveScope = or(reconstruct ? isNull(statusCards.nextEvalAt) : undefined, changedScope); + // Startup recovers lost source hints for membership deadlines as well as + // disarmed cards. Saved policy gates and retry leases keep their owner. + // Legacy untagged leases are at most five minutes from their claim time: + // preserve that bounded ambiguity, never a days-away membership deadline. + const startupScope = reconstruct ? or( + isNull(statusCards.nextEvalAt), eq(statusCards.nextEvalReason, "membership"), + and(isNull(statusCards.nextEvalReason), isNull(statusCards.pendingChangeHash), + eq(statusCards.state, "active"), gt(statusCards.nextEvalAt, new Date(now.getTime() + 5 * 60_000))), + ) : undefined; + const reactiveScope = or(startupScope, changedScope); const due = await db.select({ card: statusCards }).from(statusCards) .innerJoin(companies, eq(companies.id, statusCards.companyId)) .where(and(eq(companies.status, "active"), isNull(statusCards.archivedAt), isNull(statusCards.generatingIssueId), @@ -805,13 +819,13 @@ export function statusCardService( // event may preempt a genuine future deadline, so serialize scheduler // evaluation across connections without holding a row lock while the // existing issue/card services commit through their own transactions. - await db.transaction(async lockTx => { + await withDedicatedDbConnection(db, connection => connection.transaction(async lockTx => { const [lock] = await lockTx.execute<{ acquired: boolean }>( sql`select pg_try_advisory_xact_lock(hashtextextended(${candidate.id}, 0)) as acquired`, ); if (!lock?.acquired) { deferredCompanyIds.add(candidate.companyId); return; } const claimUntil = new Date(now.getTime() + 5 * 60 * 1000); - const [claimed] = await mutate(candidate.companyId, tx => tx.update(statusCards).set({ nextEvalAt: claimUntil }) + const [claimed] = await mutate(candidate.companyId, tx => tx.update(statusCards).set({ nextEvalAt: claimUntil, nextEvalReason: "retry" }) .where(and(eq(statusCards.id, candidate.id), isNull(statusCards.archivedAt), isNull(statusCards.generatingIssueId), candidate.nextEvalAt ? eq(statusCards.nextEvalAt, candidate.nextEvalAt) : isNull(statusCards.nextEvalAt))) .returning()); @@ -826,7 +840,7 @@ export function statusCardService( "status card scheduled refresh failed", ); } - }); + })); } return { evaluated, enqueued, deferredCompanyIds }; } @@ -859,7 +873,9 @@ export function statusCardService( // Current watched membership: compiled-query matches plus the issues this // summary mentions. let membershipAt: Date | null = null; - const watchedSnapshot = await executeQueries({ ...current, mentionedIssueIds }, {}, at => { if (at && (!membershipAt || at < membershipAt)) membershipAt = at; }); + // All watched-source reads use this transaction. Borrowing the root + // pool here would deadlock a one-connection installation. + const watchedSnapshot = await executeQueries({ ...current, mentionedIssueIds }, {}, at => { if (at && (!membershipAt || at < membershipAt)) membershipAt = at; }, tx as unknown as Db); const watchedNow = buildStatusCardFingerprint(watchedSnapshot); let snapshot: StatusCardFingerprint; if (payloadFingerprint) { @@ -931,6 +947,9 @@ export function statusCardService( pendingChangeCount: 0, pendingChangeHash: null, lastChangeAt: null, + nextEvalReason: current.refreshPolicy.mode === "reactive" && + filterStatusCardChanges(diffStatusCardFingerprint(snapshot, watchedNow), current.refreshPolicy).length > 0 + ? "refresh" : membershipAt ? "membership" : null, nextEvalAt: current.refreshPolicy.mode === "reactive" && filterStatusCardChanges(diffStatusCardFingerprint(snapshot, watchedNow), current.refreshPolicy).length > 0 ? now : nextEvaluationAt(current, membershipAt, now), From 418f9ad395ae632450a23541736adf883582c377 Mon Sep 17 00:00:00 2001 From: Dotta Date: Fri, 30 Oct 2026 18:29:21 -0500 Subject: [PATCH 50/74] fix(github): preserve terminal receipts across competing failure Guard the failure writer against another receiver's committed success. Prove the rollback race red/green and bounded HTTP body cancellation with unrelated-company progress. Co-Authored-By: Paperclip --- .../2026-10-30-github-connection-sleep.md | 12 ++++ .../github-connection-events.test.ts | 57 +++++++++++++++-- .../__tests__/github-connection-work.test.ts | 63 ++++++++++++++++--- .../src/services/github-connection-events.ts | 3 +- 4 files changed, 121 insertions(+), 14 deletions(-) diff --git a/doc/plans/2026-10-30-github-connection-sleep.md b/doc/plans/2026-10-30-github-connection-sleep.md index 40f83708a71..6d381373b19 100644 --- a/doc/plans/2026-10-30-github-connection-sleep.md +++ b/doc/plans/2026-10-30-github-connection-sleep.md @@ -47,6 +47,18 @@ grant. Binding-only retries use the existing credential refresh lease and curren secret resolver without repeating account/repository discovery. The initial OAuth callback uses the same saved debt and settlement path after connection activation. +Cloud binding calls retain the grant, its connection, and a company share lock. +That admission fence prevents a concurrent disable, revoke, or archive from +finishing before an already-admitted registration call. The API has no remote +revision fence that could safely replace those locks with two short transactions. +Each HTTP request and response body has a physical 15-second abort deadline. +Four calls run in an awaited batch; a failed batch leaves the saved debt and +releases the transaction after all four outcomes settle. Other companies can +continue through separate pool connections. Slow successful multi-batch passes +can delay writes to the same grant/connection and company status changes; the +schema permits up to 100 active and 100 removed installation IDs. This is not an +instance-wide lock or an idle hold. + Credential expiry remains enforced by the existing use path. Scheduled continuity checks run one hour before recorded access expiry or after the existing 30-day continuity window. A successful renewal that leaves the credential short-lived diff --git a/server/src/__tests__/github-connection-events.test.ts b/server/src/__tests__/github-connection-events.test.ts index aa6b1545078..68900aaa0db 100644 --- a/server/src/__tests__/github-connection-events.test.ts +++ b/server/src/__tests__/github-connection-events.test.ts @@ -257,7 +257,8 @@ describeEmbeddedPostgres("GitHub connection event delivery", () => { expect(connector.acknowledgeEvents).toHaveBeenCalledTimes(2); }); - it.each([false, true])("applies installation events once without discarding newer verified access (refreshed: %s)", async (refreshed) => { + it.each([false, true, "rollback-race"])("applies installation events once without discarding newer verified access (mode: %s)", async (mode) => { + const refreshed = mode === true; const companyId = randomUUID(); const applicationId = randomUUID(); const connectionId = randomUUID(); @@ -356,10 +357,54 @@ describeEmbeddedPostgres("GitHub connection event delivery", () => { }); try { const competing = githubConnectionEventService(db, { connector, now: () => currentTime }); - const outcomes = await Promise.all([service.pollOnce(), competing.pollOnce()]); - expect(outcomes.reduce((sum, result) => sum + result.processed, 0)).toBe(1); - expect(outcomes.reduce((sum, result) => sum + result.duplicate, 0)).toBe(1); - expect(outcomes.every(result => result.failed === 0)).toBe(true); + if (mode === "rollback-race") { + let enter!: () => void, release!: () => void, releaseFailure!: () => void; + const entered = new Promise(resolve => { enter = resolve; }); + const mayRollback = new Promise(resolve => { release = resolve; }); + const mayMarkFailure = new Promise(resolve => { releaseFailure = resolve; }); + const transaction = db.transaction.bind(db), update = db.update.bind(db); + let first = true; + vi.spyOn(db, "transaction").mockImplementation(((callback: (tx: unknown) => Promise) => { + const rejectAfterApply = first; first = false; + return transaction(async tx => { + const value = await callback(tx); + if (rejectAfterApply) { enter(); await mayRollback; throw new Error("fixture rollback before receipt commit"); } + return value; + }); + }) as typeof db.transaction); + vi.spyOn(db, "update").mockImplementation(((table: unknown) => { + const builder = update(table as typeof connectionEventDeliveries); + if (table === connectionEventDeliveries) { + const set = builder.set.bind(builder); + builder.set = ((values: { status?: string }) => { + const statement = set(values); + if (values.status === "failed") { + const where = statement.where.bind(statement); + statement.where = (async (...args: Parameters) => { await mayMarkFailure; return where(...args); }) as typeof statement.where; + } + return statement; + }) as typeof builder.set; + } + return builder; + }) as typeof db.update); + const rejected = service.pollOnce(); + const firstResult = Promise.allSettled([rejected]); + try { + await entered; + const succeeding = competing.pollOnce(); + release(); + expect(await succeeding).toMatchObject({ processed: 1, failed: 0 }); + expect((await db.select().from(connectionEventDeliveries).where(eq(connectionEventDeliveries.providerDeliveryId, leasedEvent.id)))[0]!.status).toBe("processed"); + releaseFailure(); + expect(await rejected).toMatchObject({ failed: 1 }); + expect((await db.select().from(connectionEventDeliveries).where(eq(connectionEventDeliveries.providerDeliveryId, leasedEvent.id)))[0]!.status).toBe("processed"); + } finally { release(); releaseFailure(); await firstResult; vi.restoreAllMocks(); } + } else { + const outcomes = await Promise.all([service.pollOnce(), competing.pollOnce()]); + expect(outcomes.reduce((sum, result) => sum + result.processed, 0)).toBe(1); + expect(outcomes.reduce((sum, result) => sum + result.duplicate, 0)).toBe(1); + expect(outcomes.every(result => result.failed === 0)).toBe(true); + } } finally { unsubscribe(); } await health.expectDeadline(currentTime.getTime() + 900_000 - (refreshed ? 60_000 : 0)); await health.expectQuiet(); @@ -380,6 +425,6 @@ describeEmbeddedPostgres("GitHub connection event delivery", () => { connectionEventDeliveries.providerDeliveryId, leasedEvent.id, )); - expect(receipt).toMatchObject({ status: "processed", attempts: 1 }); + expect(receipt).toMatchObject({ status: "processed", attempts: mode === "rollback-race" ? 2 : 1 }); }); }); diff --git a/server/src/__tests__/github-connection-work.test.ts b/server/src/__tests__/github-connection-work.test.ts index a079c130f02..1a268d57605 100644 --- a/server/src/__tests__/github-connection-work.test.ts +++ b/server/src/__tests__/github-connection-work.test.ts @@ -31,10 +31,13 @@ describeDatabase("GitHub coordinator with PostgreSQL and controlled Cloud HTTP ( let db: ReturnType; let temporaryDatabase: Awaited> | undefined; let server: Server, origin: string; + let leaseTimes: number[] = []; let bindings: Array>, leases: number, failBindings: boolean, githubRequests: number; const signing = generateKeyPairSync("ed25519"), sealing = generateKeyPairSync("x25519"); let releaseRefresh: (() => void) | undefined, refreshArrived: (() => void) | undefined; let refreshGate: Promise | undefined; + let heldBindingCompany: string | undefined, bindingGate: Promise | undefined; + let bindingArrived: (() => void) | undefined, bindingClosed: (() => void) | undefined; const realFetch = globalThis.fetch; let companyIds: string[] = []; beforeAll(async () => { @@ -70,11 +73,16 @@ describeDatabase("GitHub coordinator with PostgreSQL and controlled Cloud HTTP ( res.setHeader("content-type", "application/json"); res.end(JSON.stringify({ sealed: { v: 1, alg: algorithm, purpose, provider: "github", profile: "github.code", epk: epk.toString("base64url"), iv: iv.toString("base64url"), ct: ct.toString("base64url") } })); return; } if (req.url!.endsWith("/events/lease")) { - leases++; res.setHeader("content-type", "application/json"); + leases++; leaseTimes.push(Date.now()); res.setHeader("content-type", "application/json"); res.end(JSON.stringify({ events: [], ...(claims.ewv === 1 ? { eventWakeVersion: 1 } : {}) })); return; } expect(claims.sh).toBe(createHash("sha256").update(body.binding).digest("base64url")); bindings.push(JSON.parse(body.binding)); + if (claims.cid === heldBindingCompany) { + res.writeHead(200, { "content-type": "application/json" }); res.write("{"); + res.once("close", () => bindingClosed?.()); bindingArrived?.(); await bindingGate; + res.end("}"); return; + } res.writeHead(failBindings ? 503 : 200, { "content-type": "application/json" }).end("{}"); return; } githubRequests++; @@ -94,7 +102,7 @@ describeDatabase("GitHub coordinator with PostgreSQL and controlled Cloud HTTP ( afterAll(async () => { server.close(); await once(server, "close"); await db.$client.end({ timeout: 1 }); await temporaryDatabase?.cleanup(); }); async function fixture(ids: string[] = []) { - bindings = []; leases = 0; failBindings = false; githubRequests = 0; + bindings = []; leases = 0; leaseTimes = []; failBindings = false; githubRequests = 0; const companyId = randomUUID(); companyIds.push(companyId); const [company] = await db.insert(companies).values({ id: companyId, name: "GitHub work test", issuePrefix: `GW${companyId.replaceAll("-", "").slice(0, 6).toUpperCase()}` }).returning(); const [app] = await db.insert(toolApplications).values({ companyId, applicationKey: randomUUID(), name: "GitHub", type: "mcp_server", status: "active" }).returning(); @@ -219,6 +227,39 @@ describeDatabase("GitHub coordinator with PostgreSQL and controlled Cloud HTTP ( } finally { releaseRefresh?.(); await result; refreshGate = undefined; refreshArrived = undefined; vi.unstubAllEnvs(); } }); + it("bounds a held Cloud body without releasing revocation admission, while another company progresses and restart retains removal debt", async () => { + const f = await fixture([]), other = await fixture([]); + for (const row of [f, other]) await db.update(connectionGrants).set({ providerTenant: { + ...row.grant.providerTenant, github: { ...row.grant.providerTenant!.github!, bindingSync: { id: randomUUID(), removeInstallationIds: ["101"] } }, + } }).where(eq(connectionGrants.id, row.grant.id)); + const latest = async (id: string) => (await db.select().from(connectionGrants).where(eq(connectionGrants.id, id)))[0]!; + heldBindingCompany = f.company.id; + let release!: () => void; + bindingGate = new Promise(resolve => { release = resolve; }); + const arrived = new Promise(resolve => { bindingArrived = resolve; }); + const closed = new Promise(resolve => { bindingClosed = resolve; }); + const timeout = AbortSignal.timeout.bind(AbortSignal); + const deadlines = vi.spyOn(AbortSignal, "timeout").mockImplementation(ms => timeout(ms === 15_000 ? 3_000 : ms)); + let completed = false; + const attempt = f.service.syncGitHubGrantBindings(f.connection, await latest(f.grant.id)); + const outcome = attempt.then(value => ({ value, error: null }), error => ({ value: null, error })).finally(() => { completed = true; }); + try { + await arrived; + const otherResult = await other.service.syncGitHubGrantBindings(other.connection, await latest(other.grant.id)); + expect(otherResult.providerTenant!.github!.bindingSync).toBeUndefined(); + await db.update(companies).set({ name: "Unrelated company changed while body held" }).where(eq(companies.id, other.company.id)); + expect(completed).toBe(false); + expect((await outcome).error).toMatchObject({ code: "CONNECTOR_BAD_RESPONSE" }); + await closed; + expect(deadlines).toHaveBeenCalledWith(15_000); + expect((await latest(f.grant.id)).providerTenant!.github!.bindingSync!.removeInstallationIds).toEqual(["101"]); + heldBindingCompany = undefined; release(); + const restarted = createGitHubGrantContinuity(db, f.dependencies); + expect(await restarted.sweepGitHubConnectionContinuity(f.company.id)).toMatchObject({ due: 1, refreshed: 1, failed: 0 }); + expect((await latest(f.grant.id)).providerTenant!.github!.bindingSync).toBeUndefined(); + } finally { heldBindingCompany = undefined; release(); await outcome; bindingGate = undefined; bindingArrived = undefined; bindingClosed = undefined; } + }); + it("lease acquire and cleanup cannot replace a concurrent webhook metadata revision", async () => { const f = await fixture(["101"]); let interleave = true; @@ -257,7 +298,9 @@ describeDatabase("GitHub coordinator with PostgreSQL and controlled Cloud HTTP ( const [unrelatedConnection] = await db.update(toolConnections).set({ config: {}, transportConfig: {}, transport: "rest_api", authKind: "none", credentialRefs: [], credentialSecretRefs: [] }).where(eq(toolConnections.id, other.connection.id)).returning(); await db.update(connectionGrants).set({ credentialSecretRefs: [] }).where(eq(connectionGrants.id, other.grant.id)); const api = toolAccessService(db); - const load = vi.spyOn(f.service, "githubConnectionContinuitySchedule"); + const loadTimes: number[] = []; + const schedule = f.service.githubConnectionContinuitySchedule.bind(f.service); + const load = vi.spyOn(f.service, "githubConnectionContinuitySchedule").mockImplementation(companyId => { loadTimes.push(Date.now()); return schedule(companyId); }); const sweep = vi.spyOn(f.service, "sweepGitHubConnectionContinuity"); const coordinator = createDeliveryWorkCoordinator({ owner: db, canRun: () => true }); const continuity = registerGitHubConnectionContinuityWork(db, coordinator, f.service, () => true); @@ -266,7 +309,7 @@ describeDatabase("GitHub coordinator with PostgreSQL and controlled Cloud HTTP ( await Promise.all([continuity.ready, events.ready]); await vi.waitFor(() => expect(leases).toBe(1)); await vi.waitFor(() => expect(load).toHaveBeenCalledTimes(1)); - load.mockClear(); sweep.mockClear(); + load.mockClear(); loadTimes.length = 0; sweep.mockClear(); for (let i = 0; i < 30; i++) { await db.transaction(async tx => { await notifyToolConnectionHealth(tx, other.connection); await notifyChatEndpointConfiguration(tx, { id: randomUUID(), provider: "discord" }); }); await api.updateConnection(unrelatedConnection!.id, { name: `Unrelated patch ${i}` }); @@ -274,13 +317,19 @@ describeDatabase("GitHub coordinator with PostgreSQL and controlled Cloud HTTP ( await vi.advanceTimersByTimeAsync(10_000); expect(load).not.toHaveBeenCalled(); expect(sweep).not.toHaveBeenCalled(); expect(leases).toBe(1); for (let index = 0; index < 30; index++) await db.transaction(async tx => { await notifyGitHubCredentialChange(tx, f.connection); }); - expect(leases).toBe(1); + expect(leases).toBeLessThanOrEqual(2); await vi.advanceTimersByTimeAsync(6_000); await vi.waitFor(() => expect(load).toHaveBeenCalledWith(f.company.id)); expect(load.mock.calls.every(([companyId]) => companyId === f.company.id)).toBe(true); expect(sweep).not.toHaveBeenCalled(); - await vi.waitFor(() => expect(leases).toBe(2)); - expect(load).toHaveBeenCalledTimes(1); + await vi.waitFor(() => expect(leases).toBeGreaterThanOrEqual(2)); + // The first hint can run immediately; changes during it get one spaced pass. + expect(leases).toBeLessThanOrEqual(3); expect(load.mock.calls.length).toBeLessThanOrEqual(2); + if (loadTimes.length === 2) expect(loadTimes[1]! - loadTimes[0]!).toBeGreaterThanOrEqual(5_000); + if (leaseTimes.length === 3) expect(leaseTimes[2]! - leaseTimes[1]!).toBeGreaterThanOrEqual(5_000); + const quiet = { leases, loads: load.mock.calls.length }; + await vi.advanceTimersByTimeAsync(60_000); + expect({ leases, loads: load.mock.calls.length }).toEqual(quiet); } finally { await events.stop(); await continuity.stop(); await coordinator.stop(); } }); diff --git a/server/src/services/github-connection-events.ts b/server/src/services/github-connection-events.ts index 5db8585ffcf..b0cc2f2fdd4 100644 --- a/server/src/services/github-connection-events.ts +++ b/server/src/services/github-connection-events.ts @@ -10,7 +10,7 @@ import { toolConnections, type Db, } from "@paperclipai/db"; -import { and, eq, sql } from "drizzle-orm"; +import { and, eq, ne, sql } from "drizzle-orm"; import { logActivity, publishActivity, @@ -456,6 +456,7 @@ export function githubConnectionEventService( eq(connectionEventDeliveries.companyId, companyId), eq(connectionEventDeliveries.provider, event.provider), eq(connectionEventDeliveries.providerDeliveryId, event.id), + ne(connectionEventDeliveries.status, "processed"), )); throw error; } From 91261d5a038d4d009c01ba102da4a00921caa175 Mon Sep 17 00:00:00 2001 From: Dotta <34892728+cryppadotta@users.noreply.github.com> Date: Fri, 30 Oct 2026 18:36:36 -0500 Subject: [PATCH 51/74] fix(server): preserve budget history for deleted resources (#18389) ## Thinking Path > - Paperclip is the open source app people use to manage AI agents for work. > - Budget views show saved policies, incidents and spend for each company. > - A policy can remain after its agent or project has been deleted. > - The overview currently stops when it cannot resolve that old resource. > - A current agent's budget then becomes unreadable during normal setup. > - This pull request preserves the historical row in display summaries. > - Admission and policy changes still require a real company-owned resource. ## Linked Issues or Issue Description **What happened?** The budget overview throws `Agent not found` when a saved policy references a deleted agent. One old resource prevents the company from reading its current budgets. The same problem affects historical project policies and incident summaries. **Expected behavior** The overview should retain historical policy and incident IDs and spend. It should display `Deleted agent` or `Deleted project` for a missing resource. Read access must not grant permission to execute work or change a budget for that resource. **Steps to reproduce** 1. Create an agent and a budget policy for it. 2. Permanently delete the agent after its other dependencies have been cleared. 3. Read the company's budget overview. The existing service throws instead of returning its remaining current and historical rows. **Paperclip version or commit** The isolated repair starts from merged commit `d9354bbfc35ef74333e90e596ce21dc8e59149b9` and integrates the exact merged baseline test fix `8595efc20bda3ce20a9fbbf34a5d156a6fcf6d57`. It also integrates the actual merged Antigravity auth fix `18b1637b6371a6a6adade857e2a73654ee626fef` (#18254). The diff against the selected merged parent remains two budget files. **Related work** Earlier reports and implementations: #11058 and #4333. This extracts the product prerequisite from #18351 so it can qualify independently of that journey's other dependencies. The change contains only the budget service repair and four database regression cases. ## What Changed - Resolve deleted agent and project scopes only when building display summaries. Preserve their original IDs, policies, incidents and accounting records. - Reject a saved policy or incident that resolves to another company's resource. Propagate other read failures. - Keep missing-company resolution, work admission and policy changes strict. Validate company, policy and incident scope bindings before either incident action. A budget raise requires a live company-owned resource. Historical dismissal may proceed only for a genuinely deleted owned agent/project, and must reject live foreign or mismatched scopes. - Add four canonical PostgreSQL regression cases: deleted agent history, deleted project history, missing-scope admission and budget raises with safe historical dismissal, and foreign/mismatched references. The history cases also verify current ledger spend and unchanged stored rows. ## Verification - Real-service RED/GREEN against official PGlite with every canonical migration: the original overview throws for deleted history; the repair returns the historical IDs, incidents and spend while retaining current budgets. - Mutation regression RED/GREEN: missing-resource budget raises must reject before writes; owned deleted-scope dismissal must reject its linked pending approval, retain policy/agent/ledger data and incident history, and remove the incident from the active list. Live foreign and mismatched scopes reject both actions without mutations. - Current integrated head `b3434f5ad05fd3067379352700c36bfe826179c4`: all four canonical PostgreSQL regressions pass in [server 2/12](https://github.com/paperclipai/paperclip/actions/runs/41142588583/job/124361194077), and all five existing budget-service tests pass in [server 7/12](https://github.com/paperclipai/paperclip/actions/runs/41142588583/job/124361194080). Both use the CI merge against selected parent `18b1637`. - Clean-build declaration typing passes in the current CI typecheck gate. The earlier local source-alias no-emit check also passed, including the canonical actor argument. The local worktree borrows compiled dependencies from a sibling checkout; a later real-service import was blocked by its stale `sandboxCreationOperations` export, so clean-build CI is authoritative for the final integration. No production workaround was made. - `git diff --check` passes. Broad repository typecheck, tests and build run in current-head CI rather than repeated locally. Native local PostgreSQL could not start because this host exhausted its semaphore pool; no global IPC settings or unrelated processes were changed. - One unchanged Runner lane retry was authorized after a probable intermittent close-settlement fixture failure. The [original failure](https://github.com/paperclipai/paperclip/actions/runs/41142588583/job/124360234111) occurred in `omitted-context` cleanup. A [fresh passing Linux lane](https://github.com/paperclipai/paperclip/actions/runs/41142552195/job/124360110386) passed the same case and all 1,998 tests with byte-identical entire Runner and adapter-utils trees, lockfile, root package, scripts and workflows, against the same merged base. The fixture has a five-second close allowance with a 2.5-second suspension reserve; a host stall is not proven. No budget path or changed close input was found. The [single targeted retry](https://github.com/paperclipai/paperclip/actions/runs/41142588583/job/124365812609) passed the same case in 1,034 ms and all 1,998 lane tests; dependent verification also passed. All 57 current PR checks are terminal and green. - The product helper requires all current CI and clean review before merge. Actual QA budget-overview proof will run on the combined final app before hiring qualification. No deployed success is claimed. The older `b4863dd` artifact was never deployed; selected parent `18b1637` adds sandbox/lifecycle changes and migration 0418, so whole-app identity is not claimed. The budget module and relevant budget contracts remain unchanged, and final hiring requires the combined final app with explicit source identities. ## Risks Budget summaries now return historical rows whose resources have been deleted. These rows use explicit display names and preserve their accounting data. They do not create a resource or relax work admission, authorization or budget grants. Historical incident dismissal changes only that incident and its linked approval; it does not resume work or raise a budget. Cross-company references still fail closed. There is no schema migration and no new background work. ## Model Used OpenAI Codex, GPT-6 family. The session does not expose a more specific backend model ID or fixed context window. Used reasoning, repository tools, TypeScript and database test execution. ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above - [x] I have either (a) linked existing issues with `Fixes: #` / `Closes #` / `Refs #` OR (b) described the issue in-PR following the relevant issue template - [x] I have not referenced internal/instance-local Paperclip issues or links (only public GitHub `#NNN` / `github.com/paperclipai/paperclip` URLs) - [x] My branch name describes the change (e.g. `docs/...`, `fix/...`) and contains no internal Paperclip ticket id or instance-derived details - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [x] I have updated relevant documentation to reflect my changes (the PR description documents the display-only history and strict-mutation contract) - [x] I have considered and documented any risks above - [x] All Paperclip CI gates are green - [x] Greptile is 5/5 with no open P2s, recommendations, or follow-ups - [x] I will address all Greptile and reviewer comments before requesting merge --------- Co-authored-by: Paperclip --- server/src/__tests__/budgets-history.test.ts | 109 +++++++++++++++++++ server/src/services/budgets.ts | 24 +++- 2 files changed, 131 insertions(+), 2 deletions(-) create mode 100644 server/src/__tests__/budgets-history.test.ts diff --git a/server/src/__tests__/budgets-history.test.ts b/server/src/__tests__/budgets-history.test.ts new file mode 100644 index 00000000000..5b106329d72 --- /dev/null +++ b/server/src/__tests__/budgets-history.test.ts @@ -0,0 +1,109 @@ +import { randomUUID } from "node:crypto"; +import { eq } from "drizzle-orm"; +import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest"; +import { agents, approvals, budgetIncidents, budgetPolicies, companies, costEvents, createDb, projects, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS } from "@paperclipai/db"; +import { budgetService } from "../services/budgets.js"; +import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; + +vi.mock("../services/activity-log.js", () => ({ logActivity: vi.fn() })); +const support = await getEmbeddedPostgresTestSupport(); + +describe.skipIf(!support.supported)("budget history for deleted scopes", () => { + let db: ReturnType; + let database: Awaited>; + beforeAll(async () => { + database = await startEmbeddedPostgresTestDatabase("paperclip-budget-history-"); + db = createDb(database.connectionString); + }, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS); + afterEach(async () => { + await db.delete(budgetIncidents); await db.delete(approvals); await db.delete(budgetPolicies); await db.delete(costEvents); + await db.delete(projects); await db.delete(agents); await db.delete(companies); + }); + afterAll(async () => { await db?.$client.end({ timeout: 1 }); await database?.cleanup(); }); + + async function fixture() { + const companyId = randomUUID(), agentId = randomUUID(); + await db.insert(companies).values({ id: companyId, name: "History company", issuePrefix: `B${companyId.replace(/-/g, "").slice(0, 6).toUpperCase()}` }); + await db.insert(agents).values({ id: agentId, companyId, name: "Current agent", role: "engineer" }); + const [policy] = await db.insert(budgetPolicies).values({ companyId, scopeType: "agent", scopeId: agentId, + metric: "billed_cents", windowKind: "calendar_month_utc", amount: 300 }).returning(); + await db.insert(costEvents).values({ companyId, agentId, provider: "openai", model: "history-fixture", inputTokens: 1, outputTokens: 1, costCents: 17, occurredAt: new Date() }); + return { companyId, agentId, policy: policy! }; + } + + it.each(["agent", "project"] as const)("keeps current budgets and deleted %s policy/incident history readable without changing stored rows", async scopeType => { + const { companyId, policy } = await fixture(), deletedId = randomUUID(); + if (scopeType === "agent") await db.insert(agents).values({ id: deletedId, companyId, name: "Former agent", role: "engineer" }); + else await db.insert(projects).values({ id: deletedId, companyId, name: "Former project" }); + const [historical] = await db.insert(budgetPolicies).values({ companyId, scopeType, scopeId: deletedId, + metric: "billed_cents", windowKind: "calendar_month_utc", amount: 200 }).returning(); + await db.insert(budgetIncidents).values({ companyId, policyId: historical!.id, scopeType, scopeId: deletedId, + metric: "billed_cents", windowKind: "calendar_month_utc", windowStart: new Date("2020-01-01"), windowEnd: new Date("2020-02-01"), + thresholdType: "hard", amountLimit: 200, amountObserved: 201 }); + if (scopeType === "agent") await db.delete(agents).where(eq(agents.id, deletedId)); + else await db.delete(projects).where(eq(projects.id, deletedId)); + const policiesBefore = await db.select().from(budgetPolicies), incidentsBefore = await db.select().from(budgetIncidents), costsBefore = await db.select().from(costEvents); + + const overview = await budgetService(db).overview(companyId); + expect(overview.policies).toHaveLength(2); + expect(overview.policies.find(row => row.policyId === policy.id)).toMatchObject({ companyId, scopeName: "Current agent", amount: 300, observedAmount: 17 }); + expect(overview.policies.find(row => row.policyId === historical!.id)).toMatchObject({ companyId, scopeType, scopeId: deletedId, scopeName: `Deleted ${scopeType}`, amount: 200 }); + expect(overview.activeIncidents).toMatchObject([{ companyId, policyId: historical!.id, scopeId: deletedId, scopeName: `Deleted ${scopeType}`, status: "open", amountObserved: 201 }]); + expect(await db.select().from(budgetPolicies)).toEqual(policiesBefore); + expect(await db.select().from(budgetIncidents)).toEqual(incidentsBefore); + expect(await db.select().from(costEvents)).toEqual(costsBefore); + }); + it("rejects missing-scope budget grants but can dismiss their historical incidents", async () => { + const { companyId, agentId } = await fixture(), missingId = randomUUID(), service = budgetService(db); + await expect(service.evaluateAdmission(companyId, missingId)).rejects.toThrow("Agent not found"); + await expect(service.evaluateAdmission(companyId, agentId, { projectId: missingId })).rejects.toThrow("Project not found"); + for (const scopeType of ["agent", "project"] as const) { + await expect(service.upsertPolicy(companyId, { scopeType, scopeId: missingId, amount: 100 }, "board")).rejects.toThrow(`${scopeType === "agent" ? "Agent" : "Project"} not found`); + const [policy] = await db.insert(budgetPolicies).values({ companyId, scopeType, scopeId: missingId, + windowKind: "calendar_month_utc", amount: 100 }).returning(); + const [approval] = await db.insert(approvals).values({ companyId, type: "budget_override_required", payload: {} }).returning(); + const [incident] = await db.insert(budgetIncidents).values({ companyId, policyId: policy!.id, approvalId: approval!.id, scopeType, scopeId: missingId, + metric: "billed_cents", windowKind: "calendar_month_utc", windowStart: new Date("2020-01-01"), windowEnd: new Date("2020-02-01"), + thresholdType: "hard", amountLimit: 100, amountObserved: 101 }).returning(); + const before = { policies: await db.select().from(budgetPolicies), incidents: await db.select().from(budgetIncidents), + agents: await db.select().from(agents), costs: await db.select().from(costEvents), approvals: await db.select().from(approvals) }; + await expect(service.resolveIncident(companyId, incident!.id, { action: "raise_budget_and_resume", amount: 200 }, "board")) + .rejects.toThrow(`${scopeType === "agent" ? "Agent" : "Project"} not found`); + expect({ policies: await db.select().from(budgetPolicies), incidents: await db.select().from(budgetIncidents), + agents: await db.select().from(agents), costs: await db.select().from(costEvents), approvals: await db.select().from(approvals) }).toEqual(before); + const dismissed = await service.resolveIncident(companyId, incident!.id, { action: "keep_paused" }, "board"); + expect(dismissed).toMatchObject({ companyId, policyId: policy!.id, scopeId: missingId, scopeName: `Deleted ${scopeType}`, status: "dismissed" }); + const { status: _status, resolvedAt: _resolvedAt, updatedAt: _updatedAt, ...preserved } = incident!; + const [saved] = await db.select().from(budgetIncidents).where(eq(budgetIncidents.id, incident!.id)); + expect(saved).toMatchObject({ ...preserved, status: "dismissed", resolvedAt: expect.any(Date) }); + expect((await db.select().from(approvals).where(eq(approvals.id, approval!.id)))[0]).toMatchObject({ status: "rejected" }); + expect(await db.select().from(budgetPolicies)).toEqual(before.policies); + expect(await db.select().from(agents)).toEqual(before.agents); + expect(await db.select().from(costEvents)).toEqual(before.costs); + expect((await service.overview(companyId)).activeIncidents.some(row => row.id === incident!.id)).toBe(false); + } + }); + + it("rejects a corrupted policy pointing to another company's scope instead of exposing its name", async () => { + const { companyId } = await fixture(), foreignCompanyId = randomUUID(), foreignAgentId = randomUUID(); + await db.insert(companies).values({ id: foreignCompanyId, name: "Other company", issuePrefix: `F${foreignCompanyId.replace(/-/g, "").slice(0, 6).toUpperCase()}` }); + await db.insert(agents).values({ id: foreignAgentId, companyId: foreignCompanyId, name: "Private other-company agent", role: "engineer" }); + const [policy] = await db.insert(budgetPolicies).values({ companyId, scopeType: "agent", scopeId: foreignAgentId, windowKind: "calendar_month_utc", amount: 200 }).returning(); + const [incident] = await db.insert(budgetIncidents).values({ companyId, policyId: policy!.id, scopeType: "agent", scopeId: foreignAgentId, + metric: "billed_cents", windowKind: "calendar_month_utc", windowStart: new Date("2020-01-01"), windowEnd: new Date("2020-02-01"), + thresholdType: "hard", amountLimit: 200, amountObserved: 201 }).returning(); + const service = budgetService(db); + await expect(service.overview(companyId)).rejects.toThrow("Budget scope not found"); + for (const scopeId of [foreignAgentId, randomUUID()]) { + await db.update(budgetIncidents).set({ scopeId }).where(eq(budgetIncidents.id, incident!.id)); + const before = { policies: await db.select().from(budgetPolicies), incidents: await db.select().from(budgetIncidents), + agents: await db.select().from(agents), costs: await db.select().from(costEvents) }; + for (const action of ["raise_budget_and_resume", "keep_paused"] as const) { + await expect(service.resolveIncident(companyId, incident!.id, { action, amount: 300 }, "board")).rejects.toThrow("Budget scope not found"); + expect({ policies: await db.select().from(budgetPolicies), incidents: await db.select().from(budgetIncidents), + agents: await db.select().from(agents), costs: await db.select().from(costEvents) }).toEqual(before); + } + } + }); + +}); diff --git a/server/src/services/budgets.ts b/server/src/services/budgets.ts index 6777124ee90..3a15d489f0a 100644 --- a/server/src/services/budgets.ts +++ b/server/src/services/budgets.ts @@ -163,6 +163,19 @@ async function resolveScopeRecord(db: Db, scopeType: BudgetScopeType, scopeId: s }; } +async function resolveSummaryScopeRecord(db: Db, companyId: string, scopeType: BudgetScopeType, scopeId: string): Promise { + let scope: ScopeRecord; + try { + scope = await resolveScopeRecord(db, scopeType, scopeId); + } catch (error) { + if (!(error instanceof HttpError) || error.status !== 404 || (scopeType !== "agent" && scopeType !== "project")) throw error; + return { companyId, name: scopeType === "agent" ? "Deleted agent" : "Deleted project", + paused: false, pauseReason: null, budgetHeld: false }; + } + if (scope.companyId !== companyId) throw notFound("Budget scope not found"); + return scope; +} + // Keep a large dashboard from filling the database pool with queued reads. async function mapBudgetReads(rows: T[], read: (row: T) => Promise): Promise { const result: R[] = []; @@ -395,7 +408,7 @@ export function budgetServiceInTransaction(db: Db, publications: ActivityPublica } async function buildPolicySummary(policy: PolicyRow): Promise { - const scope = await resolveScopeRecord(db, policy.scopeType as BudgetScopeType, policy.scopeId); + const scope = await resolveSummaryScopeRecord(db, policy.companyId, policy.scopeType as BudgetScopeType, policy.scopeId); const observed = await computeObservedSpend(db, policy); const { total: observedAmount, totalExact: observedAmountExact, unpricedEventCount, pendingRunCount } = observed; const { windowStart: start, windowEnd: end } = observed; @@ -560,7 +573,7 @@ export function budgetServiceInTransaction(db: Db, publications: ActivityPublica const approvalStatusById = new Map(approvalRows.map((row) => [row.id, row.status])); return mapBudgetReads(rows, async (row) => { - const scope = await resolveScopeRecord(db, row.scopeType as BudgetScopeType, row.scopeId); + const scope = await resolveSummaryScopeRecord(db, row.companyId, row.scopeType as BudgetScopeType, row.scopeId); return { id: row.id, companyId: row.companyId, @@ -925,6 +938,13 @@ export function budgetServiceInTransaction(db: Db, publications: ActivityPublica if (incident.companyId !== companyId) throw notFound("Budget incident not found"); const policy = await getPolicyRow(incident.policyId); + if (policy.companyId !== companyId || policy.scopeType !== incident.scopeType || policy.scopeId !== incident.scopeId) { + throw notFound("Budget scope not found"); + } + const scope = input.action === "keep_paused" + ? await resolveSummaryScopeRecord(db, companyId, policy.scopeType as BudgetScopeType, policy.scopeId) + : await resolveScopeRecord(db, policy.scopeType as BudgetScopeType, policy.scopeId); + if (scope.companyId !== companyId) throw notFound("Budget scope not found"); if (input.action === "raise_budget_and_resume") { const nextAmount = Math.max(0, Math.floor(input.amount ?? 0)); const observed = await computeObservedSpend(db, policy); From 1353a232f2cd9e5fb58933689e485a9ff1f1480f Mon Sep 17 00:00:00 2001 From: Dotta <34892728+cryppadotta@users.noreply.github.com> Date: Fri, 30 Oct 2026 18:37:17 -0500 Subject: [PATCH 52/74] test(e2e): add a portable active follow-up journey (#18345) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Thinking Path > - Paperclip lets people direct agents while their work is active. > - Product E2E checks must prove that saved user input reaches the intended work. > - The existing active chat story depends on a file in a local agent workspace. > - That setup cannot qualify an attached deployment with remote agent execution. > - This change adds one shared active follow-up journey through the normal task UI and API. > - Independent native operation, saved input, final reply, and cleanup checks make the result inspectable across targets. ## Linked Issues or Issue Description **What existing behavior does this improve?** Product E2E coverage for a user follow-up sent while a native agent is working. **Current behavior** The classic chat story holds work open with a local file. The shared deployment-targeted harness has no active follow-up journey. A running heartbeat alone cannot prove that work was active when input arrived. **Proposed behavior** Run one bounded command through the real native provider. Observe its active execution before and after one follow-up is saved. Require one final reply with the late requirement, correct task and agent identity, an empty input queue, and visible readback after reload. Use the same story on managed and attached targets. ## What Changed - Add a shared `followup-while-running` journey with normal task creation and reply submission. - Require a bound native turn and pending shell execution at the input boundary. Retain a closed receipt with source identity and a row hash. - Accept one steered work run or one normal successor (1–2 native runs). Preserve every other journey's declared run contract. - Reuse the saved native final reply observer, run binding checks, usage collection, target lifecycle and private evidence path. - Accept canonical builtin/process execution events independent of provider tool display names; calibrate against real OpenCode, Codex and ACP mappers. Reject Paperclip semantic tool names across native wrappers/prefixes and unnamed or ambiguous builtin calls. - Recognize only the observed legacy PRP v1 builtin/bash/unknown/readOnly=false command shape and fail promptly when owned work finishes before a live boundary can be observed. - Ready the task page/composer before the observed command boundary, use a bounded 90-second command, and preserve active-after-persistence proof. - Add adversarial oracle and retained-result calibration. Register the journey, include its sources in the definition hash, and document its coverage. - Keep six real browser document-readiness checks and verify shared-deadline behavior with a deterministic clock/state test that rejects timeout-reset mutations. ## Verification Exact current head `e72e20ad111bd94728d38606015d1fdac3c92929` passed [full CI run 41142552195](https://github.com/paperclipai/paperclip/actions/runs/41142552195): all 49 workflow jobs reached accepted terminal results, including typecheck, build, every test lane and aggregate verify. All 59 reported checks are terminal: 55 success, three expected skips and one neutral 5/5 Paperclip review. Snyk passed. [Fresh Greptile review](https://github.com/paperclipai/paperclip/pull/18345#issuecomment-6528913235) explicitly names this exact head and gives 5/5. No review threads remain open. The PR is mergeable. These source gates do not establish a live QA pass at this head. Fresh protected QA at this exact source and the separately recorded deployed application/runtime remains required before primary merge and daily admission. Historical live QA staging **PASS** on qualified commit `af9e091b15b5e24b6b2cc1b424523430cc2ab024`: 6/6 checks, all five phases including cleanup, two native work runs, and the configured model/environment check passed. The [sanitized qualification report](https://pages.paperclip.ing/evals-qa-smoke-41122620146-1/) records that exact measurement. Final integration head `e72e20ad111bd94728d38606015d1fdac3c92929` merges exact actual squash `18b1637b6371a6a6adade857e2a73654ee626fef` from [#18254](https://github.com/paperclipai/paperclip/pull/18254) into previous head `e54d64743964b1a19d165537bebbe62c50389a23`. The merge had no conflicts. All 18 files in this PR's change set, the complete 45-file shared journey directory, and six shared native/evidence helpers retain their exact previous-head Git blobs: 59 scoped files in total. The production clarification helper remains blob `800a747284db19b8484650b6759e8bd1f356839d`. Follow-up 1–2 runs, question 1–3 runs, older exact counts, owned native event checks, active-before/after persistence proof, cleanup/accounting and feature mappings remain unchanged. Source integration head `e55d74491f5bcd99c1455e9ace02bd7ad7319fae` integrates exact actual squash `8595efc20bda3ce20a9fbbf34a5d156a6fcf6d57` from [#18385](https://github.com/paperclipai/paperclip/pull/18385). The merge had no conflicts. All 17 files in this PR's change set and six shared native/evidence helpers retain their exact Git blobs from previous head `162e341cda5b627639b486b4019a1675bb018ea2`. This preserves the follow-up and question readiness checks, feature mappings, six closed follow-up check IDs, immutable before/after active-operation proof, follow-up 1–2 runs, question 1–3 runs and older exact counts. Previously reviewed head `e54d64743964b1a19d165537bebbe62c50389a23` adds a separate test-only reliability repair on top of that source integration. It replaces one scheduler-sensitive browser case with a deterministic Vitest clock and locator-state model that calls the actual production helper. The six other real Chromium checks remain byte-for-byte unchanged and pass **6/6**. Focused unit suites pass **51/51**, including the deterministic deadline case. This is six browser checks plus focused unit coverage, not seven browser checks. The model makes the panel ready at 100 ms, the supplied answer ready at 600 ms, and full valid content ready at 1,300 ms. It requires rejection at the shared 1,000 ms deadline and records immutable rejection state before simulating a 10-second scheduler stall. Both restarted-timeout mutations fail: resetting all waits wrongly resolves at 1,300 ms; resetting matcher timeouts while keeping the final real deadline guard rejects only at 1,300 ms with valid content present. The genuine helper rejects at 1,000 ms with the answer still incomplete. The production helper is unchanged, Git blob `800a747284db19b8484650b6759e8bd1f356839d`. That two-file test-only patch changed no journey, application, run, accounting, model or runtime behavior. Its six-browser/51-unit results are prior-head evidence; the complete patch is unchanged in the new integration. The final source also retains the actual merged question-primary squash `d9354bbfc35ef74333e90e596ce21dc8e59149b9` from [#18346](https://github.com/paperclipai/paperclip/pull/18346). Later commits `409117451da260a03fc28a3107c7808ecbe68f8f` and `ea3e7b1d1bed562351fcf4722c7175001b987b59` bound question document readiness to one campaign deadline and test that waits cannot restart it. Commit `162e341cda5b627639b486b4019a1675bb018ea2` adds question/follow-up report feature mappings and coverage. These files are unchanged by the final merge. The imported upstream source has real application changes: [#18334](https://github.com/paperclipai/paperclip/pull/18334) makes managed skill content durable and extends native skill-test contracts; [#18350](https://github.com/paperclipai/paperclip/pull/18350) requires managed Cloud shared storage and bootstrap prerequisites. #18385 updates TestOps job metadata and repairs their combined maintenance fixture. The new merge additionally imports [#18383](https://github.com/paperclipai/paperclip/pull/18383) at `81ecaf0654fe79a2a2dbda6588f033e39ee610e4`, which bounds failed lifecycle verification; [#18357](https://github.com/paperclipai/paperclip/pull/18357) at `45a3a92e6dc52d20923b9aa34a22fee2ec5443f9`, which persists sandbox-creation ownership and updates cleanup/adapter/plugin contracts; and #18254 at `18b1637b6371a6a6adade857e2a73654ee626fef`, which accepts a valid late Antigravity authorization URL while native authentication drains. This PR's diff against exact upstream `18b1637b6371a6a6adade857e2a73654ee626fef` contains only journey tests, report coverage and documentation. Harness source and deployed application/runtime identity are separate inputs. This source integration does not deploy the imported application changes or claim the QA target runs this application revision. This final SHA has not been measured by the historical run above: **fresh QA at this exact final head is required before primary merge**, followed by normal TestOps verification/admission. Fresh exact-head CI and review have passed as recorded above. The live measurement must record the actual separately deployed QA application/runtime identity. Seven whole-file blobs remain identical to the qualified SHA. All paths are relative to `tests/runner-e2e/`: | Journey/helper | Unchanged Git blob | | --- | --- | | `product-journeys/active-followup-evidence.ts` | `83a5d9fab52abb9849a40e83eca6b6a9c29bf587` | | `product-journeys/active-followup.test.ts` | `728160c610c18292d2cf06d4a87d25a8719f1734` | | `native-task-reply.ts` | `59d11f1a178dad9db3da8fa2342ff93643cb9922` | | `native-event-envelope.ts` | `305b0b456c59c511ee328e529ad574c0583075a0` | | `run-observations.ts` | `08c8ad184a821a7cd536926b3d7e3d1156a1e3da` | | `user-actions.ts` | `e9611348dc41e17899367dab2e3b36d3d1be9480` | | `stop-redirect-flow.ts` | `4580bd605a425bc72511e02d988492acbec8d60e` | The follow-up preservation comparison against historical `af9` has three explicit exceptions; no whole-file identity is claimed for them. The separately documented question readiness and report additions above also require final-source verification: 1. `product-journeys/agent-flows.ts` adds `question-answer-resume` to the `AgentJourney` type exclusion, alongside follow-up. Its remaining contents and runtime helper bodies are byte-for-byte unchanged. Qualified blob `a0afaa968c1d854476159fef21de7b6dd50a4a98`; final blob `0cc797c0ae594e7769338e63a80c4f67e29e4a54`. 2. The common run-count helper is centralized in `success-contract.ts` and re-exported by the catalog. It accepts follow-up **1–2**, clarification **1–3**, and each older paid journey's exact count. The follow-up catalog entry, maximum of two runs, six required checks, native dispatch, collection and retained-run contract remain byte-for-byte unchanged. The helper's text differs; its preserved count semantics are verified explicitly. 3. `product-journeys/active-followup-flow.ts` changes exactly one composer readiness assertion from Playwright's implicit five-second default to an explicit 30-second timeout, matching `submitTaskReply` for attached targets. Prompts, actions, native evidence, before/after persistence proof and grading otherwise compare byte-for-byte unchanged. Qualified blob `64392329198a1a4d9828e32834d4507e175216e6`; final blob `2746b5ba89e2af26b4097ba7897913186786ac5d`. This correction is why a fresh final-head QA run is required. - **310 focused tests passed at current final head `e72e20ad111bd94728d38606015d1fdac3c92929`**: 286 journey/evidence/catalog/report/shared-deadline tests in 14 files and 24 native authentication/real-child stream-order tests in two files. Both invocations used Node24 and one Vitest worker. The earlier `e55` integration also passed 365 focused tests, including 40 Cloud configuration and 40 TestOps metadata cases; those results remain prior-source evidence. - **157 explicit count assertions passed again at the current final head**, including both interactive ranges, older exact counts, invalid/fractional/non-number values and catalog/contract export identity. The maximum bounds and six closed follow-up check IDs also matched. - Maps and diff checks passed. The official QA runtime-import preflight passed on Node24 at the current final head without network access. It uses a dummy adapter and no credentials. It does not prove target connectivity, sign-in, model readiness or deployed application/runtime identity. Those require the normal protected QA run. No model/provider work or optional Runner release was dispatched. - Current local runtime dependencies use ignored isolated workspace-source views. No shared dependencies or compilers were changed. A previous local typecheck attempt hit stale reused Runner declarations. No full repository typecheck, test run or build was repeated locally; the successful exact-head full CI run above now supplies these gates. - CI on previous head `162e341cda5b627639b486b4019a1675bb018ea2` failed the Cloud/local maintenance fixture in [server shard 9](https://github.com/paperclipai/paperclip/actions/runs/41133716802/job/124331643100), plus dependent verify. The actual tested merge combined that head with master `198218bf554877fe3b7a83742eefb3b8f12392c6`. A Cloud child selected local-disk storage and hit the new shared-storage policy. This was a deterministic upstream fixture mismatch; no unchanged failed-job retry was used. - The actual #18385 repair configures a valid Cloud child, verifies local bytes independently, checks the expected provider mismatch and retains exact stored-config readback. Its own final head `b76d5dd40d7711f1b5f6b6316b1114c0aae02d57` passed 59 checks, including [server shard 9](https://github.com/paperclipai/paperclip/actions/runs/41133876636/job/124331450635). That prior result supports the repair; it does not replace this final head's fresh CI. Separate repair candidate #18392 and its source/evidence are retained as superseded history. - A local attempt to run the database fixture could not start PostgreSQL because the host exhausted macOS semaphores. A first setup attempt also failed package resolution before test execution. Skipped database cases and these setup failures are not passes. No kernel settings, shared processes or databases were changed. - The complete previous-head CI run [41138618869](https://github.com/paperclipai/paperclip/actions/runs/41138618869) finished with one underlying failure: [Runner Vitest shard 1](https://github.com/paperclipai/paperclip/actions/runs/41138618869/job/124347859445) expected native Antigravity login success but observed failure at `antigravity-auth.test.ts:101`. The lane had one failed test, 1,994 passing tests and eight skipped tests. Aggregate verify failed only because of this lane. All other 57 checks were accepted, including build, typecheck, all server/browser suites, Runner shard 2, Snyk and exact-head 5/5 reviews. No unchanged retry was requested. - The independently reviewed #18254 repair removes an invalid cross-pipe ordering assumption: a valid URL on stderr can arrive after the authenticate response on stdout during drain. Its real-child regression forces both orders. The current final source imports that actual squash; the 24 focused authentication cases pass here. Fresh full CI and exact-head review have now passed for this source, as recorded above. The earlier failed result is not relabeled. - The original live compatibility failure is retained: an older PRP v1 pack labeled builtin `bash` as operation `unknown`, causing the observer to miss the command and never send the follow-up. Its exact shape was reproduced red before repair. Generic unknown operations, semantic tools, wrappers, read-only work and foreign provenance remain rejected. The historical qualified pass used the repaired oracle; no failed result was relabeled. - Earlier CI on `6fad965d69` hit an unchanged heartbeat test's three-second adapter-call wait; its original failure and one narrow rerun remain recorded. The qualified SHA subsequently passed CI. An earlier report-catalog subprocess failure is retained. The final focused Node24 run now includes that suite and passes after the documented report feature mappings. ## Risks The fixture asks the native agent to run one bounded `sleep 90` command. The observer must see actual active native execution; a fast or noncompliant turn fails rather than passing from elapsed time. A completed result requires one or two bounded native work runs, source attribution, configured model/environment, and collection. Unknown provider prices stay unknown. This change does not alter production execution or grant new access. ## Model Used OpenAI Codex, GPT-6 family, with reasoning and code execution. The host does not expose an exact underlying model version or context-window size. ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with available version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs; existing follow-up execution fixes do not provide this deployment-targeted test - [x] I have described the issue in-PR following the enhancement template - [x] I have not referenced internal/instance-local Paperclip issues or links - [x] My branch name describes the change and contains no internal ticket or instance-derived details - [x] I have run the new focused tests locally and they pass - [x] I have added or updated tests where applicable - [x] I have updated relevant documentation - [x] I have considered and documented risks - [x] All Paperclip CI gates are green - [x] Greptile is 5/5 with no open P2s, recommendations, or follow-ups - [x] I will address all Greptile and reviewer comments before merge --------- Co-authored-by: Paperclip --- doc/evals.md | 6 + maps/feature-map/steering.md | 7 + tests/runner-e2e/PRODUCT-JOURNEYS.md | 32 ++++ tests/runner-e2e/catalog.test.ts | 2 +- .../runner-e2e/clarification-readback.spec.ts | 34 ++++ tests/runner-e2e/journey-features.ts | 2 + tests/runner-e2e/playwright-support.config.ts | 2 +- .../active-followup-evidence.ts | 82 +++++++++ .../product-journeys/active-followup-flow.ts | 110 +++++++++++ .../product-journeys/active-followup.test.ts | 174 ++++++++++++++++++ .../product-journeys/agent-flows.ts | 4 +- tests/runner-e2e/product-journeys/catalog.ts | 3 +- .../product-journeys/clarification-flow.ts | 18 +- .../clarification-readback.test.ts | 49 +++++ tests/runner-e2e/product-journeys/launch.ts | 5 +- .../product-journeys/success-contract.ts | 5 +- tests/runner-e2e/product-journeys/types.ts | 2 +- tests/runner-e2e/report-catalog.test.ts | 25 +++ 18 files changed, 550 insertions(+), 12 deletions(-) create mode 100644 tests/runner-e2e/clarification-readback.spec.ts create mode 100644 tests/runner-e2e/product-journeys/active-followup-evidence.ts create mode 100644 tests/runner-e2e/product-journeys/active-followup-flow.ts create mode 100644 tests/runner-e2e/product-journeys/active-followup.test.ts create mode 100644 tests/runner-e2e/product-journeys/clarification-readback.test.ts diff --git a/doc/evals.md b/doc/evals.md index 78c9c3ce467..bc0b7c9f729 100644 --- a/doc/evals.md +++ b/doc/evals.md @@ -76,6 +76,12 @@ one bound continuation, and an exact saved confirmation. Its four development cells use the recorded Codex and Claude models. Registration and calibration do not establish Slack, staging authentication, warm lifecycle, or live qualification. +The explicit `followup-while-running` [Product journey](../tests/runner-e2e/PRODUCT-JOURNEYS.md#active-follow-up) +checks a saved message during independently observed native work, one attributed +reply with the late requirement, and persistence after reload. Its attached and +managed targets share the same story and permit one steered run or one normal +successor. Registration and calibration are separate from a live staging pass. + [Reusable Product journeys](../tests/runner-e2e/PRODUCT-JOURNEYS.md) add explicit deployment targets and starting-state recipes. The shared task-properties journey uses no model; Cloud onboarding supplies reusable verified state. Cloud diff --git a/maps/feature-map/steering.md b/maps/feature-map/steering.md index 840fcd1ba74..611f0e56275 100644 --- a/maps/feature-map/steering.md +++ b/maps/feature-map/steering.md @@ -60,6 +60,13 @@ inspect the resulting task and run in Paperclip. The explicit `stop-redirect` [Product journey](../../tests/runner-e2e/PRODUCT-JOURNEYS.md) covers `steering/stop-and-pause` only for Stop and one new direction. It requires active work, native cancellation acknowledgement, resource cleanup, and completion without later original-turn activity. It does not qualify pause, interrupt, or all queued-message behavior. Source registration is not a live Cloud pass. +The explicit `followup-while-running` [Product journey](../../tests/runner-e2e/PRODUCT-JOURNEYS.md#active-follow-up) +covers `steering/queue`: one follow-up is saved while a bound native command is +active, its late requirement appears in one attributed final reply, the queue +empties, and the result survives reload. One steered turn or one normal successor +is allowed. Unit calibration and source registration are not live acceptance. + + Preconditions: follow the [baseline](./README.md#before-driving-a-journey). Use an agent that stays busy long enough to inspect the queue. Record the adapter, native/legacy mode, and advertised steering capability. Test supported and diff --git a/tests/runner-e2e/PRODUCT-JOURNEYS.md b/tests/runner-e2e/PRODUCT-JOURNEYS.md index 895005cd15a..98864325189 100644 --- a/tests/runner-e2e/PRODUCT-JOURNEYS.md +++ b/tests/runner-e2e/PRODUCT-JOURNEYS.md @@ -34,6 +34,7 @@ never start a server or depend on a tenant filesystem. | `protected-actions` | Native agent; four work runs; authenticated reachable review-service fixture | Approve one action and decline another on separate owned tasks; verify zero calls before either decision, one after approval, zero after decline and a truthful saved explanation | | `create-skill-studio` | Native agent with skill-write permission; one work run | Create a uniquely named company skill, open its activity card, edit in Studio, reload and verify task-card readback | | `stop-redirect` | Native agent; three work runs; qualified Docker oracle | Download a verified checkpoint, Stop an active revision, require native cancellation acknowledgement, submit a new direction once and verify completion without original-run activity | +| `followup-while-running` | Native agent; a maximum of two work runs | Save a follow-up while an independently observed native shell command is still active, then require one attributed final reply with the late requirement after reload; in-turn steering and one normal successor are accepted | | `cloud-onboarding` | Signed-in admitted Cloud account; fresh workspace; qualified bounded provider route | Create workspace in the portal, connect API key and native Codex agent in the wizard, complete a task, download and independently check its artifact | The question-resumption journey uses the same natural-language scenario, browser @@ -228,6 +229,37 @@ its disposable agent, makes owned tasks terminal, verifies no active runs, retai the records for decision-history integrity, and releases both QA leases. This is reported as retained data, rather than confirmed deletion. +## Active follow-up + +The `followup-while-running` story uses the same ordinary task composer and saved +reply observer on managed self-hosted and attached Cloud targets. The initial +request includes one bounded `sleep 90` command. It supplies no changed launch +day or final marker. The observer requires a valid running native turn and a +pending named builtin command or process execution before sending the follow-up. +Paperclip semantic tools, including completion/block/progress calls, cannot +satisfy this boundary even if a provider labels them builtin execute or wraps +their names with a native prefix. Unnamed or ambiguous builtin calls fail closed. +The observed older PRP v1 builtin `bash` shape is also supported when its +operation is `unknown` and `readOnly` is explicitly false; other unknown +operations cannot satisfy the boundary. If all owned work settles before an +active boundary is observed, the journey fails immediately rather than polling +until the campaign deadline. +The same command must remain active after exactly one user message is saved. +The task page and composer are ready before observing the boundary, so navigation is outside the +submission window. The longer bounded command leaves room for remote persistence. +Merely observing a running heartbeat or waiting for a fixed interval cannot satisfy the boundary. + +The late message changes the launch day and supplies a unique final marker. +The independent oracle requires the expected reply from one bound native run, +one or two successful work runs in total, an empty saved-message queue, Done, +and visible readback after reload. A provider can steer the original turn or +queue one normal successor; retries, duplicate replies and unrelated work fail. +Source/model/environment checks and full usage collection remain required. +Only the closed boundary receipt and attributed final reply are retained in the +private story directory. No local workspace path or provider-side fixture hook +is required. The registered source and unit calibration do not qualify a live +Cloud target; admission to scheduled testing requires its own fresh pass. + ## Evidence and lifecycle Results use the existing campaign/result schemas and dashboard. Each result diff --git a/tests/runner-e2e/catalog.test.ts b/tests/runner-e2e/catalog.test.ts index 1e98adc848b..cac4fcc784a 100644 --- a/tests/runner-e2e/catalog.test.ts +++ b/tests/runner-e2e/catalog.test.ts @@ -59,7 +59,7 @@ describe("runner E2E catalog", () => { }); it("exports deployment journeys separately from the model matrix and default selection", () => { - expect(productJourneys.map(journey => journey.id)).toEqual(["task-create", "project-task-context", "ask-question", "question-answer-resume", "task-properties", "task-search-filter", "plan-revise-accept", "build-revise", "protected-actions", "create-skill-studio", "stop-redirect", "cloud-onboarding"]); + expect(productJourneys.map(journey => journey.id)).toEqual(["task-create", "project-task-context", "ask-question", "question-answer-resume", "task-properties", "task-search-filter", "plan-revise-accept", "build-revise", "protected-actions", "create-skill-studio", "stop-redirect", "followup-while-running", "cloud-onboarding"]); expect(runnerSuites.some(suite => suite.id === "product-journeys")).toBe(false); expect(runnerMatrix.some(execution => execution.suite.id === "product-journeys")).toBe(false); expect(selectRunnerExecutions(parseRunnerSelectors(["--all"])).some(execution => execution.suite.id === "product-journeys")).toBe(false); diff --git a/tests/runner-e2e/clarification-readback.spec.ts b/tests/runner-e2e/clarification-readback.spec.ts new file mode 100644 index 00000000000..087457d5707 --- /dev/null +++ b/tests/runner-e2e/clarification-readback.spec.ts @@ -0,0 +1,34 @@ +import { expect, test } from "@playwright/test"; +import { waitForClarificationDocument } from "./product-journeys/clarification-flow.js"; + +const answer = "Use AMBERreadback."; +const selector = '[id="document-welcome-note"]:visible, [id="side-panel-content-document:welcome-note"]:visible'; + +for (const stage of ["mount", "text"] as const) { + test(`clarification readback waits beyond the default timeout for ${stage}`, async ({ page }) => { + const body = `Welcome to the afternoon meetup. ${answer}`; + await page.setContent(stage === "mount" ? "
" : '
Loading
'); + await page.evaluate(({ stage, body }) => { + setTimeout(() => { + if (stage === "mount") document.querySelector("main")!.innerHTML = `
${body}
`; + else document.querySelector("section")!.textContent = body; + }, 5_200); + }, { stage, body }); + await waitForClarificationDocument(page.locator(selector), answer, Date.now() + 8_000); + await expect(page.locator(selector)).toHaveText(body); + }); +} + +test("clarification readback rejects an exhausted campaign before waiting", async ({ page }) => { + await page.setContent(`
Welcome to the afternoon meetup. ${answer}
`); + await expect(waitForClarificationDocument(page.locator(selector), answer, Date.now() - 1)) + .rejects.toMatchObject({ code: "clarification_document_not_ready" }); +}); + +for (const invalid of ["duplicate", "wrong-reference", "wrong-choice"] as const) { + test(`clarification readback rejects ${invalid} evidence within the campaign deadline`, async ({ page }) => { + const body = invalid === "wrong-reference" ? "Welcome to the afternoon meetup. Use OLD." : invalid === "wrong-choice" ? answer : `Welcome to the afternoon meetup. ${answer}`; + await page.setContent(`
${body}
${invalid === "duplicate" ? `
${body}
` : ""}`); + await expect(waitForClarificationDocument(page.locator(selector), answer, Date.now() + 250)).rejects.toThrow(); + }); +} diff --git a/tests/runner-e2e/journey-features.ts b/tests/runner-e2e/journey-features.ts index 683f170c1e7..b0259b504d3 100644 --- a/tests/runner-e2e/journey-features.ts +++ b/tests/runner-e2e/journey-features.ts @@ -19,6 +19,8 @@ function associate(suites: readonly string[], cases: Record; +export interface ActiveFollowupBoundary { + runId: string; + executionId: string; + startedSeq: number; + sourceEventId: string; + sourceInstanceId: string; + sourceSeq: number; + startedRowSha256: string; +} + +/** A claimed run or elapsed delay is not proof that native work is active. */ +export function activeFollowupBoundary(run: Row, events: Row[]): ActiveFollowupBoundary | null { + if (!activeNativeTurnStarted(run, events) || !events.length + || events.some((event, index) => event.seq !== index + 1)) return null; + const bound = events.filter(event => { + const envelope = event.payload?.prpEvent; + return envelope && isValidNativePrpEnvelope(envelope, event.protocolSchemaVersion) + && event.runId === run.id && event.companyId === run.companyId && event.agentId === run.agentId + && envelope.runId === run.id && envelope.eventType === event.eventType && envelope.sourceKind === "runner" + && typeof envelope.sourceEventId === "string" && Boolean(envelope.sourceEventId) && envelope.sourceEventId === event.sourceEventId + && typeof envelope.sourceInstanceId === "string" && Boolean(envelope.sourceInstanceId) && envelope.sourceInstanceId === event.sourceInstanceId + && Number.isSafeInteger(envelope.sourceSeq) && envelope.sourceSeq > 0 && envelope.sourceSeq === event.sourceSeq; + }); + const latestTurn = bound.filter(event => /^turn\.(started|completed|failed|cancelled|interrupted)$/.test(event.eventType)).at(-1); + if (latestTurn?.eventType !== "turn.started") return null; + const starts = bound.filter(event => { + const payload = event.payload.prpEvent.payload; + return event.seq > latestTurn.seq && event.eventType === "tool.execution.started" && payload?.schema === "paperclip.tool.execution.v1" + && typeof payload.executionId === "string" && Boolean(payload.executionId) + && ["builtin", "process"].includes(payload.transport) + && (payload.operation === "execute" + // Older native packs classify builtin bash as unknown. Recognize only + // that named v1 shell shape; unknown semantic/custom tools still fail. + || event.protocolSchemaVersion === 1 && payload.operation === "unknown" + && payload.transport === "builtin" && payload.name === "bash" && payload.readOnly === false) + // Provider display titles vary, but Paperclip semantic calls never prove + // native command work, even when a provider maps them to builtin execute. + && payload.namespace == null + && (payload.transport === "process" || typeof payload.name === "string" && Boolean(payload.name.trim()) + && !/^(?:unknown|tool(?:[ _-]+call)?)$/i.test(payload.name.trim()) + && !/(?:^|[._:/-])paperclip(?:[._:/-]|$)/i.test(payload.name.trim())) + && payload.readOnly !== true && payload.status === "running"; + }); + const pending = starts.filter(start => { + const id = start.payload.prpEvent.payload.executionId; + return starts.filter(other => other.payload.prpEvent.payload.executionId === id).length === 1 + && !events.some(event => event.eventType === "tool.execution.completed" + && event.payload?.prpEvent?.payload?.executionId === id); + }); + const started = pending.at(-1); + if (!started) return null; + const envelope = started.payload.prpEvent; + return { runId: run.id, executionId: envelope.payload.executionId, startedSeq: started.seq, + sourceEventId: envelope.sourceEventId, sourceInstanceId: envelope.sourceInstanceId, sourceSeq: envelope.sourceSeq, + startedRowSha256: createHash("sha256").update(JSON.stringify(started)).digest("hex") }; +} + +export function sameActiveFollowupBoundary(before: ActiveFollowupBoundary, after: ActiveFollowupBoundary | null) { + return Boolean(after && Object.keys(before).every(key => before[key as keyof ActiveFollowupBoundary] === after[key as keyof ActiveFollowupBoundary])); +} + +/** Accept in-turn steering or one normal successor; reject duplicate delivery. */ +export function activeFollowupOutcome(input: { + issueId: string; agentId: string; before: ActiveFollowupBoundary; after: ActiveFollowupBoundary | null; + runs: Row[]; expectedReply: string; marker: string; replies: Array<{ runId: string; body: string }>; + storedFollowupCount: number; queuedCommentCount: number; finalIssueStatus: string; +}) { + const ids = input.runs.map(run => run.id); + const matching = input.replies.filter(reply => Boolean(input.marker) && reply.body.includes(input.marker)); + return sameActiveFollowupBoundary(input.before, input.after) + && input.storedFollowupCount === 1 && input.queuedCommentCount === 0 && input.finalIssueStatus === "done" + && input.runs.length >= 1 && input.runs.length <= 2 && new Set(ids).size === ids.length && ids.includes(input.before.runId) + && input.runs.every(run => run.agentId === input.agentId && run.status === "succeeded" && run.runtimeMode === "native" + && (run.nativeIssueId ?? run.issueId ?? run.contextSnapshot?.issueId ?? run.contextSnapshot?.taskId) === input.issueId) + && matching.length === 1 && matching[0]!.body === input.expectedReply && ids.includes(matching[0]!.runId) + && (input.runs.length === 1 || matching[0]!.runId !== input.before.runId); +} diff --git a/tests/runner-e2e/product-journeys/active-followup-flow.ts b/tests/runner-e2e/product-journeys/active-followup-flow.ts new file mode 100644 index 00000000000..2746b5ba89e --- /dev/null +++ b/tests/runner-e2e/product-journeys/active-followup-flow.ts @@ -0,0 +1,110 @@ +import { mkdir, writeFile } from "node:fs/promises"; +import path from "node:path"; +import { expect } from "@playwright/test"; +import { createTaskThroughUi, submitTaskReply } from "../user-actions.js"; +import { collectRunEvents } from "../run-observations.js"; +import { nativeFinalReplyEvents, readBoundTaskFinalReply } from "../native-task-reply.js"; +import { fixtureReplyStoredOnce, validateAgentTaskState, waitFor } from "./agent-flows.js"; +import { activeFollowupBoundary, activeFollowupOutcome, sameActiveFollowupBoundary } from "./active-followup-evidence.js"; +import { JourneyError, type JourneyContext, type JourneyEvidence, type TargetSession } from "./types.js"; + +type Row = Record; +const id = "followup-while-running"; + +export function activeFollowupStory(campaignId: string) { + const nonce = campaignId.replace(/[^a-z0-9]/gi, ""); + const reference = `BRIEF${nonce}`, marker = `UPDATED${nonce}`; + const expectedReply = `Launch Friday. Reference ${reference}. Update ${marker}.`; + return { + first: `Our launch is planned for Monday. Run the bounded command sleep 90 exactly once before answering, so I can send additional context while you work. After the command finishes, briefly acknowledge the launch day and reference ${reference}, then finish this task. No files, projects or child tasks are needed.`, + followup: `Change the launch day to Friday. When ready, reply with exactly this one line and finish: ${expectedReply}`, + reference, marker, expectedReply, + }; +} + +/** Once owned work has settled, an unobserved live boundary cannot arrive. */ +export async function observeActiveFollowupCommand(taskRuns: Row[], loadEvents: (runId: string) => Promise) { + if (taskRuns.length > 0 && taskRuns.every(run => run.status === "succeeded")) + throw new JourneyError("failed", "followup_native_command_not_active"); + const run = taskRuns.find(run => run.status === "running"); + return { run, boundary: run ? activeFollowupBoundary(run, await loadEvents(run.id)) : null }; +} + +/** The same browser/API/native story runs against managed and attached targets. */ +export async function runActiveFollowup(context: JourneyContext, session: TargetSession, campaignId: string, + evidence: JourneyEvidence, options: { deadlineAt: number; directory: string }) { + const agent = context.agent; + if (!agent || agent.runtimeMode !== "native" || !context.environmentId) throw new JourneyError("unsupported", "native_agent_and_environment_required"); + const story = activeFollowupStory(campaignId), title = `Journey QA active followup ${campaignId}`; + const check = (name: string, passed: boolean) => { + evidence.checks.push({ id: name, passed }); + if (!passed) throw new JourneyError("failed", name.replaceAll("-", "_")); + }; + await mkdir(options.directory, { recursive: true, mode: 0o700 }); + const retain = (name: string, value: unknown) => writeFile(path.join(options.directory, name), JSON.stringify(value, null, 2), { mode: 0o600 }); + const created = await createTaskThroughUi({ page: context.page, origin: context.api.origin, issuePrefix: context.company.issuePrefix, + agentName: agent.name, title, prompt: story.first, workMode: "standard", requireExplicitTitle: true, + beforeDispatch: () => session.own({ kind: "issue", name: title, journeyId: id }) }); + await session.own({ kind: "issue", name: title, id: created.issueId, journeyId: id }); + const route = `/api/issues/${created.issueId}`; + const read = async () => { + const [currentIssue, summaries, comments, queue] = await Promise.all([ + context.api.get(route), context.api.get(`${route}/runs`), context.api.get(`${route}/comments?order=asc`), + context.api.get(`${route}/queued-comments`), + ]); + if (!Array.isArray(summaries) || summaries.length > 2) throw new JourneyError("failed", "followup_run_limit_exceeded"); + const taskRuns = await Promise.all(summaries.map(summary => context.api.get(`/api/heartbeat-runs/${summary.runId}`))); + validateAgentTaskState(context, created.issueId, { currentIssue, taskRuns, comments, interactions: [], attachments: [] }, 2); + if (currentIssue.description !== story.first) throw new JourneyError("failed", "followup_initial_prompt_changed"); + return { currentIssue, taskRuns, comments, queue }; + }; + const events = (runId: string) => collectRunEvents((afterSeq, limit) => + context.api.get(`/api/heartbeat-runs/${runId}/events?afterSeq=${afterSeq}&limit=${limit}`)); + const issue = (await read()).currentIssue; + await context.page.goto(`${context.api.origin}/${context.company.issuePrefix}/issues/${issue.identifier ?? issue.id}`, { waitUntil: "domcontentloaded" }); + await expect(context.page.getByTestId("task-chat-composer-input").last()).toBeVisible({ timeout: 30_000 }); + const beforeComments = (await read()).comments; + const active = await waitFor(options.deadlineAt, async () => { + const state = await read(); + return observeActiveFollowupCommand(state.taskRuns, events); + }, value => Boolean(value.boundary), "followup_native_command_not_active"); + const before = active.boundary!; + await submitTaskReply(context.page, story.followup); + const persisted = await waitFor(Math.min(options.deadlineAt, Date.now() + 30_000), read, state => state.comments.some(comment => comment.authorUserId === context.user.id + && !beforeComments.some(old => old.id === comment.id)), "followup_not_persisted"); + const afterRun = persisted.taskRuns.find(run => run.id === before.runId)!; + const after = activeFollowupBoundary(afterRun, await events(before.runId)); + await retain("followup-boundary.json", { before, after }); + check("followup-native-command-active", sameActiveFollowupBoundary(before, after)); + check("followup-stored-once", fixtureReplyStoredOnce(beforeComments, persisted.comments, context.user.id, story.followup)); + + const final = await waitFor(options.deadlineAt, async () => { + const state = await read(); + const replies = [] as Array<{ runId: string; body: string }>; + for (const run of state.taskRuns.filter(run => run.status === "succeeded")) { + const stream = await events(run.id); + const decision = run.resultJson?.presentationDecision?.commentId; + const posted = state.comments.filter(comment => comment.id === decision && comment.authorAgentId === agent.id && comment.createdByRunId === run.id); + if (posted.length) replies.push(...posted.map(comment => ({ runId: run.id, body: comment.body }))); + else replies.push(...nativeFinalReplyEvents(stream, run).map(reply => ({ runId: run.id, body: reply.body }))); + } + return { state, replies }; + }, value => activeFollowupOutcome({ issueId: created.issueId, agentId: agent.id, before, after, + runs: value.state.taskRuns, expectedReply: story.expectedReply, marker: story.marker, replies: value.replies, + storedFollowupCount: value.state.comments.filter(comment => comment.authorUserId === context.user.id + && comment.body?.replaceAll("\\_", "_") === story.followup).length, + queuedCommentCount: Array.isArray(value.state.queue.entries) ? value.state.queue.entries.length : -1, + finalIssueStatus: value.state.currentIssue.status }), "followup_final_outcome_missing"); + check("followup-consumed-once", true); + const finalRunId = final.replies.find(reply => reply.body === story.expectedReply)!.runId; + const companies = await context.api.get>("/api/companies?scope=accessible"); + if (!Array.isArray(companies) || !companies.some(company => company.id === context.company.id) || companies.some(company => !company.issuePrefix)) throw new JourneyError("failed", "reply_company_context_missing"); + const reply = await readBoundTaskFinalReply({ page: context.page, runId: finalRunId, agentId: agent.id, + expectedText: story.expectedReply, deadlineAt: options.deadlineAt, knownCompanyPrefixes: companies.map(company => company.issuePrefix), + load: async () => { const latest = await read(); return { run: latest.taskRuns.find(run => run.id === finalRunId)!, comments: latest.comments, events: await events(finalRunId) }; } }); + await retain("followup-final.json", { before, after, runIds: final.state.taskRuns.map(run => run.id), reply }); + check("followup-final-answer-persisted", reply.observedAfterReload === true); + check("followup-one-or-two-native-runs", final.state.taskRuns.length >= 1 && final.state.taskRuns.length <= 2); + await expect(context.page.getByText(story.expectedReply, { exact: true }).last()).toBeVisible(); + check("configured-model-and-environment", true); +} diff --git a/tests/runner-e2e/product-journeys/active-followup.test.ts b/tests/runner-e2e/product-journeys/active-followup.test.ts new file mode 100644 index 00000000000..728160c610c --- /dev/null +++ b/tests/runner-e2e/product-journeys/active-followup.test.ts @@ -0,0 +1,174 @@ +import { describe, expect, it } from "vitest"; +import { canonicalProviderEventsFromCodex, canonicalProviderEventsFromAcpxRuntimeEvent, canonicalProviderEventsFromOpenCodePart } from "../../../packages/paperclip-runner/src/provider-events.js"; +import { activeFollowupBoundary, activeFollowupOutcome, sameActiveFollowupBoundary } from "./active-followup-evidence.js"; +import { activeFollowupStory, observeActiveFollowupCommand } from "./active-followup-flow.js"; +import { journeyRequiredChecks } from "./success-contract.js"; +import { parseTargetConfig, preparationPreview, agentJourneyRunCountValid } from "./catalog.js"; +import { validateRetainedRunnerResult } from "../result-validation.js"; + +const run = { id: "run", companyId: "company", agentId: "agent", issueId: "issue", runtimeMode: "native", status: "running" }; +function event(seq: number, type: string, payload = {}): Record { + return { seq, eventType: type, runId: run.id, companyId: run.companyId, agentId: run.agentId, + protocolSchemaVersion: 2, sourceEventId: `source-${seq}`, sourceInstanceId: "source", sourceSeq: seq, + payload: { prpEvent: { schema: "paperclip.prp.event.v2", schemaVersion: 2, eventType: type, runId: run.id, + sourceKind: "runner", sourceEventId: `source-${seq}`, sourceInstanceId: "source", sourceSeq: seq, payload } } }; +} +const command = { schema: "paperclip.tool.execution.v1", executionId: "wait", transport: "builtin", operation: "execute", name: "bash", status: "running" }; +const events = () => [event(1, "turn.started"), event(2, "tool.execution.started", command)]; +const boundary = () => activeFollowupBoundary(run, events())!; +// Synthetic identity and filler rows; the v1 operation/transport/name/readOnly +// shape and native/source sequence positions match observed deployment metadata. +function legacyEvents(change: Record = {}): Record[] { + const rows: Record[] = Array.from({ length: 81 }, (_, index) => ({ seq: index + 1, eventType: "adapter.message" })); + const bind = (seq: number, sourceSeq: number, type: string, payload: Record) => { + const row = event(seq, type, payload); + row.protocolSchemaVersion = 1; row.sourceSeq = sourceSeq; + Object.assign(row.payload.prpEvent, { schema: "paperclip.prp.event.v1", schemaVersion: 1, sourceSeq }); + return row; + }; + rows[70] = bind(71, 9, "turn.started", { status: "inProgress" }); + rows[80] = bind(81, 12, "tool.execution.started", { ...command, namespace: null, operation: "unknown", readOnly: false, ...change }); + return rows; +} +const outcome = () => ({ issueId: "issue", agentId: "agent", before: boundary(), after: boundary(), + runs: [{ ...run, status: "succeeded" }], expectedReply: "Friday UPDATED", marker: "UPDATED", replies: [{ runId: "run", body: "Friday UPDATED" }], + storedFollowupCount: 1, queuedCommentCount: 0, finalIssueStatus: "done" }); + +describe("deployment-independent active follow-up", () => { + it("withholds the changed requirement and result marker until the follow-up", () => { + const story = activeFollowupStory("campaign-123"); + expect(story.first).toContain("sleep 90"); expect(story.first).toContain(story.reference); + expect(story.first).not.toContain(story.marker); expect(story.first).not.toContain("Friday"); + expect(story.followup).toContain(story.expectedReply); + }); + it("requires a bound running native turn and an actual pending shell execution", () => { + expect(boundary()).toMatchObject({ runId: "run", executionId: "wait", startedSeq: 2 }); + for (const rows of [[], events().slice(0, 1), [...events(), event(3, "tool.execution.completed", { ...command, status: "completed" })], + [...events(), event(3, "turn.completed")], [...events(), event(3, "turn.started")], [events()[0]!, event(2, "tool.execution.started", { ...command, operation: "read" })], + [events()[0]!, event(2, "tool.execution.started", { ...command, readOnly: true })], + [events()[0]!, event(2, "tool.execution.started", { ...command, name: "paperclip_finish" })], + [events()[0]!, event(2, "tool.execution.started", { ...command, name: "paperclip_block" })], + [events()[0]!, event(2, "tool.execution.started", { ...command, namespace: "paperclip" })], + [events()[0]!, event(2, "tool.execution.started", { ...command, name: null })], + [events()[0]!, event(2, "tool.execution.started", { ...command, transport: "mcp" })], + [events()[0]!, event(3, "tool.execution.started", command)], + [...events(), event(3, "tool.execution.started", command)]]) expect(activeFollowupBoundary(run, rows)).toBeNull(); + for (const field of ["companyId", "agentId", "runId", "sourceEventId", "sourceInstanceId", "sourceSeq", "protocolSchemaVersion"]) { + const rows = events(); rows[1]![field] = "wrong"; expect(activeFollowupBoundary(run, rows)).toBeNull(); + } + const rows = events(); rows[0]!.payload.prpEvent.sourceKind = "control_plane"; + expect(activeFollowupBoundary(run, rows)).toBeNull(); + for (const changed of [{ status: "succeeded" }, { runtimeMode: "legacy" }]) expect(activeFollowupBoundary({ ...run, ...changed }, events())).toBeNull(); + }); + it("accepts canonical Codex command text and ACP display titles without provider-name coupling", () => { + const codex = canonicalProviderEventsFromCodex("item/started", { + item: { id: "wait", type: "commandExecution", command: "sleep 90", status: "inProgress" }, + })[0]!; + const acp = canonicalProviderEventsFromAcpxRuntimeEvent({ + type: "tool_call", tag: "tool_call", toolCallId: "wait", title: "Bounded settlement fixture", kind: "execute", status: "pending", + } as never, "fallback")[0]!; + expect(codex.payload).toMatchObject({ transport: "process", operation: "execute", name: "sleep 90", status: "running" }); + expect(acp.payload).toMatchObject({ transport: "builtin", operation: "execute", name: "Bounded settlement fixture", readOnly: false, status: "running" }); + for (const mapped of [codex, acp]) { + expect(activeFollowupBoundary(run, [event(1, "turn.started"), event(2, mapped.eventType, mapped.payload)])).toMatchObject({ runId: "run", startedSeq: 2 }); + } + }); + it("accepts the real OpenCode shell fixture and rejects semantic calls normalized as builtin execute", () => { + const mapped = (tool: string, callID = "") => canonicalProviderEventsFromOpenCodePart({ + id: "wait", type: "tool", tool, callID, state: { status: "pending", input: { command: "sleep 90" } }, + })[0]!; + const shell = mapped("bash"); + expect(shell.payload).toMatchObject({ transport: "builtin", operation: "execute", name: "bash", readOnly: false, status: "running" }); + expect(activeFollowupBoundary(run, [event(1, "turn.started"), event(2, shell.eventType, shell.payload)])).toMatchObject({ runId: "run", startedSeq: 2 }); + for (const semantic of [mapped("paperclip_finish"), mapped("paperclip_block"), mapped("paperclip_report_progress"), + mapped("paperclip_paperclip_finish"), mapped("unknown", "functions.paperclip_block:0"), + mapped("functions.paperclip_finish"), mapped("mcp__paperclip__paperclip_block"), mapped("paperclip.paperclip_report_progress")]) { + expect(semantic.payload).toMatchObject({ transport: "builtin", operation: "execute", readOnly: false, status: "running" }); + expect(activeFollowupBoundary(run, [event(1, "turn.started"), event(2, semantic.eventType, semantic.payload)])).toBeNull(); + } + for (const name of [null, "", " ", "unknown", "tool", "tool call", "tool_call", "tool-call"]) { + expect(activeFollowupBoundary(run, [event(1, "turn.started"), event(2, shell.eventType, { ...shell.payload, name })])).toBeNull(); + } + }); + it("recognizes the observed legacy v1 builtin bash shape without admitting unknown operations generally", () => { + const before = activeFollowupBoundary(run, legacyEvents()); + expect(before).toMatchObject({ runId: "run", executionId: "wait", startedSeq: 81, sourceSeq: 12 }); + for (const change of [{ name: "unknown" }, { name: "custom-command" }, { name: "paperclip_finish" }, { name: "paperclip_block" }, + { name: "paperclip_report_progress" }, { name: "functions.paperclip_finish" }, { readOnly: true }, { readOnly: undefined }, + { transport: "process" }, { namespace: "paperclip" }]) expect(activeFollowupBoundary(run, legacyEvents(change))).toBeNull(); + for (const field of ["companyId", "agentId", "runId", "sourceEventId", "sourceInstanceId", "sourceSeq", "protocolSchemaVersion"]) { + const rows = legacyEvents(); rows[80]![field] = "foreign"; + expect(activeFollowupBoundary(run, rows)).toBeNull(); + } + expect(activeFollowupBoundary(run, [events()[0]!, event(2, "tool.execution.started", { ...command, operation: "unknown", readOnly: false })])).toBeNull(); + const after = legacyEvents(); + after.push(event(82, "tool.execution.progressed", { ...after[80]!.payload.prpEvent.payload })); + expect(sameActiveFollowupBoundary(before!, activeFollowupBoundary(run, after))).toBe(true); + after.push(event(83, "tool.execution.completed", { ...after[80]!.payload.prpEvent.payload, status: "completed" })); + expect(activeFollowupBoundary(run, after)).toBeNull(); + }); + it("fails promptly after owned work settles without an observed active boundary", async () => { + let eventReads = 0; + const load = async () => { eventReads++; return legacyEvents(); }; + for (const taskRuns of [[{ ...run, status: "succeeded" }], [{ ...run, status: "succeeded" }, { ...run, id: "second", status: "succeeded" }]]) + await expect(observeActiveFollowupCommand(taskRuns, load)).rejects.toMatchObject({ outcome: "failed", code: "followup_native_command_not_active" }); + expect(eventReads).toBe(0); + for (const taskRuns of [[], [{ ...run, status: "queued" }], [{ ...run, status: "succeeded" }, { ...run, id: "second", status: "queued" }]]) + expect((await observeActiveFollowupCommand(taskRuns, load)).boundary).toBeNull(); + expect(eventReads).toBe(0); + expect((await observeActiveFollowupCommand([run], load)).boundary).toMatchObject({ runId: "run", startedSeq: 81 }); + expect(eventReads).toBe(1); + }); + it("proves the same execution remains active after the user message is stored", () => { + expect(sameActiveFollowupBoundary(boundary(), boundary())).toBe(true); + for (const changed of [null, { ...boundary(), executionId: "other" }, { ...boundary(), startedRowSha256: "changed" }]) + expect(sameActiveFollowupBoundary(boundary(), changed)).toBe(false); + }); + it("accepts one steered run or one normal successor with the unique late answer", () => { + expect(activeFollowupOutcome(outcome())).toBe(true); + const value = outcome(); value.runs.push({ ...value.runs[0]!, id: "successor" }); value.replies[0]!.runId = "successor"; + expect(activeFollowupOutcome(value)).toBe(true); + }); + it("rejects lost input, duplicate delivery/work, stale answers and unfinished queues", () => { + const changes = [ + (v: ReturnType) => { v.storedFollowupCount = 0; }, + (v: ReturnType) => { v.storedFollowupCount = 2; }, + (v: ReturnType) => { v.queuedCommentCount = 1; }, + (v: ReturnType) => { v.finalIssueStatus = "in_progress"; }, + (v: ReturnType) => { v.replies = []; }, + (v: ReturnType) => { v.replies.push(v.replies[0]!); }, + (v: ReturnType) => { v.replies[0]!.body = "Monday"; }, + (v: ReturnType) => { v.replies.push({ runId: "run", body: "UPDATED again" }); }, + (v: ReturnType) => { v.replies[0]!.runId = "other"; }, + (v: ReturnType) => { v.runs[0]!.status = "failed"; }, + (v: ReturnType) => { v.runs[0]!.agentId = "other"; }, + (v: ReturnType) => { v.runs[0]!.issueId = "other"; }, + (v: ReturnType) => { v.runs = []; }, + (v: ReturnType) => { v.runs.push(v.runs[0]!); }, + (v: ReturnType) => { v.runs.push({ ...v.runs[0]!, id: "successor" }); }, + ]; + for (const change of changes) { const value = outcome(); change(value); expect(activeFollowupOutcome(value)).toBe(false); } + }); + it("allows a maximum of two declared runs while preserving every other exact bound", () => { + for (const count of [1, 2]) expect(agentJourneyRunCountValid("followup-while-running", count)).toBe(true); + for (const count of [undefined, 0, 3, NaN, 1.5]) expect(agentJourneyRunCountValid("followup-while-running", count)).toBe(false); + expect(agentJourneyRunCountValid("build-revise", 1)).toBe(false); + const config = { version: 1, id: "qa", mode: "existing-cloud", adapterModule: "/private/provider.mjs", journeys: ["followup-while-running"], + executionEnvironment: "daytona", baseline: { features: {}, connections: [] }, limits: { durationMs: 600_000, modelBudgetCents: 100, maxRuns: 2 } }; + expect(preparationPreview(parseTargetConfig(config)).journeys[0]).toMatchObject({ id: "followup-while-running", paid: true }); + expect(() => parseTargetConfig({ ...config, limits: { ...config.limits, maxRuns: 1 } })).toThrow("agent_journeys_require_bounded_runs"); + }); + it("retains one-run and two-run passes only with the whole success contract", () => { + const result = { schema: "paperclip.runner-e2e.result/v2", executionId: "product-journeys.qa.configured-agent.daytona.followup-while-running", + suiteId: "product-journeys", suiteDefinitionHash: "test", profileId: "configured-agent", environmentId: "daytona", caseId: "followup-while-running", + provider: "opencode", model: "configured-model", runtimeMode: "native", status: "passed", cleanup: "passed", attempt: 1, + startedAt: "2026-01-01T00:00:00Z", finishedAt: "2026-01-01T00:00:01Z", durationMs: 1000, runIds: ["run"], + productJourney: { outcome: "passed", startingState: "recipe", privacy: "private", resourceDisposition: "retained", + phases: [{ phase: "journey", outcome: "passed" }], checks: journeyRequiredChecks["followup-while-running"]!.map(id => ({ id, passed: true })), + deployment: { kind: "cloud", targetId: "qa", origin: "https://example.test", commit: "a".repeat(40), cloudRelease: "b".repeat(40), runtimeImage: "pinned", configurationHash: "test" } } }; + for (const runIds of [["run"], ["run", "successor"]]) expect(() => validateRetainedRunnerResult({ ...result, runIds })).not.toThrow(); + for (const runIds of [[], ["a", "b", "c"]]) expect(() => validateRetainedRunnerResult({ ...result, runIds })).toThrow("result.agentJourneyRun"); + for (const check of result.productJourney.checks) expect(() => validateRetainedRunnerResult({ ...result, productJourney: { ...result.productJourney, + checks: result.productJourney.checks.filter(row => row.id !== check.id) } })).toThrow("result.productJourney"); + }); +}); diff --git a/tests/runner-e2e/product-journeys/agent-flows.ts b/tests/runner-e2e/product-journeys/agent-flows.ts index 37ccb8995bf..0cc797c0ae5 100644 --- a/tests/runner-e2e/product-journeys/agent-flows.ts +++ b/tests/runner-e2e/product-journeys/agent-flows.ts @@ -18,7 +18,7 @@ import { agentJourneyRuns } from "./catalog.js"; import { JourneyError, type JourneyContext, type JourneyEvidence, type TargetSession } from "./types.js"; type Row = Record; -type AgentJourney = Exclude; +type AgentJourney = Exclude; interface TaskState { currentIssue: Row & { id: string; status: string }; taskRuns: Array; comments: Row[]; interactions: Array; attachments: StoryDelivery[] } interface Options { deadlineAt: number; directory: string; capture(id: string, label: string, file: string): Promise } const active = (run: Row) => ["queued", "running"].includes(run.status); @@ -46,7 +46,7 @@ export function validateAgentTaskState(context: JourneyContext, issueId: string, } } -async function waitFor(deadlineAt: number, load: () => Promise, accept: (value: T) => boolean, code: string): Promise { +export async function waitFor(deadlineAt: number, load: () => Promise, accept: (value: T) => boolean, code: string): Promise { while (Date.now() < deadlineAt) { const value = await load(); if (accept(value)) return value; diff --git a/tests/runner-e2e/product-journeys/catalog.ts b/tests/runner-e2e/product-journeys/catalog.ts index 3d40c2cdc66..3013851a360 100644 --- a/tests/runner-e2e/product-journeys/catalog.ts +++ b/tests/runner-e2e/product-journeys/catalog.ts @@ -1,5 +1,5 @@ import { agentJourneyRuns } from "./success-contract.js"; -export { agentJourneyRuns } from "./success-contract.js"; +export { agentJourneyRuns, agentJourneyRunCountValid } from "./success-contract.js"; import type { JourneyId, TargetConfig } from "./types.js"; export const productJourneys = [ { id: "task-create", label: "Create a task for yourself and reopen it", requires: ["company", "board-user", "task-write"], capabilities: ["tasks/create"], paid: false }, @@ -13,6 +13,7 @@ export const productJourneys = [ { id: "protected-actions", label: "Approve and decline protected service actions", requires: ["company", "board-user", "task-write", "agent", "review-service"], capabilities: ["questions-and-approvals/tool-review"], paid: true }, { id: "create-skill-studio", label: "Create a company skill and persist a Skill Studio edit", requires: ["company", "board-user", "task-write", "agent", "skill-write"], capabilities: ["skills/authoring"], paid: true }, { id: "stop-redirect", label: "Stop active work and send a new direction once", requires: ["company", "board-user", "task-write", "agent"], capabilities: ["steering/stop-and-pause"], paid: true }, + { id: "followup-while-running", label: "Send a follow-up during active work and verify its saved result", requires: ["company", "board-user", "task-write", "agent"], capabilities: ["steering/queue"], paid: true }, { id: "cloud-onboarding", label: "Create a Cloud workspace and complete a first task", requires: ["cloud-account", "fresh-workspace", "provider-key"], capabilities: ["onboarding/first-task"], paid: true }, ] as const; diff --git a/tests/runner-e2e/product-journeys/clarification-flow.ts b/tests/runner-e2e/product-journeys/clarification-flow.ts index 932efbebf92..800a747284d 100644 --- a/tests/runner-e2e/product-journeys/clarification-flow.ts +++ b/tests/runner-e2e/product-journeys/clarification-flow.ts @@ -1,6 +1,6 @@ import { mkdir, writeFile } from "node:fs/promises"; import path from "node:path"; -import { expect } from "@playwright/test"; +import { expect, type Locator } from "@playwright/test"; import { continuationScenario } from "../continuation-cases.js"; import { continuationAnswerCommitted, continuationCheckpointReady } from "../continuation-readiness.js"; import type { ContinuationCheckpoint } from "../continuation-scoring.js"; @@ -13,6 +13,19 @@ import { JourneyError, type JourneyContext, type JourneyEvidence, type TargetSes type Row = Record; interface Options { deadlineAt: number; directory: string; capture(id: string, label: string, file: string): Promise } +export async function waitForClarificationDocument(visible: Locator, answer: string, deadlineAt: number) { + const remaining = () => { + const timeout = deadlineAt - Date.now(); + if (!Number.isFinite(timeout) || timeout <= 0) throw new JourneyError("failed", "clarification_document_not_ready"); + return timeout; + }; + await expect(visible).toHaveCount(1, { timeout: remaining() }); + await expect(visible).toBeVisible({ timeout: remaining() }); + await expect(visible).toContainText(answer, { timeout: remaining() }); + await expect(visible).toContainText(/afternoon/i, { timeout: remaining() }); + remaining(); +} + export async function readClarificationState(context: JourneyContext, issueId: string) { const route = `/api/issues/${issueId}`; const [currentIssue, summaries, interactions, documents, attachments, children] = await Promise.all([ @@ -96,8 +109,7 @@ export async function runClarificationJourney(context: JourneyContext, session: await context.page.goto(new URL(href, context.api.origin).href, { waitUntil: "domcontentloaded" }); await context.page.reload({ waitUntil: "domcontentloaded" }); const visible = context.page.locator(`[id=${JSON.stringify(`document-${document.key}`)}]:visible, [id=${JSON.stringify(`side-panel-content-document:${document.key}`)}]:visible`); - await expect(visible).toHaveCount(1); await expect(visible).toBeVisible(); - await expect(visible).toContainText(scenario.answer); await expect(visible).toContainText(/afternoon/i); + await waitForClarificationDocument(visible, scenario.answer, options.deadlineAt); const persisted = await context.api.get(`/api/issues/${issue.id}/documents/${encodeURIComponent(document.key)}`); readback = { issueId: persisted.issueId, key: persisted.key, revisionId: persisted.latestRevisionId, body: persisted.body, visibleBody: await visible.innerText() }; await options.capture("reloaded", "Saved answers after reload", "clarification-reloaded.png"); diff --git a/tests/runner-e2e/product-journeys/clarification-readback.test.ts b/tests/runner-e2e/product-journeys/clarification-readback.test.ts new file mode 100644 index 00000000000..5ec51654630 --- /dev/null +++ b/tests/runner-e2e/product-journeys/clarification-readback.test.ts @@ -0,0 +1,49 @@ +import { afterEach, expect, it, vi } from "vitest"; +import type { Locator } from "@playwright/test"; + +const clock = vi.hoisted(() => ({ now: 0, locator: {} })); +const answer = "Use AMBERreadback."; +const body = `Welcome to the afternoon meetup. ${answer}`; + +function stateAt(now: number) { + return { count: now >= 100 ? 1 : 0, visible: now >= 100, + text: now >= 1_300 ? body : now >= 600 ? answer : "Loading" }; +} + +vi.mock("@playwright/test", async importOriginal => { + const actual = await importOriginal(); + const wait = (accept: (state: ReturnType) => boolean, options: { timeout: number }) => { + if (!Number.isFinite(options.timeout) || options.timeout <= 0) throw Error("Invalid wait budget"); + if (accept(stateAt(clock.now))) return; + const deadline = clock.now + options.timeout; + const readyAt = [100, 600, 1_300].find(time => time > clock.now && time <= deadline && accept(stateAt(time))); + clock.now = readyAt ?? deadline; + if (readyAt === undefined) throw Error("Document not ready within wait budget"); + }; + return { ...actual, expect: (locator: Locator) => { + if (locator !== clock.locator) throw Error("Unexpected document locator"); + return { + async toHaveCount(count: number, options: { timeout: number }) { wait(state => state.count === count, options); }, + async toBeVisible(options: { timeout: number }) { wait(state => state.visible, options); }, + async toContainText(text: string | RegExp, options: { timeout: number }) { + wait(state => typeof text === "string" ? state.text.includes(text) : text.test(state.text), options); + }, + }; + } }; +}); + +import { waitForClarificationDocument } from "./clarification-flow.js"; + +afterEach(() => vi.restoreAllMocks()); + +it("shares one document deadline and retains rejection evidence through a later scheduler stall", async () => { + clock.now = 0; + vi.spyOn(Date, "now").mockImplementation(() => clock.now); + const observed = await waitForClarificationDocument(clock.locator as Locator, answer, 1_000).then( + () => ({ outcome: "resolved", at: clock.now, text: stateAt(clock.now).text }), + (error: Error) => ({ outcome: "rejected", error: error.message, at: clock.now, text: stateAt(clock.now).text }), + ); + clock.now += 10_000; + expect(stateAt(clock.now).text).toBe(body); + expect(observed).toEqual({ outcome: "rejected", error: "Document not ready within wait budget", at: 1_000, text: answer }); +}); diff --git a/tests/runner-e2e/product-journeys/launch.ts b/tests/runner-e2e/product-journeys/launch.ts index 4be973f1e81..72706d53ba9 100644 --- a/tests/runner-e2e/product-journeys/launch.ts +++ b/tests/runner-e2e/product-journeys/launch.ts @@ -14,6 +14,7 @@ import { taskSearchFilter } from "./task-search-filter.js"; import { askQuestion, taskCreate } from "./basic-flows.js"; import { projectTaskContext } from "./project-context-flow.js"; import { runAgentJourney } from "./agent-flows.js"; +import { runActiveFollowup } from "./active-followup-flow.js"; import { runClarificationJourney } from "./clarification-flow.js"; import { JourneyError, type JourneyContext, type JourneyEvidence, type JourneyOutcome, type TargetLifecycleInput, type TargetProvider, type TargetSession } from "./types.js"; import type { RunnerE2EResult } from "../types.js"; @@ -106,7 +107,7 @@ export async function launchProductJourneys(args: string[], repositoryRoot: stri try { sha = execFileSync("git", ["rev-parse", "HEAD"], { cwd: repositoryRoot, encoding: "utf8", stdio: ["ignore", "pipe", "ignore"] }).trim(); } catch { /* Packaged harnesses may not include Git metadata. */ } const source = resolveRunnerE2ESource({ sha, ref: null, workflowRunUrl: null }); const definitionHash = createHash("sha256").update(JSON.stringify(productJourneys)); - for (const name of ["catalog.ts", "task-properties.ts", "task-search-filter.ts", "task-search-evidence.ts", "basic-flows.ts", "basic-evidence.ts", "project-context-flow.ts", "project-context-readback.ts", "project-context-evidence.ts", "agent-flows.ts", "clarification-flow.ts", "clarification-evidence.ts", "../question-resume-ui.ts", "../question-resume-scoring.ts", "../runtime-question-readiness.ts", "../chat-flow.ts", "../continuation-readiness.ts", "../continuation-cases.ts", "success-contract.ts", "product-screenshot.ts", "cloud-onboarding.ts", "onboarding-connect.ts", "onboarding-first-task.ts", "workspace-deferral.ts", "onboarding-pre-acceptance.ts", "artifact-evidence.ts", "fixtures.ts", "launch.ts", "types.ts", "collection.ts", "local-target.ts"]) definitionHash.update(await readFile(path.join(import.meta.dirname, name))); + for (const name of ["catalog.ts", "task-properties.ts", "task-search-filter.ts", "task-search-evidence.ts", "basic-flows.ts", "basic-evidence.ts", "project-context-flow.ts", "project-context-readback.ts", "project-context-evidence.ts", "agent-flows.ts", "active-followup-flow.ts", "active-followup-evidence.ts", "clarification-flow.ts", "clarification-evidence.ts", "../question-resume-ui.ts", "../question-resume-scoring.ts", "../runtime-question-readiness.ts", "../chat-flow.ts", "../continuation-readiness.ts", "../continuation-cases.ts", "success-contract.ts", "product-screenshot.ts", "cloud-onboarding.ts", "onboarding-connect.ts", "onboarding-first-task.ts", "workspace-deferral.ts", "onboarding-pre-acceptance.ts", "artifact-evidence.ts", "fixtures.ts", "launch.ts", "types.ts", "collection.ts", "local-target.ts"]) definitionHash.update(await readFile(path.join(import.meta.dirname, name))); for (const file of ["catalog.ts", "api.ts", "run-observations.ts", "native-task-reply.ts", "native-event-envelope.ts", "user-actions.ts", "plan-revision-flow.ts", "skill-studio-flow.ts", "protected-action-flow.ts", "story-artifact.ts", "stop-redirect-flow.ts", "everyday-cases.ts", "everyday-delivery.ts", "everyday-decisions.ts", "connection-guidance-evidence.ts", "everyday-artifact.py"]) definitionHash.update(await readFile(path.join(import.meta.dirname, "..", file))); definitionHash.update(runnerSuites.flatMap(suite => suite.tasks).find(task => task.id === "ask-question")!.buildPrompt("definition")); const suiteDefinitionHash = definitionHash.digest("hex"); @@ -203,6 +204,8 @@ export async function launchProductJourneys(args: string[], repositoryRoot: stri result.model = ready.agent.model; result.runtimeMode = ready.agent.runtimeMode; if (id === "ask-question") await askQuestion(ready, session!, campaignId, evidence); + else if (id === "followup-while-running") await runActiveFollowup(ready, session!, campaignId, evidence, + { deadlineAt, directory: path.join(outputDir, "private-stories", id) }); else if (id === "question-answer-resume") await runClarificationJourney(ready, session!, campaignId, evidence, { deadlineAt, directory: path.join(outputDir, "private-stories", id), capture: async (_id, _label, file) => { diff --git a/tests/runner-e2e/product-journeys/success-contract.ts b/tests/runner-e2e/product-journeys/success-contract.ts index f49db32a938..67d37578e3e 100644 --- a/tests/runner-e2e/product-journeys/success-contract.ts +++ b/tests/runner-e2e/product-journeys/success-contract.ts @@ -1,11 +1,11 @@ /** Closed, target-independent success requirements used by dispatch and retained evidence. */ export const agentJourneyRuns = { "ask-question": 1, "plan-revise-accept": 3, "build-revise": 2, - "protected-actions": 4, "create-skill-studio": 1, "stop-redirect": 3, "question-answer-resume": 3 } as const; + "followup-while-running": 2, "protected-actions": 4, "create-skill-studio": 1, "stop-redirect": 3, "question-answer-resume": 3 } as const; export function agentJourneyRunCountValid(id: string, count: unknown): boolean { if (!Object.hasOwn(agentJourneyRuns, id) || !Number.isSafeInteger(count)) return false; const maximum = agentJourneyRuns[id as keyof typeof agentJourneyRuns]; - const minimum = id === "question-answer-resume" ? 1 : maximum; + const minimum = id === "question-answer-resume" || id === "followup-while-running" ? 1 : maximum; return (count as number) >= minimum && (count as number) <= maximum; } export const journeyRequiredChecks: Record = { @@ -24,5 +24,6 @@ export const journeyRequiredChecks: Record = { "create-skill-studio": ["created-skill-persisted", "skill-creation-attributed", "skill-studio-edit-persisted", "configured-model-and-environment"], "stop-redirect": ["checkpoint-artifact-verified", "active-work-stopped", "stop-native-terminal-retained", "new-direction-completed", "stopped-run-remained-terminal", "stopped-run-no-later-events", "stop-preserved-download", "redirect-consumed-once", "user-reply-stored-once", "configured-model-and-environment"], + "followup-while-running": ["followup-native-command-active", "followup-stored-once", "followup-consumed-once", "followup-final-answer-persisted", "followup-one-or-two-native-runs", "configured-model-and-environment"], "cloud-onboarding": ["fresh-workspace", "ui-connection-and-native-agent", "first-task-complete", "downloadable-artifact-verified", "native-trace-verified"], }; diff --git a/tests/runner-e2e/product-journeys/types.ts b/tests/runner-e2e/product-journeys/types.ts index 0f899efca5e..24f57c2cf7f 100644 --- a/tests/runner-e2e/product-journeys/types.ts +++ b/tests/runner-e2e/product-journeys/types.ts @@ -3,7 +3,7 @@ import type { ConnectionApi } from "../connection-target.js"; export type JourneyOutcome = "passed" | "failed" | "blocked" | "unsupported"; export type JourneyId = "task-properties" | "task-create" | "project-task-context" | "ask-question" | "cloud-onboarding" - | "question-answer-resume" | "task-search-filter" | "plan-revise-accept" | "build-revise" | "protected-actions" | "create-skill-studio" | "stop-redirect"; + | "question-answer-resume" | "task-search-filter" | "followup-while-running" | "plan-revise-accept" | "build-revise" | "protected-actions" | "create-skill-studio" | "stop-redirect"; export type TargetMode = "managed-self-hosted" | "existing-cloud" | "new-cloud"; export interface DeploymentIdentity { kind: "self-hosted" | "cloud"; diff --git a/tests/runner-e2e/report-catalog.test.ts b/tests/runner-e2e/report-catalog.test.ts index 28929783eb1..09d6bd6d6d7 100644 --- a/tests/runner-e2e/report-catalog.test.ts +++ b/tests/runner-e2e/report-catalog.test.ts @@ -11,6 +11,7 @@ import { buildRunnerCampaign } from "./history.js"; import { renderRunnerE2EDashboard } from "./dashboard.js"; import { regenerateRunnerDashboard } from "./dashboard-regenerate.js"; import { journeyCapabilities, journeyFeatureIds } from "./journey-features.js"; +import { journeyRequiredChecks } from "./product-journeys/success-contract.js"; import type { RunnerE2EResult } from "./types.js"; const temporary: string[] = []; @@ -91,6 +92,30 @@ describe("trusted report catalog discovery", () => { expect(journeyCapabilities({ ...product, task: { ...known.task, id: "task-search-filter" } }).map(capability => capability.id)).toEqual(["tasks/organize"]); }); + it.each([ + { caseId: "question-answer-resume", feature: "questions-and-approvals", capabilities: ["questions-and-approvals/questions", "questions-and-approvals/continuation"] }, + { caseId: "followup-while-running", feature: "steering", capabilities: ["steering/queue"] }, + ])("reports deployment journey $caseId in its feature coverage", ({ caseId, feature, capabilities }) => { + const known = runnerMatrix.find(execution => execution.id === "core-compatibility.legacy-codex.local.ask-question")!; + const execution = { ...known, id: `product-journeys.${known.profile.id}.${known.environment.id}.${caseId}`, + suite: { ...known.suite, id: "product-journeys" }, task: { ...known.task, id: caseId } }; + expect(journeyCapabilities(execution).map(capability => capability.id)).toEqual(capabilities); + expect(journeyFeatureIds(execution)).toEqual([feature]); + expect(journeyCapabilities({ ...execution, suite: { ...execution.suite, id: "unrecognized-suite" } })).toEqual([]); + const page = renderRunnerE2EDashboard({ title: "Deployment coverage", generatedAt: "2026-10-20T12:00:00Z", + expected: [execution.id], catalog: [execution], entries: [{ valid: true, errors: [], + result: { ...result(), executionId: execution.id, suiteId: execution.suite.id, profileId: execution.profile.id, + environmentId: execution.environment.id, caseId, provider: "opencode", runIds: ["run-1"], + productJourney: { targetId: "qa", outcome: "passed", startingState: "recipe", privacy: "private", resourceDisposition: "retained", + phases: [{ phase: "journey", outcome: "passed" }], checks: journeyRequiredChecks[caseId]!.map(id => ({ id, passed: true })), + deployment: { kind: "cloud", targetId: "qa", origin: "https://qa.example.test", commit: "a".repeat(40), + cloudRelease: "b".repeat(40), runtimeImage: "immutable-image", configurationHash: "config" } } } }] }); + expect(page).toContain(`data-execution-id="${execution.id}"`); + expect(page).toContain(`data-report-features="${feature}"`); + expect(page).toContain(`title="${capabilities.join(", ")}"`); + expect(page).toContain("1/1 passed"); + }); + it("counts the project/task deployment journey toward projects and task creation coverage", () => { const known = runnerMatrix.find(execution => execution.id === "core-compatibility.legacy-codex.local.ask-question")!; const project = { ...known, suite: { ...known.suite, id: "product-journeys" }, task: { ...known.task, id: "project-task-context" } }; From 594a4ee657b6194ec31c26bd0e088cd94bfd77d9 Mon Sep 17 00:00:00 2001 From: Dotta Date: Fri, 30 Oct 2026 18:41:23 -0500 Subject: [PATCH 53/74] fix(external-objects): fence selected plugin results Revalidate the exact selected plugin before result persistence even when an allowed fallback still supports the object. Preserve due debt and token ownership so a fresh allowed resolver can proceed. Co-Authored-By: Paperclip --- .../2026-10-30-external-objects-sleep.md | 10 +++- .../external-object-refresh-work.test.ts | 54 ++++++++++++++++++ server/src/services/external-objects.ts | 55 +++++++++++-------- 3 files changed, 96 insertions(+), 23 deletions(-) diff --git a/doc/plans/2026-10-30-external-objects-sleep.md b/doc/plans/2026-10-30-external-objects-sleep.md index 41bf59440d1..e5401596a0e 100644 --- a/doc/plans/2026-10-30-external-objects-sleep.md +++ b/doc/plans/2026-10-30-external-objects-sleep.md @@ -18,6 +18,14 @@ ready plugin with `external.objects.read`, a matching provider/object declaratio and no company setting that disables it. Built-in resolvers remain available independently of optional plugin support. +A plugin result carries its selected plugin ID inside the host owner. Persistence +rechecks that exact plugin's readiness, read capability, object declaration and +same-company enabled setting in the result update. Another available resolver +cannot authorize an obsolete plugin response. A rejected response releases only +its own claim; the due row remains for a fresh allowed resolver. Plugin RPC +payloads and public result shapes do not contain this host selection context. + + Terminal objects are passive. The existing exception remains: GitHub `not_found` snapshots without `data.probeVerified=true` need a repository-visibility probe before their terminal verdict is definitive. A null deadline retains its @@ -94,7 +102,7 @@ reactivation in the tested process; it does not establish Cloud alarm coverage, host/database suspension or deployed savings. -## Host cancellation and explicit legacy entry points +## Host cancellation and registered ownership The resolver contract accepts an optional host `AbortSignal`. The worker passes its coordinator signal through refresh to the resolver. This reuses the narrow diff --git a/server/src/__tests__/external-object-refresh-work.test.ts b/server/src/__tests__/external-object-refresh-work.test.ts index 4c1817b9722..8aab42b35b1 100644 --- a/server/src/__tests__/external-object-refresh-work.test.ts +++ b/server/src/__tests__/external-object-refresh-work.test.ts @@ -257,6 +257,60 @@ const active: ExternalObjectResolveResult = { ok: true, snapshot: { expect(calls).toHaveBeenCalledOnce(); }); + it.each(["company setting", "status", "uninstall", "manifest", "other company"] as const)( + "revalidates the selected plugin after %s changes even while the built-in fallback remains available", async change => { + clock(); const registry = pluginRegistryService(db); + const manifest: PaperclipPluginManifestV1 = { + id: `paperclip.selected-${randomUUID()}`, apiVersion: 1, version: "1.0.0", displayName: "Selected provider", + description: "Selected resolver fixture", author: "Paperclip", categories: ["connector"], + capabilities: ["external.objects.detect", "external.objects.read"], entrypoints: { worker: "worker.js" }, + objectReferences: [{ providerKey: "github", displayName: "GitHub", objectTypes: ["pull_request"], urlPatterns: ["https://github.com/:owner/:repo/pull/:id"] }], + }; + const plugin = await registry.install({ packageName: "selected-provider-fixture" }, manifest); + await registry.updateStatus(plugin!.id, { status: "ready" }); + let finishPlugin!: (result: ExternalObjectResolveResult) => void; + const pluginResult = new Promise(resolve => { finishPlugin = resolve; }); + let selected = false; + const calls = vi.fn(async (_id: string, method: string) => { + if (method === "detectExternalObjects") return { detections: [] }; + selected = true; return pluginResult; + }); + let finishFallback!: () => void; + const fallbackResult = new Promise(resolve => { finishFallback = resolve; }); + const fallback = vi.fn(async () => { + await fallbackResult; + return new Response(JSON.stringify({ state: "closed", merged: true, title: "Allowed fallback" }), { status: 200 }); + }); + const svc = externalObjectService(db, { pluginWorkerManager: { call: calls } as unknown as PluginWorkerManager, + github: { fetch: fallback, credentialProvider: () => ({ token: "simulated-company-token", source: "company_secret" }) } }); + try { + await start(svc); const f = await fixture(); + await db.update(issues).set({ description: "https://github.com/acme/app/pull/42" }).where(eq(issues.id, f.issueId)); + await svc.syncIssue(f.issueId); await settle(() => selected); + if (change === "company setting") await registry.upsertCompanySettings(plugin!.id, f.companyId, { enabled: false, settingsJson: {} }); + else if (change === "status") await registry.updateStatus(plugin!.id, { status: "error" }); + else if (change === "uninstall") await registry.uninstall(plugin!.id, true); + else if (change === "manifest") await registry.update(plugin!.id, { manifest: { ...manifest, capabilities: [], objectReferences: [] } }); + else { + const other = await fixture(); + await registry.upsertCompanySettings(plugin!.id, other.companyId, { enabled: false, settingsJson: {} }); + } + finishPlugin({ ok: true, snapshot: { statusKey: "plugin_result", statusCategory: "succeeded", statusTone: "success", isTerminal: true } }); + if (change === "other company") { + await settle(async () => (await row(f.companyId))?.statusKey === "plugin_result"); await quiet(); + expect(fallback).not.toHaveBeenCalled(); + } else { + await settle(async () => fallback.mock.calls.length > 0 || Boolean((await row(f.companyId))?.lastResolvedAt)); + const beforeFallback = await row(f.companyId); + expect(beforeFallback.isTerminal).toBe(false); expect(beforeFallback.statusKey).toBeNull(); expect(beforeFallback.lastResolvedAt).toBeNull(); + expect(fallback).toHaveBeenCalledOnce(); + finishFallback(); + await settle(async () => (await row(f.companyId))?.statusKey === "merged"); await quiet(); + expect((await row(f.companyId)).displayTitle).toBe("acme/app#42: Allowed fallback"); + } + } finally { finishPlugin(terminal); finishFallback(); } + }); + it("reactivates refresh after ordinary budget pause and budget raise", async () => { clock(); const svc = service(); provider.mockResolvedValue(active); await start(svc); const f = await fixture(); await svc.syncIssue(f.issueId); diff --git a/server/src/services/external-objects.ts b/server/src/services/external-objects.ts index a4d26014d26..81e10a208f2 100644 --- a/server/src/services/external-objects.ts +++ b/server/src/services/external-objects.ts @@ -346,7 +346,7 @@ async function resolveViaPluginProvider( db: Db, pluginWorkerManager: PluginWorkerManager | undefined, object: ExternalObjectRecord, -): Promise { +): Promise<{ pluginId: string; result: PluginExternalObjectResolveResult } | null> { if (!pluginWorkerManager) return null; const providers = await readyObjectReferencePlugins(db, object.companyId); for (const provider of providers) { @@ -354,23 +354,27 @@ async function resolveViaPluginProvider( if (!manifest.capabilities.includes("external.objects.read")) continue; if (!manifestProvidesObject(manifest, object)) continue; try { - return await pluginWorkerManager.call(provider.id, "resolveExternalObject", { + const result = await pluginWorkerManager.call(provider.id, "resolveExternalObject", { companyId: object.companyId, providerKey: object.providerKey, objectType: object.objectType, externalId: object.externalId, object: objectSnapshot(object), }); + return { pluginId: provider.id, result }; } catch (err) { logger.warn( { err, pluginId: provider.id, pluginKey: provider.pluginKey, objectId: object.id }, "plugin external object resolver failed", ); return { - ok: false, - liveness: "unreachable", - errorCode: "plugin_resolver_failed", - errorMessage: err instanceof Error ? err.message : String(err), + pluginId: provider.id, + result: { + ok: false, + liveness: "unreachable", + errorCode: "plugin_resolver_failed", + errorMessage: err instanceof Error ? err.message : String(err), + }, }; } } @@ -402,12 +406,9 @@ export function externalObjectService( ]; const resolverRegistry = createExternalObjectResolverRegistry(resolvers); - function automaticRefreshWhere(database: Db = db) { - const supported = resolvers.map(resolver => and( - eq(externalObjects.providerKey, resolver.providerKey), - resolver.objectType ? eq(externalObjects.objectType, resolver.objectType) : undefined, - )!); - if (opts.pluginWorkerManager) supported.push(exists(database.select({ id: plugins.id }).from(plugins).where(and( + function pluginRefreshWhere(database: Db, selectedPluginId?: string) { + return exists(database.select({ id: plugins.id }).from(plugins).where(and( + selectedPluginId ? eq(plugins.id, selectedPluginId) : undefined, eq(plugins.status, "ready"), sql`${plugins.manifestJson}->'capabilities' ? 'external.objects.read'`, sql`exists (select 1 from jsonb_array_elements(coalesce(${plugins.manifestJson}->'objectReferences', '[]'::jsonb)) as reference @@ -418,7 +419,15 @@ export function externalObjectService( eq(pluginCompanySettings.companyId, externalObjects.companyId), eq(pluginCompanySettings.enabled, false), )))), - )))); + ))); + } + + function automaticRefreshWhere(database: Db = db) { + const supported = resolvers.map(resolver => and( + eq(externalObjects.providerKey, resolver.providerKey), + resolver.objectType ? eq(externalObjects.objectType, resolver.objectType) : undefined, + )!); + if (opts.pluginWorkerManager) supported.push(pluginRefreshWhere(database)); return and( refreshableObjectWhere(), isNotNull(externalObjects.nextRefreshAt), supported.length ? or(...supported) : sql`false`, @@ -863,12 +872,14 @@ export function externalObjectService( } async function persistRefresh(object: ExternalObjectRecord, token: string, - patch: Partial, scheduled = false) { + patch: Partial, scheduled = false, selectedPluginId?: string) { return db.transaction(async tx => { await notifyExternalObjectRefresh(tx, object.companyId, object.id); const rows = await tx.update(externalObjects).set(patch) - .where(and(refreshOwnerWhere(object, token), scheduled ? automaticRefreshWhere(tx as unknown as Db) : undefined)).returning(); - if (scheduled && rows.length === 0) await tx.update(externalObjects) + .where(and(refreshOwnerWhere(object, token), + scheduled ? automaticRefreshWhere(tx as unknown as Db) : undefined, + selectedPluginId ? pluginRefreshWhere(tx as unknown as Db, selectedPluginId) : undefined)).returning(); + if ((scheduled || selectedPluginId) && rows.length === 0) await tx.update(externalObjects) .set({ refreshStartedAt: null, refreshToken: null }).where(refreshOwnerWhere(object, token)); return rows; }); @@ -881,9 +892,9 @@ export function externalObjectService( refreshToken: string, ) { input.signal?.throwIfAborted(); - const pluginResult = await observeRefreshCancellation(resolveViaPluginProvider(db, opts.pluginWorkerManager, object), input.signal); - const resolver = pluginResult ? null : resolverRegistry.find(object); - if (!pluginResult && !resolver) { + const pluginResolution = await observeRefreshCancellation(resolveViaPluginProvider(db, opts.pluginWorkerManager, object), input.signal); + const resolver = pluginResolution ? null : resolverRegistry.find(object); + if (!pluginResolution && !resolver) { const [updated] = await persistRefresh(object, refreshToken, { liveness: object.isTerminal ? object.liveness : visibleLiveness(object, now) === "fresh" ? "stale" : object.liveness, nextRefreshAt: addSeconds(now, DEFAULT_RETRY_AFTER_SECONDS), @@ -895,7 +906,7 @@ export function externalObjectService( return { object: toObjectPayload(updated ?? object, now), refreshed: false, reason: "no_resolver" as const }; } - const result = pluginResult ?? await observeRefreshCancellation( + const result = pluginResolution?.result ?? await observeRefreshCancellation( resolver!.resolve({ companyId: object.companyId, object, signal: input.signal }), input.signal); input.signal?.throwIfAborted(); if (!result.ok) { @@ -913,7 +924,7 @@ export function externalObjectService( refreshStartedAt: null, refreshToken: null, updatedAt: now, - }, input.scheduled); + }, input.scheduled, pluginResolution?.pluginId); if (!updated) return refreshSupersededResult(object, now); publishLiveEvent({ companyId: object.companyId, @@ -950,7 +961,7 @@ export function externalObjectService( const [updated] = await persistRefresh(object, refreshToken, { ...patch, lastChangedAt: objectChanged(object, { ...object, ...patch }) ? now : object.lastChangedAt, - }, input.scheduled); + }, input.scheduled, pluginResolution?.pluginId); if (!updated) return refreshSupersededResult(object, now); const next = updated ?? object; if (objectChanged(object, next) && input.actor) { From 4c4e4a469c87ba4e27f0e2d16976962d8f864a04 Mon Sep 17 00:00:00 2001 From: Dotta Date: Fri, 30 Oct 2026 18:44:34 -0500 Subject: [PATCH 54/74] fix: retain expired backup cleanup after listing failures --- .../2026-10-30-backup-expiration-sleep.md | 6 +++++ .../native-harness-backup-expiration.test.ts | 24 +++++++++++++++++++ .../native-harness-backup-expiration.ts | 2 +- .../native-session-executor.test.ts | 22 ++++++++++++----- 4 files changed, 47 insertions(+), 7 deletions(-) diff --git a/doc/plans/2026-10-30-backup-expiration-sleep.md b/doc/plans/2026-10-30-backup-expiration-sleep.md index e5b3132fbc8..307b4f67f47 100644 --- a/doc/plans/2026-10-30-backup-expiration-sleep.md +++ b/doc/plans/2026-10-30-backup-expiration-sleep.md @@ -55,3 +55,9 @@ Acceptance uses isolated real PostgreSQL for service writes, transaction signals safety checks, resume pointer deletion, and restart reconstruction. A temporary filesystem is the controlled storage boundary. These tests do not claim cloud object storage behavior, deployed sleep, or fleet savings. + +Review validation also covers expired-only directories whose listing fails with +EIO or EACCES. Only ENOENT means absent; other listing failures retain retry debt. +Both controls failed before the fix; all 104 expiration tests pass afterward. +Executor release tests assert both the preserved broad cleanup scope and the +precise source identity used for backup invalidation. diff --git a/server/src/__tests__/native-harness-backup-expiration.test.ts b/server/src/__tests__/native-harness-backup-expiration.test.ts index 9dddca536f3..9d16ec519ff 100644 --- a/server/src/__tests__/native-harness-backup-expiration.test.ts +++ b/server/src/__tests__/native-harness-backup-expiration.test.ts @@ -868,6 +868,30 @@ describeEmbeddedPostgres("native harness backup expiration", () => { expect(await exists(own.current)).toBe(false); expect(await taskSessionCount(own.agentId)).toBe(0); await quiet(); }); + it.each(["EIO", "EACCES"])("retains expired-only cleanup after a directory read failure (%s)", async code => { + const session = await seedSession(); + const backupRoot = path.dirname(session.current); + const expired = path.join(backupRoot, `.expired-${randomUUID()}`); + await fs.rename(session.current, expired); + const read = fs.readdir; + const fault = vi.spyOn(fs, "readdir").mockImplementation(async (...args) => { + if (String(args[0]) === backupRoot) throw Object.assign(new Error("controlled cleanup listing failure"), { code }); + return read(...args); + }); + await startBackups(); + expect(backupErrors).toHaveLength(1); + const retry = coordinator!.nextWakeAt(); + expect(retry).not.toBeNull(); + expect(retry! - Date.now()).toBeGreaterThanOrEqual(29_000); + expect(await exists(session.current)).toBe(false); + expect(await exists(expired)).toBe(true); + fault.mockRestore(); + await quiet(5000); + await vi.advanceTimersByTimeAsync(retry! - Date.now()); await settled(); + expect(await fs.readdir(backupRoot)).toEqual([]); + await quiet(); + }); + it("reconstructs future retention without issue row locks", async () => { await seedSession({ closedDaysAgo: 2 }); const queries: string[] = []; const previous = db.$client.options.debug; diff --git a/server/src/services/native-runtime/native-harness-backup-expiration.ts b/server/src/services/native-runtime/native-harness-backup-expiration.ts index b1d8c7c6f1d..1d6a43169b9 100644 --- a/server/src/services/native-runtime/native-harness-backup-expiration.ts +++ b/server/src/services/native-runtime/native-harness-backup-expiration.ts @@ -352,7 +352,7 @@ export function nativeHarnessBackupExpirationService( async function settleLeftovers(backupRoot: string) { const current = path.join(backupRoot, "current"); - for (const name of await fs.readdir(backupRoot).catch(() => [] as string[])) { + for (const name of (await fs.readdir(backupRoot).catch(missingPath)) ?? []) { const target = path.join(backupRoot, name); if (name.startsWith(RETIRING_PREFIX) && UUID_PATTERN.test(name.slice(RETIRING_PREFIX.length))) { if (!(await isRealDirectory(current))) await fs.rename(target, current); diff --git a/server/src/services/native-runtime/native-session-executor.test.ts b/server/src/services/native-runtime/native-session-executor.test.ts index 4dabeda6954..d67ce4074df 100644 --- a/server/src/services/native-runtime/native-session-executor.test.ts +++ b/server/src/services/native-runtime/native-session-executor.test.ts @@ -5103,7 +5103,10 @@ describe("retained native cleanup activation", () => { } const outcome = await pendingOutcome; if (mode === "legacy_busy_copy") { - expect(maintenanceReleased).toHaveBeenCalledExactlyOnceWith({ companyId: run.companyId, runId: run.id }); + expect(maintenanceReleased).toHaveBeenCalledExactlyOnceWith( + { companyId: run.companyId, runId: run.id }, + { companyId: run.companyId, runId: run.id }, + ); unsubscribeMaintenance?.(); } if (canonicalSource) { @@ -6425,7 +6428,10 @@ describe("native startup cancellation fence", () => { })).rejects.toThrow("native_cancellation_pending_recovery"); expect(cancel).toHaveBeenCalledOnce(); expect(submit).not.toHaveBeenCalled(); - expect(released).toHaveBeenCalledWith({ companyId: execution.binding.companyId, agentId: execution.binding.agentId }); + expect(released).toHaveBeenCalledWith( + { companyId: execution.binding.companyId, agentId: execution.binding.agentId }, + { companyId: execution.binding.companyId, runId: execution.binding.runId, issueId: execution.binding.issueId }, + ); } finally { unsubscribe(); } }); }); @@ -9872,9 +9878,10 @@ describe("native warm session supervision", () => { ); expect(released).not.toHaveBeenCalled(); releaseClose(); - await vi.waitFor(() => expect(released).toHaveBeenCalledExactlyOnceWith({ - companyId: base.binding.companyId, agentId: base.binding.agentId, - })); + await vi.waitFor(() => expect(released).toHaveBeenCalledExactlyOnceWith( + { companyId: base.binding.companyId, agentId: base.binding.agentId }, + { companyId: base.binding.companyId, runId: base.binding.runId, issueId: base.binding.issueId }, + )); } finally { releaseClose(); unsubscribe(); } }); @@ -12453,7 +12460,10 @@ describe("runnerd provider runtime wiring", () => { releaseEnvironment(); releaseAssigned(); const result = await preparation; - expect(released).toHaveBeenCalledExactlyOnceWith({ companyId: execution.binding.companyId, agentId: execution.binding.agentId }); + expect(released).toHaveBeenCalledExactlyOnceWith( + { companyId: execution.binding.companyId, agentId: execution.binding.agentId }, + { companyId: execution.binding.companyId, runId: execution.binding.runId, issueId: execution.binding.issueId }, + ); if (failure !== "success") { expect(result).toMatchObject({ error: { message: failure === "assigned" ? "assigned failed" : "environment failed" } }); expect(state.createBackend).not.toHaveBeenCalled(); From 9e904ee5194f6f126701f1798faf792695a1d327 Mon Sep 17 00:00:00 2001 From: Nicky Leach Date: Fri, 30 Oct 2026 16:49:20 -0700 Subject: [PATCH 55/74] perf(server): overlap independent heartbeat preparation reads (#18372) --- doc/observability.md | 8 + .../heartbeat-execution-configuration.test.ts | 57 ++++++- .../__tests__/heartbeat-task-context.test.ts | 100 ++++++++++- server/src/services/heartbeat.ts | 47 +----- .../heartbeat/execution-configuration.ts | 47 ++++-- .../heartbeat/run-preparation.test.ts | 43 +++++ .../src/services/heartbeat/run-preparation.ts | 155 +++++++----------- server/src/services/heartbeat/task-context.ts | 92 +++++++---- .../runner-api.integration.test.ts | 2 +- 9 files changed, 354 insertions(+), 197 deletions(-) diff --git a/doc/observability.md b/doc/observability.md index e2bb4065caf..996d3a807f5 100644 --- a/doc/observability.md +++ b/doc/observability.md @@ -328,6 +328,14 @@ once known, `paperclip.native_reconciliation.candidates` (a count). A failed pass marks the pass span and the phase that was running with an error status. Faster passes record nothing. +Heartbeat configuration preparation overlaps independent reads. Skill preparation +runs alongside connector assignments; standing connection instructions wait for +assignment catalog repairs, but can overlap the skill branch. These spans can +finish out of order. Do not add their durations as serial elapsed time. The +historical `skills.prepare` span ends when the skill branch settles, rather than +when its concurrent branches finish. Each preparation attempt settles its started +branches before it returns an error or releases acquired credentials. + ### Detailed execution phases The `paperclip.execution` tracer adds these content-free spans beneath the current diff --git a/server/src/__tests__/heartbeat-execution-configuration.test.ts b/server/src/__tests__/heartbeat-execution-configuration.test.ts index 509a9cfa7e6..2cd52f84849 100644 --- a/server/src/__tests__/heartbeat-execution-configuration.test.ts +++ b/server/src/__tests__/heartbeat-execution-configuration.test.ts @@ -19,11 +19,20 @@ import { secretService } from "../services/secrets.js"; import { instanceSettingsService } from "../services/instance-settings.js"; import { aiConnectionService } from "../services/ai-connections.js"; import * as aiRuntime from "../services/ai-connection-runtime.js"; +import * as connectorRuntime from "../services/connector-runtime.js"; +import * as assignedMcp from "../services/native-runtime/assigned-mcp-tools.js"; import * as sentry from "../sentry.js"; import { waitForPendingRunFailureReports } from "../services/run-failure-report.js"; import { recoveryPostgresSupport, startRecoveryPostgres } from "./helpers/recovery-postgres.js"; +function deferred() { + let resolve!: (value: T) => void; + const promise = new Promise(complete => { resolve = complete; }); + return { promise, resolve }; +} + const skillFailures = vi.hoisted(() => new Map()); +const skillWaits = vi.hoisted(() => new Map Promise>()); describe("Runner home mode during recovery preparation", () => { it.each([ @@ -52,6 +61,7 @@ vi.mock("../services/company-skills.js", async importOriginal => { return { ...service, withRuntimeSkillEntries: async (companyId: string, options: Parameters[1], consume: (entries: Awaited>) => T | Promise) => { + await skillWaits.get(companyId)?.(); const failure = skillFailures.get(companyId); if (failure) throw new Error(failure); return service.withRuntimeSkillEntries(companyId, options, consume); @@ -91,7 +101,7 @@ describePostgres("heartbeat execution configuration through adapter dispatch", ( testEnvironment: async () => ({ adapterType, status: "pass", checks: [], testedAt: new Date().toISOString() }), }); }, 60_000); - afterEach(() => { vi.restoreAllMocks(); vi.unstubAllEnvs(); skillFailures.clear(); }); + afterEach(() => { vi.restoreAllMocks(); vi.unstubAllEnvs(); skillFailures.clear(); skillWaits.clear(); }); afterAll(async () => { await heartbeat?.drainActiveRunExecutions(); unregisterServerAdapter(adapterType); @@ -288,6 +298,45 @@ describePostgres("heartbeat execution configuration through adapter dispatch", ( const [unchangedAgent] = await db.select().from(agents).where(eq(agents.id, f.agentId)); expect(unchangedAgent.adapterConfig).toMatchObject({ paperclipSkillSync: { desiredSkills: [other.key] } }); }, 60_000); + it("overlaps skills with assignments and dependent connection instructions before composition", async () => { + const f = await seed(); + const skillGate = deferred(); + skillWaits.set(f.companyId, () => skillGate.promise); + const assignments = vi.spyOn(connectorRuntime, "resolveConnectorAssignments"); + const instructions = vi.spyOn(assignedMcp, "resolveAssignedConnectionInstructionsForRun"); + const configured = await configuration(f, { provider: "codex", model: "gpt-6.1-sol", homeMode: "isolated" }); + const opening = configured.prepare(); + try { + await vi.waitFor(() => { + expect(assignments).toHaveBeenCalledOnce(); + expect(instructions).toHaveBeenCalledOnce(); + }); + } finally { + skillGate.resolve(); + } + expect(await opening).toMatchObject({ prepared: true, runtimeConfig: { homeMode: "isolated", model: "gpt-6.1-sol" } }); + }); + + it("settles concurrent configuration reads before exposing the first preparation failure", async () => { + const f = await seed(); + const assignmentGate = deferred>>(); + const assignments = vi.spyOn(connectorRuntime, "resolveConnectorAssignments").mockReturnValue(assignmentGate.promise); + const instructions = vi.spyOn(assignedMcp, "resolveAssignedConnectionInstructionsForRun").mockRejectedValue(new Error("later instruction failure")); + skillFailures.set(f.companyId, "first skill failure"); + const configured = await configuration(f, { provider: "codex", homeMode: "isolated" }); + let settled = false; + const opening = configured.prepare(); + void opening.then(() => { settled = true; }, () => { settled = true; }); + try { + await vi.waitFor(() => expect(assignments).toHaveBeenCalledOnce()); + await Promise.resolve(); + expect(settled).toBe(false); + expect(instructions).not.toHaveBeenCalled(); + } finally { + assignmentGate.resolve([]); + } + await expect(opening).rejects.toThrow("first skill failure"); + }); it("does not install new environment declarations when recovering a saved skill selection", async () => { const f = await seed(), pluginId = randomUUID(), pluginKey = `fixture.${randomUUID()}`; @@ -445,11 +494,11 @@ describePostgres("heartbeat execution configuration through adapter dispatch", ( const runIdHash = createHash("sha256").update(finished.id).digest("hex").slice(0, 12); const steps = exporter.getFinishedSpans().filter(span => span.name.startsWith("heartbeat.prepare.configuration_") && span.attributes["paperclip.task.run.run_id"] === runIdHash); - expect(steps.map(span => span.name)).toEqual([ + expect(steps.map(span => span.name).sort()).toEqual([ "configuration_mentions", "configuration_github_identity", "configuration_secrets", "configuration_ai_runtime", "configuration_skills", "configuration_connector_assignments", "configuration_connector_skills", "configuration_connector_delivery", "configuration_connection_instructions", "configuration_agent_identity", - ].map(step => `heartbeat.prepare.${step}`)); + ].map(step => `heartbeat.prepare.${step}`).sort()); for (const span of steps) { expect(span.status).toEqual({ code: 0 }); expect(span.attributes).toEqual({ "paperclip.task.run.run_id": runIdHash }); @@ -494,7 +543,7 @@ describePostgres("heartbeat execution configuration through adapter dispatch", ( && span.attributes["paperclip.task.run.run_id"] === runIdHash); expect(steps.find(span => span.name === "heartbeat.prepare.configuration_ai_runtime")?.status.code).toBe(0); expect(steps.find(span => span.name === "heartbeat.prepare.configuration_skills")?.status).toEqual({ code: 2 }); - expect(steps.some(span => span.name === "heartbeat.prepare.configuration_connector_assignments")).toBe(false); + expect(steps.some(span => span.name === "heartbeat.prepare.configuration_connector_assignments")).toBe(true); for (const span of steps) { expect(span.attributes).toEqual({ "paperclip.task.run.run_id": runIdHash }); expect(span.events).toEqual([]); diff --git a/server/src/__tests__/heartbeat-task-context.test.ts b/server/src/__tests__/heartbeat-task-context.test.ts index 5449d7def6c..e073db37676 100644 --- a/server/src/__tests__/heartbeat-task-context.test.ts +++ b/server/src/__tests__/heartbeat-task-context.test.ts @@ -1,10 +1,11 @@ import { createHash, randomUUID } from "node:crypto"; import { eq } from "drizzle-orm"; -import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { afterAll, beforeAll, describe, expect, it, vi } from "vitest"; import { agents, authUsers, companies, companyMemberships, createDb, heartbeatRuns, issueComments, issues } from "@paperclipai/db"; import { FAST_RESPONSE_AGENT_GUIDANCE } from "@paperclipai/shared"; import { registerServerAdapter, unregisterServerAdapter, type AdapterExecutionContext } from "../adapters/index.js"; import { heartbeatService } from "../services/heartbeat.js"; +import { prepareHeartbeatTaskContext } from "../services/heartbeat/task-context.js"; import { documentService } from "../services/documents.js"; import { ISSUE_CONTINUATION_SUMMARY_DOCUMENT_KEY } from "../services/issue-continuation-summary.js"; import { createRunSecretRedactionRegistry } from "../services/run-secret-redaction.js"; @@ -19,6 +20,7 @@ if (!support.supported) console.warn(`Skipping heartbeat task context tests: ${s describePostgres("heartbeat task context delivered to an adapter", () => { const adapterType = "task_context_test"; const captured = new Map(); + const inspectPreparation = new Map void>(); let temporary: Awaited>; let db: ReturnType; let heartbeat: ReturnType; @@ -33,6 +35,7 @@ describePostgres("heartbeat task context delivered to an adapter", () => { type: adapterType, supportsLocalAgentJwt: false, execute: async (input) => { + inspectPreparation.get(input.agent.id)?.(); captured.set(input.runId, input); return { exitCode: 0, signal: null, timedOut: false, sessionId: "context-session", resultJson: {} }; }, @@ -118,6 +121,101 @@ describePostgres("heartbeat task context delivered to an adapter", () => { }); }); + it("reads selected comments once for both wake projections", async () => { + const f = await seed(); + const commentId = randomUUID(); + await db.insert(issueComments).values({ + id: commentId, companyId: f.companyId, issueId: f.issueId, authorType: "user", authorUserId: "context-user", + body: "selected wake canary", + }); + const select = vi.spyOn(db, "select"); + inspectPreparation.set(f.agentId, () => { + const commentReads = select.mock.calls.filter(([projection]) => projection + && projection.body === issueComments.body && projection.id === issueComments.id && projection.deletedAt === issueComments.deletedAt); + expect(commentReads).toHaveLength(1); + }); + try { + const { context } = await execute(f, { wakeCommentId: commentId, wakeCommentIds: [commentId] }); + expect(context.paperclipWakeComment).toMatchObject({ id: commentId, body: "selected wake canary" }); + expect(context.paperclipWake).toMatchObject({ comments: [{ id: commentId, body: "selected wake canary" }] }); + } finally { + inspectPreparation.delete(f.agentId); + select.mockRestore(); + } + }); + + it("cancels deleted-only queued comments before adapter dispatch", async () => { + const f = await seed(), commentId = randomUUID(); + await db.insert(issueComments).values({ id: commentId, companyId: f.companyId, issueId: f.issueId, + authorType: "user", authorUserId: "context-user", body: "discarded wake", deletedAt: new Date() }); + const run = await heartbeat.wakeup(f.agentId, { source: "on_demand", triggerDetail: "manual", reason: "issue_commented", + payload: { issueId: f.issueId }, contextSnapshot: { issueId: f.issueId, wakeCommentId: commentId, wakeCommentIds: [commentId] } }); + expect(run).not.toBeNull(); + await heartbeat.drainActiveRunExecutions(); + const finished = await heartbeat.getRun(run!.id); + expect(finished).toMatchObject({ status: "cancelled", error: "Queued messages were discarded before dispatch" }); + expect(captured.has(run!.id)).toBe(false); + }); + + it("preserves deleted comment metadata without delivering its body", async () => { + const f = await seed(); + const commentId = randomUUID(); + await db.insert(issueComments).values({ id: commentId, companyId: f.companyId, issueId: f.issueId, + authorType: "user", authorUserId: "context-user", body: "deleted wake canary", deletedAt: new Date(), + presentation: { fixture: "deleted" }, metadata: { fixture: "deleted" } }); + const [run] = await db.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: f.agentId, + status: "running", issueId: f.issueId }).returning(); + const [agent] = await db.select().from(agents).where(eq(agents.id, f.agentId)); + const [issueRef] = await db.select().from(issues).where(eq(issues.id, f.issueId)); + const context: Record = { issueId: f.issueId, wakeCommentId: commentId, wakeCommentIds: [commentId] }; + await prepareHeartbeatTaskContext(db, { + run, agent, task: { issueRef, issueContext: null, context, taskSession: null }, + policy: { trustPreset: { kind: "standard" }, experimentalInstanceSettings: {} }, + services: { issuesSvc: { getAncestors: async () => [] }, getPinnedSkillTestContext: async () => null }, + }); + expect(context.paperclipWakeComment).toMatchObject({ id: commentId, body: "", presentation: null, metadata: null }); + expect(context.paperclipWake).toMatchObject({ comments: [{ id: commentId, body: "", presentation: null, metadata: null }] }); + expect(context.paperclipTaskMarkdown).not.toContain("deleted wake canary"); + }); + + it.each([false, true])("uses admitted skill-test scope when recovery is %s", async (recovering) => { + const f = await seed(); + const [run] = await db.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: f.agentId, + status: "running", issueId: f.issueId }).returning(); + const [agent] = await db.select().from(agents).where(eq(agents.id, f.agentId)); + const [issueRef] = await db.select().from(issues).where(eq(issues.id, f.issueId)); + const context: Record = { issueId: f.issueId, paperclipAgentJwtScope: { kind: "skill_test", issueId: f.issueId } }; + const pinned = vi.fn(async () => null); + const preparing = prepareHeartbeatTaskContext(db, { + run, agent, task: { issueRef, issueContext: null, context, taskSession: null }, + policy: { recovering, trustPreset: { kind: "standard" }, experimentalInstanceSettings: {} }, + services: { issuesSvc: { getAncestors: async () => [] }, getPinnedSkillTestContext: pinned }, + }); + if (recovering) await expect(preparing).rejects.toThrow("skill test's saved version is unavailable"); + else await preparing; + expect(pinned).toHaveBeenCalledTimes(recovering ? 1 : 0); + expect(context.paperclipAgentJwtScope).toEqual({ kind: recovering ? "skill_test" : "standard", ...(recovering ? { issueId: f.issueId } : {}) }); + }); + + it("retains the context issue scope when no issue reference is available", async () => { + const f = await seed(); + const otherIssueId = randomUUID(), commentId = randomUUID(); + await db.insert(issues).values({ id: otherIssueId, companyId: f.companyId, title: "Other issue", issueNumber: 2 }); + await db.insert(issueComments).values({ id: commentId, companyId: f.companyId, issueId: otherIssueId, + authorType: "user", authorUserId: "context-user", body: "other issue canary" }); + const [run] = await db.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: f.agentId, + status: "running", issueId: f.issueId }).returning(); + const [agent] = await db.select().from(agents).where(eq(agents.id, f.agentId)); + const context: Record = { issueId: f.issueId, wakeCommentId: commentId, wakeCommentIds: [commentId] }; + await prepareHeartbeatTaskContext(db, { + run, agent, task: { issueRef: null, issueContext: null, context, taskSession: null }, + policy: { trustPreset: { kind: "standard" }, experimentalInstanceSettings: {} }, + services: { issuesSvc: { getAncestors: async () => [] }, getPinnedSkillTestContext: async () => null }, + }); + expect(JSON.stringify(context.paperclipWake)).not.toContain("other issue canary"); + expect(context.paperclipWakeComment).toBeUndefined(); + }); + it("quarantines low-trust summaries and comments before preparing higher-trust prompts", async () => { const f = await seed(); const sourceTrust = buildLowTrustSourceTrust({ issueId: f.issueId }); diff --git a/server/src/services/heartbeat.ts b/server/src/services/heartbeat.ts index eb655fdef25..20b8ebe11c4 100644 --- a/server/src/services/heartbeat.ts +++ b/server/src/services/heartbeat.ts @@ -2007,8 +2007,11 @@ export function heartbeatService( const dispatchIssueId = readNonEmptyString(parseObject(run.contextSnapshot).issueId); const resumingAdmittedConversationTurn = !!runOptions.nativeLeaseOwner && typeof run.contextSnapshot?.conversationSessionGeneration === "number"; - if (dispatchIssueId && isConversation(await getIssueExecutionContext(run.companyId, dispatchIssueId)) - && !resumingAdmittedConversationTurn && !(await instanceSettings.getExperimental()).enableAgentChat) { + const conversationIdentity = dispatchIssueId && !resumingAdmittedConversationTurn + ? await db.select({ conversationAgentId: issues.conversationAgentId, conversationUserId: issues.conversationUserId }) + .from(issues).where(and(eq(issues.companyId, run.companyId), eq(issues.id, dispatchIssueId))).then(rows => rows[0] ?? null) + : null; + if (isConversation(conversationIdentity) && !(await instanceSettings.getExperimental()).enableAgentChat) { await setRunStatus(run.id, "cancelled", { finishedAt: new Date(), error: "Agent Chat is disabled", errorCode: "agent_chat_disabled" }); await setWakeupStatus(run.wakeupRequestId, "cancelled", { finishedAt: new Date() }); await releaseIssueExecutionAndPromote((await getRun(run.id))!, { suppressImmediateRecovery: true }); @@ -2204,44 +2207,6 @@ export function heartbeatService( context[PAPERCLIP_EXTERNAL_CHAT_EXECUTION_BOUND_KEY] = true; } const wakeCommentId = deriveCommentId(context, null); - const wakeCommentContext = - issueContext && wakeCommentId - ? await db - .select({ - id: issueComments.id, - body: issueComments.body, - authorType: issueComments.authorType, - authorAgentId: issueComments.authorAgentId, - authorUserId: issueComments.authorUserId, - presentation: issueComments.presentation, - metadata: issueComments.metadata, - deletedAt: issueComments.deletedAt, - deletedByType: issueComments.deletedByType, - deletedByAgentId: issueComments.deletedByAgentId, - deletedByUserId: issueComments.deletedByUserId, - deletedByRunId: issueComments.deletedByRunId, - sourceTrust: issueComments.sourceTrust, - }) - .from(issueComments) - .where( - and( - eq(issueComments.id, wakeCommentId), - eq(issueComments.issueId, issueContext.id), - eq(issueComments.companyId, agent.companyId), - ), - ) - .then((rows) => { - const row = rows[0] ?? null; - return row?.deletedAt - ? { - ...row, - body: "", - presentation: null, - metadata: null, - } - : row; - }) - : null; let issueAssigneeOverrides = issueContext && issueContext.assigneeAgentId === agent.id ? parseIssueAssigneeAdapterOverrides( @@ -2610,7 +2575,7 @@ export function heartbeatService( continuationSummary, executionContinuation, paperclipWakePayload, safeWakeComments, safeWakeCommentContext, pinnedSkillTest, } = await timePreparation("task_context", () => prepareHeartbeatTaskContext(db, { run, agent, - task: { issueRef, issueContext, wakeCommentContext, context, taskSession }, + task: { issueRef, issueContext, context, taskSession }, policy: { trustPreset, experimentalInstanceSettings, recovering: Boolean(runOptions.nativeRestartRecovery || runOptions.nativeLeaseOwner) }, services: { issuesSvc, getPinnedSkillTestContext }, })); diff --git a/server/src/services/heartbeat/execution-configuration.ts b/server/src/services/heartbeat/execution-configuration.ts index 9dd767a3811..fb9626b932f 100644 --- a/server/src/services/heartbeat/execution-configuration.ts +++ b/server/src/services/heartbeat/execution-configuration.ts @@ -139,28 +139,27 @@ export async function prepareHeartbeatExecutionConfiguration(db: Db, input: Hear stripWorkspaceRuntimeFromExecutionRunConfig(mergedConfig); const skillActor: AuthorizationActor = { type: "agent", agentId: agent.id, companyId: agent.companyId, runId: run.id, source: "agent_jwt", onBehalfOfUserId: responsibleUserId === "local-board" ? null : responsibleUserId }; - const runScopedMentionedSkillKeys = - preconfiguredHome || pinnedSkillTest ? [] : await time("configuration_mentions", () => resolveRunScopedMentionedSkillKeys({ + const [mentionedSkills, githubIdentity] = await Promise.allSettled([ + preconfiguredHome || pinnedSkillTest ? Promise.resolve([]) : time("configuration_mentions", () => resolveRunScopedMentionedSkillKeys({ db, companyId: agent.companyId, issueId, commentIds: extractWakeCommentIds(context), actor: skillActor, - })); + })), + time("configuration_github_identity", () => resolveManagedGitHubIdentitySelection(db, agent.companyId, { + agentId: agent.id, responsibleUserId, allowStandingDelegation: false, + })), + ]); + if (mentionedSkills.status === "rejected") throw mentionedSkills.reason; + if (githubIdentity.status === "rejected") throw githubIdentity.reason; + const runScopedMentionedSkillKeys = mentionedSkills.value; const runScopedSkillKeys = !pinnedSkillTest && acceptedPlanContinuationWake && !acceptedPlanWakeRoutingDecision?.suppressAcceptedContinuation ? [...runScopedMentionedSkillKeys, ACCEPTED_PLAN_CONVERSION_SKILL_KEY] : runScopedMentionedSkillKeys; - const githubSelection = await time("configuration_github_identity", () => resolveManagedGitHubIdentitySelection( - db, - agent.companyId, - { - agentId: agent.id, - responsibleUserId, - allowStandingDelegation: false, - }, - )); + const githubSelection = githubIdentity.value; const useHostGitHub = !githubSelection.configured && trustPreset.kind === "standard" && @@ -274,6 +273,7 @@ export async function prepareHeartbeatExecutionConfiguration(db: Db, input: Hear ); const nativeRunnerPreparationSpans: NativeRunHistoricalSpan[] = []; const skillsPrepareStartedAtMs = Date.now(); + let skillsPrepareEndedAtMs = skillsPrepareStartedAtMs; let skillDeliveryConfig = effectiveResolvedConfig; const prepareSkills = async () => { @@ -346,24 +346,35 @@ export async function prepareHeartbeatExecutionConfiguration(db: Db, input: Hear throw error; } }; - const runtimeSkillEntries = preconfiguredHome ? [] : await time("configuration_skills", prepareSkills); + const skillsPreparation = (preconfiguredHome ? Promise.resolve([]) : time("configuration_skills", prepareSkills)) + .finally(() => { skillsPrepareEndedAtMs = Date.now(); }); + const assignmentsPreparation = preconfiguredHome ? Promise.resolve([]) : time("configuration_connector_assignments", () => resolveConnectorAssignments(db, { companyId: agent.companyId, agentId: agent.id, runId: run.id, issueId: typeof context.issueId === "string" ? context.issueId : undefined })); + const [preparedSkills, preparedAssignments, preparedInstructions] = await Promise.allSettled([ + skillsPreparation, + assignmentsPreparation, + assignmentsPreparation.then(() => time("configuration_connection_instructions", () => resolveAssignedConnectionInstructionsForRun(db, { companyId: agent.companyId, agentId: agent.id, runId: run.id }))), + ]); + if (preparedSkills.status === "rejected") throw preparedSkills.reason; + if (preparedAssignments.status === "rejected") throw preparedAssignments.reason; + if (preparedInstructions.status === "rejected") throw preparedInstructions.reason; + const runtimeSkillEntries = preparedSkills.value; nativeRunnerPreparationSpans.push({ name: "skills.prepare", parentName: "task.prepare", startedAtMs: skillsPrepareStartedAtMs, - endedAtMs: Date.now(), + endedAtMs: skillsPrepareEndedAtMs, }); - const connectorAssignments = preconfiguredHome ? [] : await time("configuration_connector_assignments", () => resolveConnectorAssignments(db, { companyId: agent.companyId, agentId: agent.id, runId: run.id, issueId: typeof context.issueId === "string" ? context.issueId : undefined })); + const connectorAssignments = preparedAssignments.value; const connectorSkillConfig = await time("configuration_connector_skills", () => applyConnectorSkills(skillDeliveryConfig, runtimeSkillEntries, connectorAssignments)); // Both CLI adapters and native context materialization use the same resolved set. runtimeSkillEntries.splice(0, runtimeSkillEntries.length, ...connectorSkillConfig.paperclipRuntimeSkills); const connectorDelivery = await time("configuration_connector_delivery", () => prepareConnectorSkillDelivery(connectorSkillConfig, agent.adapterType)); // Always replace this runtime-only field; caller wake data cannot supply skills. context.paperclipWake = { ...parseObject(context.paperclipWake), connectorSkillInstructions: connectorDelivery.instructions }; - let runtimeConfig = await time("configuration_connection_instructions", () => prepareConnectionInstructionDelivery({ - resolve: () => resolveAssignedConnectionInstructionsForRun(db, { companyId: agent.companyId, agentId: agent.id, runId: run.id }), + let runtimeConfig = await prepareConnectionInstructionDelivery({ + resolve: async () => preparedInstructions.value, context, config: connectorDelivery.config, native: agent.adapterType === "paperclip_runner", - })); + }); const agentIdentity = supportsManagedAgentIdentity(agent.adapterType, agent.adapterConfig, run.runtimeMode === "native" ? run.driverKind : undefined) ? await time("configuration_agent_identity", () => agentIdentityService(db).ensureAgentIdentity(agent.companyId, agent.id)) : undefined; diff --git a/server/src/services/heartbeat/run-preparation.test.ts b/server/src/services/heartbeat/run-preparation.test.ts index 10a447e1a74..7d5a070d91d 100644 --- a/server/src/services/heartbeat/run-preparation.test.ts +++ b/server/src/services/heartbeat/run-preparation.test.ts @@ -14,6 +14,7 @@ import { ConfigurationIncompleteFailure, createHeartbeatRunPreparation, resolveExecutionRunAdapterConfig, + readHeartbeatWakeComments, } from "./run-preparation.js"; // Existing integration suites exercise the real database and dispatch path. @@ -38,6 +39,12 @@ function databaseWithResults(...results: unknown[][]) { return { db: { select } as unknown as Db, select, scopes }; } +function deferred() { + let resolve!: (value: T) => void; + const promise = new Promise(complete => { resolve = complete; }); + return { promise, resolve }; +} + const emptyRoutineContext = { routineId: null, env: null, responsibleUserId: null }; describe("heartbeat run preparation module", () => { @@ -120,6 +127,42 @@ describe("heartbeat run preparation module", () => { expect(database.select).toHaveBeenCalledTimes(2); }); + it("scopes one selected-comment snapshot for both wake consumers", async () => { + const database = databaseWithResults([{ id: "comment-one", body: "selected wake" }]); + expect(await readHeartbeatWakeComments(database.db, "company-one", "issue-one", ["comment-one"])) + .toEqual([{ id: "comment-one", body: "selected wake" }]); + expect(database.scopes[0]).toEqual(expect.arrayContaining(["company-one", "issue-one", "comment-one"])); + expect(await readHeartbeatWakeComments(database.db, "company-one", "issue-one", [])).toEqual([]); + expect(database.select).toHaveBeenCalledOnce(); + }); + + it("settles independent binding checks before surfacing a failure or resolving values", async () => { + const projectCheck = deferred<[]>(); + const firstFailure = new Error("agent binding check failed"); + const secretsSvc = { + resolveAdapterConfigForRuntime: vi.fn(), resolveEnvBindings: vi.fn(), + collectMissingRuntimeBindings: vi.fn(async (_company, _env, context) => { + if (context.consumerType === "agent") throw firstFailure; + return context.consumerType === "project" ? projectCheck.promise : []; + }), + collectMissingAdapterConfigRuntimeBindings: vi.fn(async () => []), + }; + let settled = false; + const opening = resolveExecutionRunAdapterConfig({ + companyId: "company-one", agentId: "agent-one", projectId: "project-one", + environmentId: "environment-one", environmentEnv: { SHARED: "environment" }, + executionRunConfig: { env: { SHARED: "agent" } }, projectEnv: { SHARED: "project" }, secretsSvc, + }); + void opening.then(() => { settled = true; }, () => { settled = true; }); + await vi.waitFor(() => expect(secretsSvc.collectMissingRuntimeBindings).toHaveBeenCalledTimes(3)); + expect(secretsSvc.collectMissingAdapterConfigRuntimeBindings).toHaveBeenCalledOnce(); + expect(settled).toBe(false); + expect(secretsSvc.resolveAdapterConfigForRuntime).not.toHaveBeenCalled(); + expect(secretsSvc.resolveEnvBindings).not.toHaveBeenCalled(); + projectCheck.resolve([]); + await expect(opening).rejects.toBe(firstFailure); + }); + it("keeps missing scoped credentials as a pre-dispatch failure", async () => { const secretsSvc = { resolveAdapterConfigForRuntime: vi.fn(), diff --git a/server/src/services/heartbeat/run-preparation.ts b/server/src/services/heartbeat/run-preparation.ts index bf288599a51..bfbe6c862c1 100644 --- a/server/src/services/heartbeat/run-preparation.ts +++ b/server/src/services/heartbeat/run-preparation.ts @@ -394,76 +394,31 @@ export async function resolveExecutionRunAdapterConfig(input: { // no binding before resolving any secret value. Missing bindings short-circuit // to a configuration-incomplete blocker routed to a human owner instead of a // dispatched-then-failed run (which previously surfaced as opaque setup_failed). - if (typeof input.secretsSvc.collectMissingRuntimeBindings === "function") { - const missingBindings: MissingRuntimeBinding[] = []; - if (environmentEnv && input.environmentId) { - missingBindings.push( - ...(await input.secretsSvc.collectMissingRuntimeBindings( - input.companyId, - environmentEnv, - { - consumerType: "environment", - consumerId: input.environmentId, - responsibleUserId: input.responsibleUserId ?? null, - }, - )), - ); - } + const collectMissing = input.secretsSvc.collectMissingRuntimeBindings; + if (typeof collectMissing === "function") { + const checks: Array<() => Promise> = []; + const checkEnv = (env: unknown, consumerType: "environment" | "agent" | "project" | "routine", consumerId: string) => { + checks.push(() => collectMissing(input.companyId, env, { + consumerType, consumerId, responsibleUserId: input.responsibleUserId ?? null, + })); + }; + if (environmentEnv && input.environmentId) checkEnv(environmentEnv, "environment", input.environmentId); if (input.agentId) { - missingBindings.push( - ...(await input.secretsSvc.collectMissingRuntimeBindings( - input.companyId, - parseObject(executionRunConfig.env), - { - consumerType: "agent", - consumerId: input.agentId, - responsibleUserId: input.responsibleUserId ?? null, - }, - )), - ); - if ( - typeof input.secretsSvc.collectMissingAdapterConfigRuntimeBindings === - "function" - ) { - missingBindings.push( - ...(await input.secretsSvc.collectMissingAdapterConfigRuntimeBindings( - input.companyId, - executionRunConfig, - input.adapterType ?? null, - { - consumerType: "agent", - consumerId: input.agentId, - responsibleUserId: input.responsibleUserId ?? null, - }, - )), - ); + checkEnv(parseObject(executionRunConfig.env), "agent", input.agentId); + const collectAdapterMissing = input.secretsSvc.collectMissingAdapterConfigRuntimeBindings; + if (typeof collectAdapterMissing === "function") { + checks.push(() => collectAdapterMissing(input.companyId, executionRunConfig, input.adapterType ?? null, { + consumerType: "agent", consumerId: input.agentId!, responsibleUserId: input.responsibleUserId ?? null, + })); } } - if (projectEnv && input.projectId) { - missingBindings.push( - ...(await input.secretsSvc.collectMissingRuntimeBindings( - input.companyId, - projectEnv, - { - consumerType: "project", - consumerId: input.projectId, - responsibleUserId: input.responsibleUserId ?? null, - }, - )), - ); - } - if (routineEnv && input.routineId) { - missingBindings.push( - ...(await input.secretsSvc.collectMissingRuntimeBindings( - input.companyId, - routineEnv, - { - consumerType: "routine", - consumerId: input.routineId, - responsibleUserId: input.responsibleUserId ?? null, - }, - )), - ); + if (projectEnv && input.projectId) checkEnv(projectEnv, "project", input.projectId); + if (routineEnv && input.routineId) checkEnv(routineEnv, "routine", input.routineId); + const results = await Promise.allSettled(checks.map(check => Promise.resolve().then(check))); + const missingBindings: MissingRuntimeBinding[] = []; + for (const result of results) { + if (result.status === "rejected") throw result.reason; + missingBindings.push(...result.value); } if (requiredScopedEnvBinding) { const requiredEnvKeys = new Set(requiredScopedEnvBinding.keys); @@ -2244,12 +2199,45 @@ export async function attestReviewedExternalChatRun(input: { throw new Error("reviewed_chat_execution_binding_not_ready"); } +export async function readHeartbeatWakeComments(db: Db, companyId: string, issueId: string | null, commentIds: string[]) { + if (commentIds.length === 0) return []; + return db + .select({ + id: issueComments.id, + companyId: issueComments.companyId, + issueId: issueComments.issueId, + body: issueComments.body, + origin: issueComments.origin, + authorType: issueComments.authorType, + authorAgentId: issueComments.authorAgentId, + authorUserId: issueComments.authorUserId, + presentation: issueComments.presentation, + metadata: issueComments.metadata, + deletedAt: issueComments.deletedAt, + deletedByType: issueComments.deletedByType, + deletedByAgentId: issueComments.deletedByAgentId, + deletedByUserId: issueComments.deletedByUserId, + deletedByRunId: issueComments.deletedByRunId, + sourceTrust: issueComments.sourceTrust, + createdAt: issueComments.createdAt, + }) + .from(issueComments) + .where( + and( + eq(issueComments.companyId, companyId), + issueId ? eq(issueComments.issueId, issueId) : undefined, + inArray(issueComments.id, commentIds), + ), + ); +} + export async function buildPaperclipWakePayload(input: { db: Db; companyId: string; agentId?: string | null; runId?: string | null; contextSnapshot: Record; + wakeCommentRows?: Awaited>; continuationSummary?: { key: string; title: string | null; @@ -2314,36 +2302,9 @@ export async function buildPaperclipWakePayload(input: { ) return null; - const commentRows = - commentIds.length === 0 - ? [] - : await input.db - .select({ - id: issueComments.id, - issueId: issueComments.issueId, - body: issueComments.body, - origin: issueComments.origin, - authorType: issueComments.authorType, - authorAgentId: issueComments.authorAgentId, - authorUserId: issueComments.authorUserId, - presentation: issueComments.presentation, - metadata: issueComments.metadata, - deletedAt: issueComments.deletedAt, - deletedByType: issueComments.deletedByType, - deletedByAgentId: issueComments.deletedByAgentId, - deletedByUserId: issueComments.deletedByUserId, - deletedByRunId: issueComments.deletedByRunId, - sourceTrust: issueComments.sourceTrust, - createdAt: issueComments.createdAt, - }) - .from(issueComments) - .where( - and( - eq(issueComments.companyId, input.companyId), - issueId ? eq(issueComments.issueId, issueId) : undefined, - inArray(issueComments.id, commentIds), - ), - ); + const commentRows = input.wakeCommentRows + ? input.wakeCommentRows.filter(row => row.companyId === input.companyId && (!issueId || row.issueId === issueId) && commentIds.includes(row.id)) + : await readHeartbeatWakeComments(input.db, input.companyId, issueId, commentIds); const commentsById = new Map( commentRows.map((comment) => [comment.id, comment]), diff --git a/server/src/services/heartbeat/task-context.ts b/server/src/services/heartbeat/task-context.ts index 85bdd65ca61..9c1fb122dd4 100644 --- a/server/src/services/heartbeat/task-context.ts +++ b/server/src/services/heartbeat/task-context.ts @@ -1,5 +1,5 @@ import { createHash } from "node:crypto"; -import type { Db, agents, agentTaskSessions, heartbeatRuns, issueComments } from "@paperclipai/db"; +import type { Db, agents, agentTaskSessions, heartbeatRuns } from "@paperclipai/db"; import { FAST_RESPONSE_AGENT_GUIDANCE, fastResponseHistoryBody } from "@paperclipai/shared"; import type { PaperclipTurnContext } from "@paperclipai/adapter-utils/server-utils"; import { parseObject } from "../../adapters/utils.js"; @@ -15,8 +15,9 @@ import { getTaskPlanContext } from "../task-plan-context.js"; import { caseExecutionMarkdown, getCaseExecutionContext } from "../case-context.js"; import type { TrustPresetResolution } from "../trust-preset-resolver.js"; import { liveVoiceExecutionGuidance } from "../voice/voice-execution-guidance.js"; -import { buildPaperclipWakePayload, ConfigurationIncompleteFailure, PAPERCLIP_WAKE_PAYLOAD_KEY, type createHeartbeatRunPreparation } from "./run-preparation.js"; +import { buildPaperclipWakePayload, ConfigurationIncompleteFailure, readHeartbeatWakeComments, PAPERCLIP_WAKE_PAYLOAD_KEY, type createHeartbeatRunPreparation } from "./run-preparation.js"; import { buildPaperclipTaskMarkdown } from "./task-markdown.js"; +import { deriveCommentId, extractWakeCommentIds } from "../../modules/run-dispatch/index.js"; import { workspacePreferenceService } from "../workspace-preferences.js"; type RunPreparation = ReturnType; @@ -30,8 +31,6 @@ export interface HeartbeatTaskContextInput { "id" | "identifier" | "title" | "status" | "priority" | "workMode" | "conversationAgentId" | "description" | "projectId"> | null; issueContext: IssueContext | null; - wakeCommentContext: Pick | null; context: Record; taskSession: Pick | null; }; @@ -53,35 +52,9 @@ function readNonEmptyString(value: unknown): string | null { /** Rebuilds task prompt fields on the run's context; acquires no execution resources. */ export async function prepareHeartbeatTaskContext(db: Db, input: HeartbeatTaskContextInput) { const { run, agent } = input; - const { issueRef, issueContext, wakeCommentContext, context, taskSession } = input.task; + const { issueRef, issueContext, context, taskSession } = input.task; const { trustPreset, experimentalInstanceSettings } = input.policy; const { issuesSvc, getPinnedSkillTestContext } = input.services; - const continuationSummary = issueRef && !isConversation(issueContext) - ? await getIssueContinuationSummaryDocument(db, issueRef.id) - : null; - const exposeLowTrustRaw = trustPreset.kind === "low_trust_review"; - const safeContinuationSummary = - continuationSummary && !exposeLowTrustRaw - ? redactQuarantinedBodyForHigherTrust(continuationSummary) - : continuationSummary; - const safeWakeCommentContext = - wakeCommentContext && !exposeLowTrustRaw - ? sanitizeQuarantinedCommentForHigherTrust(wakeCommentContext) - : wakeCommentContext; - const issueAncestors = issueRef - ? await issuesSvc.getAncestors(issueRef.id) - : []; - if (continuationSummary) { - context.paperclipContinuationSummary = { - key: safeContinuationSummary!.key, - title: safeContinuationSummary!.title, - body: safeContinuationSummary!.body, - sourceTrust: safeContinuationSummary!.sourceTrust ?? null, - updatedAt: safeContinuationSummary!.updatedAt.toISOString(), - }; - } else { - delete context.paperclipContinuationSummary; - } const admittedScope = parseObject(context.paperclipAgentJwtScope); if (!input.policy.recovering || (admittedScope.kind !== "standard" && admittedScope.kind !== "skill_test")) { const savedIssue = parseObject(context.paperclipIssue); @@ -93,10 +66,58 @@ export async function prepareHeartbeatTaskContext(db: Db, input: HeartbeatTaskCo : { kind: "standard" }; } const isSkillTest = parseObject(context.paperclipAgentJwtScope).kind === "skill_test"; - const pinnedSkillTestContext = - isSkillTest && issueRef - ? await getPinnedSkillTestContext(agent.companyId, issueRef.id) - : null; + const wakeCommentId = deriveCommentId(context, null); + const commentIds = [...new Set([...extractWakeCommentIds(context), ...(wakeCommentId ? [wakeCommentId] : [])])]; + const [summary, ancestors, pinnedSkillTestRead, wakeComments] = await Promise.allSettled([ + issueRef && !isConversation(issueContext) ? getIssueContinuationSummaryDocument(db, issueRef.id) : Promise.resolve(null), + issueRef ? issuesSvc.getAncestors(issueRef.id) : Promise.resolve([]), + isSkillTest && issueRef ? getPinnedSkillTestContext(agent.companyId, issueRef.id) : Promise.resolve(null), + readHeartbeatWakeComments(db, agent.companyId, readNonEmptyString(context.issueId), commentIds), + ]); + if (summary.status === "rejected") throw summary.reason; + if (ancestors.status === "rejected") throw ancestors.reason; + if (pinnedSkillTestRead.status === "rejected") throw pinnedSkillTestRead.reason; + if (wakeComments.status === "rejected") throw wakeComments.reason; + const wakeCommentRows = wakeComments.value; + const selectedWakeComment = issueRef && wakeCommentId ? wakeCommentRows.find(comment => comment.id === wakeCommentId && comment.issueId === issueRef.id) ?? null : null; + const wakeCommentContext = selectedWakeComment ? { + id: selectedWakeComment.id, + body: selectedWakeComment.deletedAt ? "" : selectedWakeComment.body, + authorType: selectedWakeComment.authorType, + authorAgentId: selectedWakeComment.authorAgentId, + authorUserId: selectedWakeComment.authorUserId, + presentation: selectedWakeComment.deletedAt ? null : selectedWakeComment.presentation, + metadata: selectedWakeComment.deletedAt ? null : selectedWakeComment.metadata, + deletedAt: selectedWakeComment.deletedAt, + deletedByType: selectedWakeComment.deletedByType, + deletedByAgentId: selectedWakeComment.deletedByAgentId, + deletedByUserId: selectedWakeComment.deletedByUserId, + deletedByRunId: selectedWakeComment.deletedByRunId, + sourceTrust: selectedWakeComment.sourceTrust, + } : null; + const continuationSummary = summary.value; + const issueAncestors = ancestors.value; + const pinnedSkillTestContext = pinnedSkillTestRead.value; + const exposeLowTrustRaw = trustPreset.kind === "low_trust_review"; + const safeContinuationSummary = + continuationSummary && !exposeLowTrustRaw + ? redactQuarantinedBodyForHigherTrust(continuationSummary) + : continuationSummary; + const safeWakeCommentContext = + wakeCommentContext && !exposeLowTrustRaw + ? sanitizeQuarantinedCommentForHigherTrust(wakeCommentContext) + : wakeCommentContext; + if (continuationSummary) { + context.paperclipContinuationSummary = { + key: safeContinuationSummary!.key, + title: safeContinuationSummary!.title, + body: safeContinuationSummary!.body, + sourceTrust: safeContinuationSummary!.sourceTrust ?? null, + updatedAt: safeContinuationSummary!.updatedAt.toISOString(), + }; + } else { + delete context.paperclipContinuationSummary; + } if (isSkillTest && !pinnedSkillTestContext) { throw new ConfigurationIncompleteFailure("The skill test's saved version is unavailable.", { configurationIncomplete: { reason: "skill_unavailable", actionUrl: "/skills" }, @@ -132,6 +153,7 @@ export async function prepareHeartbeatTaskContext(db: Db, input: HeartbeatTaskCo agentId: agent.id, runId: run.id, contextSnapshot: context, + wakeCommentRows, continuationSummary, issueSummary: issueRef ? { diff --git a/server/src/services/native-runtime/runner-api.integration.test.ts b/server/src/services/native-runtime/runner-api.integration.test.ts index 4e1831b734a..fdb5b7f70a2 100644 --- a/server/src/services/native-runtime/runner-api.integration.test.ts +++ b/server/src/services/native-runtime/runner-api.integration.test.ts @@ -497,7 +497,7 @@ else if(m.id!==undefined) send({id:m.id,result:{}}); const [agent] = await server.db.select().from(agents).where(eq(agents.id, fixture.agentId)); const context = structuredClone(run.contextSnapshot ?? {}); await prepareHeartbeatTaskContext(server.db, { run, agent, - task: { issueRef: issue, issueContext: null, wakeCommentContext: null, context, taskSession: null }, + task: { issueRef: issue, issueContext: null, context, taskSession: null }, policy: { recovering: true, trustPreset: { kind: "standard" }, experimentalInstanceSettings: {} }, services: { issuesSvc: { getAncestors: async () => [] }, getPinnedSkillTestContext: createHeartbeatRunPreparation(server.db).getPinnedSkillTestContext }, From cd4caaab266ecd857563be32e7aec9974fc26e4a Mon Sep 17 00:00:00 2001 From: Dotta Date: Fri, 30 Oct 2026 18:49:20 -0500 Subject: [PATCH 56/74] fix: batch status-card source notifications Coalesce committed company hints in a fixed one-second window without flooring real deadlines. Verify one hundred ordinary issue/comment writes, company isolation, deadline preemption, retained hints and settled SQL quietness. Await actual evaluation I/O in fake-clock acceptance instead of jumping across connection timers. Co-Authored-By: Paperclip --- doc/plans/2026-10-30-status-cards-sleep.md | 12 ++ .../status-card-evaluation-work.test.ts | 130 +++++++++++++++++- server/src/services/status-card-evaluation.ts | 4 + 3 files changed, 139 insertions(+), 7 deletions(-) diff --git a/doc/plans/2026-10-30-status-cards-sleep.md b/doc/plans/2026-10-30-status-cards-sleep.md index dbf229cdace..5b98962698b 100644 --- a/doc/plans/2026-10-30-status-cards-sleep.md +++ b/doc/plans/2026-10-30-status-cards-sleep.md @@ -123,3 +123,15 @@ five minutes keeps its remaining time without extension. This bounds migration compatibility delay while preserving prior crash/failure recovery. After the first successful evaluation all newly saved deadlines have explicit ownership. No recurring compatibility scan is added. + +Source notifications use the shared deadline helper's fixed one-second batching +window. The first hint after idle can reconcile immediately. Later hints merge +by company until that window ends. The window does not slide with each new +write, so a sustained source stream cannot postpone reconciliation forever. +One second keeps reactive cards prompt (the removed heartbeat default was 30s) +and bounds burst SQL. No minimum spacing is set: an already-known policy, +expiry, or retry deadline can preempt batching. The window alone creates no +scheduled work after settlement. Real issue/comment producer acceptance measures +due/next queries and actual searches under 100 writes, preserves both companies' +hints, dispatches another company's saved deadline inside the batching window, +and checks SQL quietness after authorized results. diff --git a/server/src/__tests__/status-card-evaluation-work.test.ts b/server/src/__tests__/status-card-evaluation-work.test.ts index 4e00e326855..6bb466dea89 100644 --- a/server/src/__tests__/status-card-evaluation-work.test.ts +++ b/server/src/__tests__/status-card-evaluation-work.test.ts @@ -1,5 +1,7 @@ import { budgetService } from "../services/budgets.js"; import { documentService } from "../services/documents.js"; +import * as companySearch from "../services/company-search.js"; +import { setImmediate as immediate } from "node:timers/promises"; import { randomUUID } from "node:crypto"; import { eq, sql } from "drizzle-orm"; import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; @@ -24,6 +26,7 @@ const support = external ? { supported: true } : await getEmbeddedPostgresTestSu let draining = false; let companyId: string, agentId: string; const errors: unknown[] = []; + const evaluations = new Set>(); let compiledQuery = companySearchQuerySchema.parse({ scope: "issues", status: ["blocked"] }); const actor = { agentId: null, userId: "board-user" }; const policy = (mode: "manual" | "reactive" | "interval" = "reactive") => statusCardRefreshPolicySchema.parse({ @@ -60,16 +63,29 @@ const support = external ? { supported: true } : await getEmbeddedPostgresTestSu await db.delete(instanceSettings); await db.delete(agents); await db.delete(companyMemberships); await db.delete(companies); }); afterAll(async () => { await db?.$client.end({ timeout: 1 }); await temporary?.cleanup(); }); - async function start() { + async function start(service = statusCardEvaluationService(db, { wakeup })) { coordinator = createDeliveryWorkCoordinator({ owner: db, canRun: () => !draining, canReconcile: () => true, onError: error => errors.push(error) }); - await registerStatusCardEvaluationWork(coordinator, statusCardEvaluationService(db, { wakeup }), () => !draining).ready; + await registerStatusCardEvaluationWork(coordinator, { ...service, run: signal => { + const work = service.run(signal); + evaluations.add(work); + void work.then(() => evaluations.delete(work), () => evaluations.delete(work)); + return work; + } }, () => !draining).ready; } async function settled() { - await vi.waitFor(async () => { - await vi.advanceTimersByTimeAsync(30_000); - expect(idleWorkSnapshot().active).toBe(0); - }, { timeout: 5000 }); + // Do not leap across PostgreSQL connect timers while real I/O is pending. + // Await actual domain work, then require several settled event-loop turns + // so a transient zero hold cannot stand in for transaction reconciliation. + for (let turn = 0; turn < 3; turn++) { + await vi.waitFor(async () => { + await Promise.allSettled([...evaluations]); + await vi.advanceTimersByTimeAsync(500); + await Promise.allSettled([...evaluations]); + expect(idleWorkSnapshot().active).toBe(0); + }, { timeout: 5000 }); + await immediate(); + } } async function quiet() { await settled(); @@ -128,11 +144,111 @@ const support = external ? { supported: true } : await getEmbeddedPostgresTestSu expect(errors).toEqual([]); }); + it("coalesces one hundred committed source writes without delaying another company's saved deadline", async () => { + const reactive = await card(); + const originalCompany = companyId, originalAgent = agentId; + companyId = randomUUID(); agentId = randomUUID(); + await db.insert(companies).values({ id: companyId, name: "Other", issuePrefix: `O${companyId.slice(0, 6)}` }); + await db.insert(agents).values({ id: agentId, companyId, name: "Other summarizer", role: "general", status: "idle", adapterType: "process", adapterConfig: {} }); + const other = await card("interval"); + const otherIssue = await issueService(db).create(companyId, { title: "Other interval blocker", status: "blocked", priority: "medium" }); + const otherReactive = await card(); + companyId = originalCompany; agentId = originalAgent; + const service = statusCardEvaluationService(db, { wakeup }); + const originalRun = service.run; + let running: Promise | undefined; + const run = vi.spyOn(service, "run").mockImplementation(signal => { + const result = originalRun(signal); running = result; return result; + }); + await start(service); await settled(); + await vi.advanceTimersByTimeAsync(1000); + // Give each committed producer and any actual domain work time to settle. + // This prevents incidental overlap from hiding per-write reconstruction. + const flush = async () => { + for (let i = 0; i < 10; i++) { + await running; await immediate(); + } + }; + await flush(); + const select = vi.spyOn(db, "select"); + const factory = companySearch.companySearchService; + const searchedCompanies: string[] = []; + vi.spyOn(companySearch, "companySearchService").mockImplementation(owner => { + const search = factory(owner); + const actual = search.search; + search.search = async (...args) => { searchedCompanies.push(args[0]); return actual(...args); }; + return search; + }); + run.mockClear(); + const burstStarted = Date.now(); + for (let i = 0; i < 50; i++) { + const issue = await issueService(db).create(companyId, { title: `Burst blocker ${i}`, status: "blocked", priority: "medium" }); + await flush(); + await issueService(db).addComment(issue.id, `Human update ${i}`, { userId: "board-user" }); + await flush(); + } + await flush(); + // One eager reconciliation after idle and one bounded batched continuation; + // the producers themselves are not counted as background SQL. + const dueReads = () => select.mock.calls.filter(([fields]) => fields && "card" in fields && fields.card === statusCards).length; + const nextReads = () => select.mock.calls.filter(([fields]) => fields && "at" in fields && fields.at === statusCards.nextEvalAt).length; + const elapsedMs = Date.now() - burstStarted; + const maximum = Math.ceil(elapsedMs / 1000) + 2; + console.info("status-card burst", { elapsedMs, maximum, runs: run.mock.calls.length, due: dueReads(), next: nextReads(), searches: searchedCompanies.length }); + expect(run.mock.calls.length).toBeLessThanOrEqual(maximum); + expect(dueReads()).toBeLessThanOrEqual(maximum); expect(nextReads()).toBeLessThanOrEqual(maximum); + expect(searchedCompanies.length).toBeLessThanOrEqual(maximum); + expect(searchedCompanies.length).toBeGreaterThan(0); + expect(new Set(searchedCompanies)).toEqual(new Set([originalCompany])); + expect(wakeup).not.toHaveBeenCalled(); + // Re-register with an existing durable deadline inside the coalescing + // window. A company hint must not make that real deadline wait one second. + await coordinator!.stop(); coordinator = undefined; + const dueAt = Date.now() + 500; + await db.update(statusCards).set({ nextEvalAt: new Date(dueAt), nextEvalReason: "refresh" }).where(eq(statusCards.id, other.id)); + await start(service); await flush(); + await issueService(db).addComment(otherIssue.id, "Another company's coalesced human update", { userId: "board-user" }); + await issueService(db).addComment((await db.select({ id: issues.id }).from(issues).where(eq(issues.companyId, originalCompany)))[0]!.id, + "More changes while another company is due", { userId: "board-user" }); + await flush(); + await vi.advanceTimersByTimeAsync(500); await flush(); + await vi.waitFor(() => expect(wakeup).toHaveBeenCalledOnce()); + expect(Date.now()).toBeLessThan(dueAt + 500); + expect(wakeup.mock.calls[0]![0]).not.toBe(originalAgent); + const generated = (await statusCardService(db).getById(other.id))!; + expect(generated.generatingIssueId).toBeTruthy(); + expect((await statusCardService(db).getById(reactive.id))!.pendingChangeCount).toBeGreaterThan(0); + expect((await statusCardService(db).getById(otherReactive.id))!.pendingChangeCount).toBeGreaterThan(0); + expect(searchedCompanies).toContain(other.companyId); + // Complete all ordinary results, then prove there is no recurring batch + // timer once the requested work has settled. + companyId = other.companyId; agentId = other.agentId!; + await writeback(other.id, generated.generatingIssueId!); + await statusCardService(db).update((await statusCardService(db).getById(other.id))!, { archived: true }, actor); + companyId = originalCompany; agentId = originalAgent; + await vi.advanceTimersByTimeAsync(120_000); await settled(); await flush(); + await vi.waitFor(async () => expect((await statusCardService(db).getById(reactive.id))!.generatingIssueId).toBeTruthy()); + const pending = (await statusCardService(db).getById(reactive.id))!; + expect(pending.generatingIssueId).toBeTruthy(); + await writeback(reactive.id, pending.generatingIssueId!); + const otherPending = (await statusCardService(db).getById(otherReactive.id))!; + expect(otherPending.generatingIssueId).toBeTruthy(); + companyId = other.companyId; agentId = other.agentId!; + await writeback(otherReactive.id, otherPending.generatingIssueId!); + companyId = originalCompany; agentId = originalAgent; + select.mockRestore(); + await quiet(); expect(errors).toEqual([]); + }, 30_000); + it("cancels archived or disabled future work and replaces an earlier interval deadline", async () => { await start(); let created = await card("interval"); await settled(); const old = coordinator!.nextWakeAt(); created = await statusCardService(db).update(created, { refreshPolicy: { ...policy("interval"), intervalMinutes: 1 } }, actor); - await settled(); expect(coordinator!.nextWakeAt()).toBeLessThan(old!); + await vi.waitFor(async () => { + await vi.advanceTimersByTimeAsync(100); + await Promise.allSettled([...evaluations]); + expect(coordinator!.nextWakeAt()).toBeLessThan(old!); + }); await statusCardService(db).update(created, { archived: true }, actor); await quiet(); expect(wakeup).not.toHaveBeenCalled(); expect(errors).toEqual([]); }); diff --git a/server/src/services/status-card-evaluation.ts b/server/src/services/status-card-evaluation.ts index 3e33c0d0885..e6086535fc9 100644 --- a/server/src/services/status-card-evaluation.ts +++ b/server/src/services/status-card-evaluation.ts @@ -90,5 +90,9 @@ export function registerStatusCardEvaluationWork( ) { return registerDeadlineWork(coordinator, DELIVERY_QUEUES.statusCardEvaluation, { run: service.run, next: service.next, onSettled: service.changed, canRun, retryMs: 30_000, + // Batch source hints for at most one second after reconciliation. Do not + // use minimumSpacingMs: known policy/retry deadlines must preempt batching. + // This window alone arms no timer when the feature has no pending work. + notificationCoalesceMs: 1_000, }); } From 105a67b52c66a738c0a8eb7807332b99931ac45d Mon Sep 17 00:00:00 2001 From: Devin Foley Date: Fri, 30 Oct 2026 16:54:04 -0700 Subject: [PATCH 57/74] Reuse signed master images for npm canaries (#18404) ## Thinking Path > - Paperclip is the open source app people use to manage AI agents for work. > - Release workflows publish npm packages and multi-platform images. > - A master push can finish its image before the npm canary dispatch starts. > - The dispatch currently builds that same source again on two runners. > - This PR verifies and reuses the signed master image by digest. > - Missing images retain the native build path. Channel promotion still follows npm. ## Linked Issues or Issue Description **What existing behavior does this improve?** The Docker workflow dispatched after npm canary publication. **Current behavior** Every canary dispatch builds both architectures, even when master has already published the same source. This consumes runner capacity and delays the image channel update. **Proposed behavior** Resolve the full-SHA master image. Verify its platform index, digest and signed source identity. Retag that immutable digest. Build normally when the image does not exist. **Reason and benefit** Remove duplicate image builds while keeping missing-image recovery and the current npm channel check. A recent duplicate used about 25 runner-minutes. The saving depends on whether the master image exists. ## What Changed - Add a canary-only reuse job before the native matrix. - Require signed provenance from the canonical Docker workflow on master at the exact source and signer commit. Read the proof from GHCR. - Reuse the validated digest or build both platforms when the full-SHA image is absent. Stop on registry and verification errors. - Keep the serialized npm dist-tag promotion after either path. Keep other build lanes independent. - Add failure tests, register them in CI and update release documentation. ## Verification - Passed 603 tests: `node --test '.github/scripts/tests/*.test.mjs' scripts/standard-image-contract.test.mjs scripts/reuse-canary-image.test.mjs scripts/__tests__/release-verify-workflow.test.mjs scripts/cloud-source-verification.test.mjs`. - Passed `actionlint -shellcheck='' .github/workflows/docker.yml .github/workflows/pr-trusted.yml` and `git diff --check`. - Full actionlint reports the same six existing shellcheck findings as the unchanged master workflow. No new findings. - A published master image passed the exact provenance command. This was a read-only check. No images were retagged. - Gitleaks and manual diff inspection found no secrets or private references. - Attempted `pnpm -r typecheck`, `pnpm test:run` and `pnpm build`. They cannot start because this isolated checkout has no installed dependencies (`tsc` and `tsx` are absent). Repository CI passed on `3c46f0152d4db94a7f393bc79f466aa6c2c745e2` after the documented browser-test rerun. All checks are complete. The original failure and prior baseline occurrence remain recorded in the PR comments; its root cause remains unknown. ## Risks - The image can appear just before its attestation. Verification then fails closed. Rerun after the master producer completes. - Reuse preserves the first build's resolved dependencies and tool versions. It intentionally does not rebuild the same source to refresh floating inputs. - A skipped or unfinished master image still needs a full build. No speedup is claimed for that path. - End-to-end reuse and runner savings require observation after merge. No live publication was performed. ## Model Used OpenAI `gpt-6.1-sol`, through the Codex harness in T3 Code, with xhigh reasoning effort and shell/code tools. The context window size is not exposed by this runtime. ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above - [x] I have either (a) linked existing issues with `Fixes: #` / `Closes #` / `Refs #` OR (b) described the issue in-PR following the relevant issue template - [x] I have not referenced internal/instance-local Paperclip issues or links (only public GitHub `#NNN` / `github.com/paperclipai/paperclip` URLs) - [x] My branch name describes the change (e.g. `docs/...`, `fix/...`) and contains no internal Paperclip ticket id or instance-derived details - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [x] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [x] All Paperclip CI gates are green - [x] Greptile is 5/5 with no open P2s, recommendations, or follow-ups - [x] I will address all Greptile and reviewer comments before requesting merge Co-authored-by: Paperclip --- .../tests/docker-canary-promotion.test.mjs | 2 +- .github/workflows/docker.yml | 39 ++++++++-- .github/workflows/pr-trusted.yml | 2 +- doc/RELEASING.md | 23 +++++- scripts/reuse-canary-image.mjs | 34 +++++++++ scripts/reuse-canary-image.test.mjs | 75 +++++++++++++++++++ scripts/standard-image-contract.mjs | 6 +- scripts/standard-image-contract.test.mjs | 14 ++++ 8 files changed, 183 insertions(+), 12 deletions(-) create mode 100644 scripts/reuse-canary-image.mjs create mode 100644 scripts/reuse-canary-image.test.mjs diff --git a/.github/scripts/tests/docker-canary-promotion.test.mjs b/.github/scripts/tests/docker-canary-promotion.test.mjs index a9b0727465f..fe80e25168b 100644 --- a/.github/scripts/tests/docker-canary-promotion.test.mjs +++ b/.github/scripts/tests/docker-canary-promotion.test.mjs @@ -11,7 +11,7 @@ const script = job.split(" run: |\n")[1].split("\n").map(line => line.rep const sha = "a".repeat(40); test("canary promotion waits for the standard manifest and keeps its serialized channel", () => { - assert.match(job, /needs: \[merge-and-push\]/); + assert.match(job, /needs: \[reuse_canary_image, merge-and-push\]/); assert.match(job, /group: docker-canary-channel-promotion/); assert.match(job, /cancel-in-progress: false/); }); diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index ed52dfa700d..ec7c71000e5 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -33,6 +33,35 @@ concurrency: cancel-in-progress: false jobs: + reuse_canary_image: + if: github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && startsWith(github.ref, 'refs/tags/canary/v') + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + contents: read + packages: write + attestations: read + outputs: + reused: ${{ steps.reuse.outputs.reused }} + steps: + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + - name: Setup Node.js + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 + with: + node-version: 24 + - name: Login to GitHub Container Registry + uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f + with: + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Reuse the verified master image when present + id: reuse + env: + GH_TOKEN: ${{ github.token }} + run: node scripts/reuse-canary-image.mjs + # Multi-arch by native runner, not QEMU. # # This was one job building linux/amd64,linux/arm64 together on an x86 @@ -56,6 +85,8 @@ jobs: # digest; `merge` assembles the manifest list. Both architectures remain # available to self-hosted installations and downstream image composers. build-and-push: + needs: [reuse_canary_image] + if: ${{ !cancelled() && (needs.reuse_canary_image.result == 'skipped' || (needs.reuse_canary_image.result == 'success' && needs.reuse_canary_image.outputs.reused == 'false')) }} strategy: # Independent legs: one architecture failing should still publish # nothing, but it must not also hide the other's logs behind a @@ -277,6 +308,7 @@ jobs: # tags. Nothing is publicly tagged until this runs, so a half-published # multi-arch image is not a state anything can pull. merge-and-push: + if: ${{ !cancelled() && needs.build-and-push.result == 'success' }} needs: build-and-push runs-on: ubuntu-latest timeout-minutes: 20 @@ -407,11 +439,8 @@ jobs: # promotion is gated on its own completed pushes). Every interleaving # converges the Docker channel onto the npm channel. promote_canary_channel: - if: startsWith(github.ref, 'refs/tags/canary/v') - # merge-and-push, not build-and-push: the per-arch legs push untagged - # digests, and the production `sha-*` tags this promotion retags only - # exist once the manifest merge has named them. - needs: [merge-and-push] + if: ${{ !cancelled() && startsWith(github.ref, 'refs/tags/canary/v') && (needs.merge-and-push.result == 'success' || (needs.reuse_canary_image.result == 'success' && needs.reuse_canary_image.outputs.reused == 'true')) }} + needs: [reuse_canary_image, merge-and-push] runs-on: ubuntu-latest timeout-minutes: 10 permissions: diff --git a/.github/workflows/pr-trusted.yml b/.github/workflows/pr-trusted.yml index b1b73c1f085..dae203d6216 100644 --- a/.github/workflows/pr-trusted.yml +++ b/.github/workflows/pr-trusted.yml @@ -202,7 +202,7 @@ jobs: run: node "$CI_NODE_TEST" ./scripts/__tests__/e2e-shard.test.mjs - name: Test release verify workflow wiring - run: node "$CI_NODE_TEST" ./scripts/__tests__/release-verify-workflow.test.mjs ./scripts/__tests__/eval-schedule-monitor.test.mjs ./scripts/__tests__/prepare-runner-release-image.test.mjs ./scripts/cloud-source-verification.test.mjs ./scripts/standard-image-contract.test.mjs + run: node "$CI_NODE_TEST" ./scripts/__tests__/release-verify-workflow.test.mjs ./scripts/__tests__/eval-schedule-monitor.test.mjs ./scripts/__tests__/prepare-runner-release-image.test.mjs ./scripts/cloud-source-verification.test.mjs ./scripts/standard-image-contract.test.mjs ./scripts/reuse-canary-image.test.mjs - name: Test standalone package build concurrency run: node "$CI_NODE_TEST" ./scripts/__tests__/build-standalone-concurrency.test.mjs diff --git a/doc/RELEASING.md b/doc/RELEASING.md index dbf40956a50..70e28fe798f 100644 --- a/doc/RELEASING.md +++ b/doc/RELEASING.md @@ -86,7 +86,7 @@ internal traceability tag. moves them to the versioned name - canaries, nightlies, and betas never create GitHub Releases - canaries, nightlies, and betas never require changelog generation -- Docker `:latest` moves only on stable releases; master builds publish +- Docker `:latest` moves only on stable releases; canary dispatches promote `:canary`, nightly builds `:nightly`, and beta builds `:beta` ## TL;DR @@ -310,12 +310,12 @@ The workflow: ## Docker Image Tags -[`docker.yml`](../.github/workflows/docker.yml) publishes both the self-hosted -image and the `-cloud` variant with the same lane mapping: +[`docker.yml`](../.github/workflows/docker.yml) publishes the standard image: | Build ref | Tags | | --- | --- | -| `master` push | `:canary`, `:sha-` | +| `master` push | `:sha-`, `:sha-` | +| `canary/v*` dispatch | `:sha-`, then `:canary` for the current npm canary | | `nightly/v*` tag | `:nightly`, `:sha-` | | `beta/v*` tag | `:beta`, `:sha-` | | `v*` tag (stable) | `:latest`, `:YYYY.MDD.P`, `:YYYY.MDD`, `:sha-` | @@ -325,6 +325,21 @@ suppresses push-triggered workflow runs for those pushes. The release jobs therefore dispatch `docker.yml` explicitly at the new tag ref; the tag mapping keys off `github.ref` either way. +Canary dispatches first look up the full-SHA master image. Reuse requires a +valid multi-platform index with both `linux/amd64` and `linux/arm64`, a matching +registry digest, and signed provenance from this repository's `docker.yml` +on `master` at the exact source commit. The image is retagged by immutable +digest. This preserves the original build inputs and skips both native builds +and the manifest merge. A missing full-SHA image runs the normal build instead; +registry errors or invalid provenance fail the run. Rerun after the master +image's attestation finishes if the image became visible before its proof. + +Both paths retain the serialized promotion that resolves the current npm +`canary` dist-tag before moving Docker `:canary`. Nightly, beta, stable, and +master builds retain their existing build paths. Downstream image builds can +continue to consume the signed full-SHA master image independently of npm +canary publication. + The Docker build matrix and the release preview image builder configure BuildKit to check `mirror.gcr.io` for Docker Hub images first. This covers both the Dockerfile frontend and base images. BuildKit keeps Docker Hub as diff --git a/scripts/reuse-canary-image.mjs b/scripts/reuse-canary-image.mjs new file mode 100644 index 00000000000..3a43524dd0b --- /dev/null +++ b/scripts/reuse-canary-image.mjs @@ -0,0 +1,34 @@ +import assert from "node:assert/strict"; +import { execFileSync } from "node:child_process"; +import { appendFile } from "node:fs/promises"; +import { pathToFileURL } from "node:url"; +import { resolveStandardImageDigest } from "./standard-image-contract.mjs"; + +export async function reuseCanaryImage(sha, { + resolve = source => resolveStandardImageDigest(source, fetch, { missing: "return-null" }), + run = (command, args) => execFileSync(command, args, { stdio: "inherit" }), +} = {}) { + assert.match(sha ?? "", /^[a-f0-9]{40}$/); + const digest = await resolve(sha); + if (digest === null) return false; + assert.match(digest, /^sha256:[a-f0-9]{64}$/); + const image = `ghcr.io/paperclipai/paperclip@${digest}`; + await run("gh", ["attestation", "verify", `oci://${image}`, + "--repo", "paperclipai/paperclip", "--bundle-from-oci", + "--signer-workflow", "paperclipai/paperclip/.github/workflows/docker.yml", + "--source-ref", "refs/heads/master", "--source-digest", sha, + "--signer-digest", sha, "--deny-self-hosted-runners"]); + await run("docker", ["buildx", "imagetools", "create", "-t", + `ghcr.io/paperclipai/paperclip:sha-${sha.slice(0, 7)}`, image]); + return true; +} + +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + const reused = await reuseCanaryImage(process.env.GITHUB_SHA); + if (process.env.GITHUB_OUTPUT) await appendFile(process.env.GITHUB_OUTPUT, `reused=${reused}\n`); + const summary = reused + ? "Reused the signed multi-platform master image. Skipped both image builds and the manifest merge." + : "No full-SHA master image exists. Building both platforms for this canary."; + console.log(summary); + if (process.env.GITHUB_STEP_SUMMARY) await appendFile(process.env.GITHUB_STEP_SUMMARY, `${summary}\n`); +} diff --git a/scripts/reuse-canary-image.test.mjs b/scripts/reuse-canary-image.test.mjs new file mode 100644 index 00000000000..26dfbc83c5f --- /dev/null +++ b/scripts/reuse-canary-image.test.mjs @@ -0,0 +1,75 @@ +import assert from "node:assert/strict"; +import { readFileSync } from "node:fs"; +import test from "node:test"; +import { reuseCanaryImage } from "./reuse-canary-image.mjs"; + +const sha = "a".repeat(40); +const digest = `sha256:${"b".repeat(64)}`; + +test("reuses only an exact-source, master-signed multi-platform image by digest", async () => { + const calls = []; + const reused = await reuseCanaryImage(sha, { + resolve: async source => { assert.equal(source, sha); return digest; }, + run: async (...args) => calls.push(args), + }); + assert.equal(reused, true); + assert.deepEqual(calls, [ + ["gh", ["attestation", "verify", `oci://ghcr.io/paperclipai/paperclip@${digest}`, + "--repo", "paperclipai/paperclip", "--bundle-from-oci", + "--signer-workflow", "paperclipai/paperclip/.github/workflows/docker.yml", + "--source-ref", "refs/heads/master", "--source-digest", sha, + "--signer-digest", sha, "--deny-self-hosted-runners"]], + ["docker", ["buildx", "imagetools", "create", "-t", + "ghcr.io/paperclipai/paperclip:sha-aaaaaaa", `ghcr.io/paperclipai/paperclip@${digest}`]], + ]); +}); + +test("a missing master image enables the fallback without retagging", async () => { + assert.equal(await reuseCanaryImage(sha, { + resolve: async () => null, + run: () => assert.fail("A missing image must not be promoted"), + }), false); +}); + +test("registry and provenance failures stop promotion rather than bypass verification", async () => { + await assert.rejects(reuseCanaryImage(sha, { + resolve: async () => { throw new Error("HTTP 403"); }, + run: () => assert.fail("A registry failure must not be promoted"), + }), /HTTP 403/); + let commands = 0; + await assert.rejects(reuseCanaryImage(sha, { + resolve: async () => digest, + run: command => { + assert.equal(command, "gh"); + commands++; + throw new Error("Untrusted source or signature"); + }, + }), /Untrusted source/); + assert.equal(commands, 1); +}); + +test("invalid source refs and digests cannot reach external commands", async () => { + for (const invalid of ["master", "a".repeat(7), "a".repeat(40) + "\n"]) { + await assert.rejects(reuseCanaryImage(invalid, { + resolve: () => assert.fail("Invalid source accepted"), + })); + } + await assert.rejects(reuseCanaryImage(sha, { + resolve: async () => "mutable:tag", + run: () => assert.fail("Invalid digest accepted"), + })); +}); + +test("canary reuse gates both native builds and still runs convergent promotion", () => { + const workflow = readFileSync(new URL("../.github/workflows/docker.yml", import.meta.url), "utf8"); + const reuse = workflow.split(" reuse_canary_image:\n")[1].split(" build-and-push:\n")[0]; + assert.match(reuse, /github.repository_id == '1170821064'.*startsWith\(github.ref, 'refs\/tags\/canary\/v'\)/); + assert.match(reuse, /node scripts\/reuse-canary-image.mjs/); + const build = workflow.split(" build-and-push:\n")[1].split(" merge-and-push:\n")[0]; + assert.match(build, /needs: \[reuse_canary_image\]/); + assert.match(build, /!cancelled\(\).*result == 'skipped'.*result == 'success'.*outputs.reused == 'false'/); + assert.match(build, /platform: linux\/amd64/); + assert.match(build, /platform: linux\/arm64/); + const promotion = workflow.split(" promote_canary_channel:\n")[1]; + assert.match(promotion, /!cancelled\(\).*needs.merge-and-push.result == 'success'.*outputs.reused == 'true'/); +}); diff --git a/scripts/standard-image-contract.mjs b/scripts/standard-image-contract.mjs index 62a8249e2da..5e0ae3cc24c 100644 --- a/scripts/standard-image-contract.mjs +++ b/scripts/standard-image-contract.mjs @@ -21,6 +21,7 @@ export function standardImageDigest(bytes) { /** Read registry bytes directly; CLI stdout formatting is not artifact content. */ export async function resolveStandardImageDigest(sha, fetchImpl = fetch, { sleep = ms => new Promise(resolve => setTimeout(resolve, ms)), + missing = "error", } = {}) { assert.match(sha ?? "", /^[a-f0-9]{40}$/); const request = async (url, headers, consume, allowNotFound = false) => { @@ -35,6 +36,7 @@ export async function resolveStandardImageDigest(sha, fetchImpl = fetch, { } if (!response.ok) { await response.body?.cancel(); + if (allowNotFound && response.status === 404 && missing === "return-null") return null; const transient = [408, 425, 429].includes(response.status) || response.status >= 500 || (allowNotFound && response.status === 404); assert.ok(transient && attempt < 5, `Registry lookup failed: HTTP ${response.status}`); const retryAfter = Number(response.headers.get("retry-after")); @@ -53,7 +55,7 @@ export async function resolveStandardImageDigest(sha, fetchImpl = fetch, { }; const { token } = await request("https://ghcr.io/token?service=ghcr.io&scope=repository:paperclipai/paperclip:pull", {}, response => response.json()); assert.ok(typeof token === "string" && token, "Missing public pull token"); - const { raw, expected } = await request(`https://ghcr.io/v2/paperclipai/paperclip/manifests/sha-${sha}`, { + const manifest = await request(`https://ghcr.io/v2/paperclipai/paperclip/manifests/sha-${sha}`, { authorization: `Bearer ${token}`, accept: "application/vnd.oci.image.index.v1+json, application/vnd.docker.distribution.manifest.list.v2+json", }, async response => { assert.ok(response.body, "Standard image index has no body"); @@ -71,6 +73,8 @@ export async function resolveStandardImageDigest(sha, fetchImpl = fetch, { } finally { await reader.cancel(); } return { raw: Buffer.concat(chunks), expected: response.headers.get("docker-content-digest") }; }, true); + if (manifest === null) return null; + const { raw, expected } = manifest; const digest = standardImageDigest(raw); assert.equal(digest, expected, "Registry digest does not match the index bytes"); return digest; diff --git a/scripts/standard-image-contract.test.mjs b/scripts/standard-image-contract.test.mjs index af2d4462977..d61f3fb862d 100644 --- a/scripts/standard-image-contract.test.mjs +++ b/scripts/standard-image-contract.test.mjs @@ -77,3 +77,17 @@ test("permanent registry errors fail immediately and transient errors exhaust a assert.equal(calls, expectedCalls); } }); + +test("optional lookup falls back only for a missing manifest", async () => { + for (const status of [401, 403, 404, 503]) { + let calls = 0; + const lookup = resolveStandardImageDigest("a".repeat(40), async url => { + if (url.includes("/token?")) return Response.json({ token: "public-fixture" }); + calls++; + return new Response(null, { status }); + }, { missing: "return-null", sleep: async () => {} }); + if (status === 404) assert.equal(await lookup, null); + else await assert.rejects(lookup, /Registry lookup failed/); + assert.equal(calls, status === 503 ? 5 : 1); + } +}); From 32044a63e138e7b6a38ec801878d0db65cce6bbe Mon Sep 17 00:00:00 2001 From: Dotta Date: Fri, 30 Oct 2026 19:00:51 -0500 Subject: [PATCH 58/74] fix: scope status-card deletion notifications to its company --- doc/plans/2026-10-30-status-cards-sleep.md | 6 ++++ .../status-card-evaluation-work.test.ts | 32 +++++++++++++++++++ server/src/services/status-cards.ts | 6 ++-- 3 files changed, 42 insertions(+), 2 deletions(-) diff --git a/doc/plans/2026-10-30-status-cards-sleep.md b/doc/plans/2026-10-30-status-cards-sleep.md index 5b98962698b..61d451caed5 100644 --- a/doc/plans/2026-10-30-status-cards-sleep.md +++ b/doc/plans/2026-10-30-status-cards-sleep.md @@ -135,3 +135,9 @@ scheduled work after settlement. Real issue/comment producer acceptance measures due/next queries and actual searches under 100 writes, preserves both companies' hints, dispatches another company's saved deadline inside the batching window, and checks SQL quietness after authorized results. + +Card deletion resolves the card's owning company and emits only that company's +deadline-reconstruction hint. Every card mutation requires a company ID; only +instance-level settings retain wildcard ingress. Deleting an already-missing +card emits no hint. The real two-company regression proves deletion does not +re-run unchanged reactive searches or wake either summarizer, then stays quiet. diff --git a/server/src/__tests__/status-card-evaluation-work.test.ts b/server/src/__tests__/status-card-evaluation-work.test.ts index 6bb466dea89..e4588b16be1 100644 --- a/server/src/__tests__/status-card-evaluation-work.test.ts +++ b/server/src/__tests__/status-card-evaluation-work.test.ts @@ -144,6 +144,38 @@ const support = external ? { supported: true } : await getEmbeddedPostgresTestSu expect(errors).toEqual([]); }); + it("keeps card deletion scoped to its company and a missing deletion quiet", async () => { + const removed = await card(); + await card(); + const ownCompany = companyId, ownAgent = agentId; + companyId = randomUUID(); agentId = randomUUID(); + await db.insert(companies).values({ id: companyId, name: "Other", issuePrefix: `O${companyId.slice(0, 6)}` }); + await db.insert(agents).values({ id: agentId, companyId, name: "Other summarizer", role: "general", status: "idle", adapterType: "process", adapterConfig: {} }); + const other = await card(); + companyId = ownCompany; agentId = ownAgent; + const factory = companySearch.companySearchService; + const searchedCompanies: string[] = []; + vi.spyOn(companySearch, "companySearchService").mockImplementation(owner => { + const search = factory(owner), actual = search.search; + search.search = async (...args) => { searchedCompanies.push(args[0]); return actual(...args); }; + return search; + }); + await start(); await quiet(); searchedCompanies.length = 0; + const cards = statusCardService(db); + expect(await cards.remove(removed.id)).toMatchObject({ id: removed.id, companyId: ownCompany }); + await settled(); + // Deleting a card only reconstructs deadlines; it is not an issue change + // that warrants re-running any unchanged reactive card's saved search. + expect(searchedCompanies).toEqual([]); + expect(await cards.getById(other.id)).toMatchObject({ nextEvalAt: null, generatingIssueId: null, pendingChangeCount: 0 }); + expect(wakeup).not.toHaveBeenCalled(); + searchedCompanies.length = 0; + expect(await cards.remove(removed.id)).toBeNull(); + await quiet(); + expect(searchedCompanies).toEqual([]); + expect(errors).toEqual([]); + }); + it("coalesces one hundred committed source writes without delaying another company's saved deadline", async () => { const reactive = await card(); const originalCompany = companyId, originalAgent = agentId; diff --git a/server/src/services/status-cards.ts b/server/src/services/status-cards.ts index f9e6372ea2e..d4660227c28 100644 --- a/server/src/services/status-cards.ts +++ b/server/src/services/status-cards.ts @@ -158,7 +158,7 @@ export function statusCardService( const issuesSvc = () => deps.issuesSvc ?? issueService(db); const searchSvc = companySearchService(db); type Transaction = Parameters[0]>[0]; - function mutate(companyId: string | undefined, write: (tx: Transaction) => Promise) { + function mutate(companyId: string, write: (tx: Transaction) => Promise) { return db.transaction(async tx => { await notifyStatusCardEvaluation(tx, companyId); return write(tx); @@ -335,7 +335,9 @@ export function statusCardService( } async function remove(id: string) { - return mutate(undefined, tx => tx.delete(statusCards).where(eq(statusCards.id, id)).returning().then((rows) => rows[0] ?? null)); + const card = await getById(id); + if (!card) return null; + return mutate(card.companyId, tx => tx.delete(statusCards).where(eq(statusCards.id, id)).returning().then((rows) => rows[0] ?? null)); } async function listUpdates(cardId: string) { From e41bde214638004b51752091938834691eb1c616 Mon Sep 17 00:00:00 2001 From: Dotta Date: Fri, 30 Oct 2026 19:01:12 -0500 Subject: [PATCH 59/74] fix(server): bound graph queries and scope issue retention work Chunk graph recovery, agent and workspace inputs; prove a70000-node PostgreSQL graph stays below parameter limits. Route ordinary issue writes through identity-scoped dependency invalidation for old/new parents, blocker ancestors, run links and liveness origins. Correlate all issue-related attention sources, preserve unrelated deadlines, and canonicalize notification keys. Exercise actual held/burst producers, visibility, relation and incident transitions. Co-Authored-By: Paperclip --- .../2026-10-30-attention-retention-sleep.md | 2 +- .../attention-retention-work.test.ts | 111 ++++++++++++++- server/src/services/attention-retention.ts | 31 +++-- .../issue-review-state-notifications.ts | 2 - server/src/services/issues.ts | 127 +++++++++--------- ...ative-status-reassessment-notifications.ts | 4 +- .../services/retention-work-notifications.ts | 48 ++++++- 7 files changed, 237 insertions(+), 88 deletions(-) diff --git a/doc/plans/2026-10-30-attention-retention-sleep.md b/doc/plans/2026-10-30-attention-retention-sleep.md index 463fbe94188..24212b4b98a 100644 --- a/doc/plans/2026-10-30-attention-retention-sleep.md +++ b/doc/plans/2026-10-30-attention-retention-sleep.md @@ -25,7 +25,7 @@ The feed and worker share confirmation collapse, newer-run probes, issue visibil ## Existing mutation boundaries reactivate work -Committed hints identify a company, source kind, and source ID where those values are known. Retention's own writes use exact source hints; blocker retention changes invalidate the company/source because multiple blocked roots can project one terminal subject. Reconstruction pages and due batches remain bounded. Review attention and interaction invokability are computed in company batches per candidate page; a full 100-candidate review page uses at most 20 statements and 100 confirmations use at most six. Decisions, recovery source links, runs, eligible agents, issue visibility, newer-run probes and blocker graphs also load in company batches per page. A 100-source decision or recovery page uses at most three statements; 100 failed runs use at most six; 100 blocker roots use at most 30 graph/source statements plus retention-state reads for their emitted entries. These evidence maps exist only for one load. The blocker owner shares fetched node and edge facts while tracking depth, node count, expanded nodes and live-work reachability independently for each root. An unrelated graph reaching its limit cannot change a shallow root’s terminal subject or deadline. Archive checks still lock and freshly re-read sources, terminal subjects and related eligibility. Ordinary execution, decision and budget mutations pass their affected company; exact decision sources are included. +Committed hints identify a company, source kind, source ID or changed issue where those values are known. Routine issue writers pass identity to the retention domain helper before their mutation. The helper captures old and new parent paths, reverse blocker ancestors, run-linked issues and liveness-origin references; issue-scoped source queries then select reviews, blockers, interactions, decisions, recovery actions and failed runs by their issue identity. Budget incidents and agent errors remain tied to their own source changes. Ordinary issue writes do not reconstruct the unrelated company feed. Queue keys explicitly pick supported identity fields. Retention's own writes use exact source hints; blocker retention changes invalidate the company/source because multiple blocked roots can project one terminal subject. Reconstruction pages and due batches remain bounded. Review attention and interaction invokability are computed in company batches per candidate page; a full 100-candidate review page uses at most 20 statements and 100 confirmations use at most six. Decisions, recovery source links, runs, eligible agents, issue visibility, newer-run probes and blocker graphs also load in company batches per page. A 100-source decision or recovery page uses at most three statements; 100 failed runs use at most six; 100 blocker roots use at most 30 graph/source statements plus retention-state reads for their emitted entries. These evidence maps exist only for one load. The blocker owner shares fetched node and edge facts while tracking depth, node count, expanded nodes and live-work reachability independently for each root. An unrelated graph reaching its limit cannot change a shallow root’s terminal subject or deadline. All later graph queries, including recovery source IDs, agent IDs and workspace-finalize inputs, use bounded chunks. A SQL-generated 70,000-node graph proves PostgreSQL parameter limits are respected at the driver boundary. Archive checks still lock and freshly re-read sources, terminal subjects and related eligibility. Ordinary execution, decision and budget mutations pass their affected company; exact decision sources are included. Producer coverage includes decision creation and changes; interaction creation, resolution, expiry and supersession; issue creation, updates, blocker relations, checkout, adoption and release; agent hire, configuration, lifecycle, error clearing and deletion; budget incidents and company pause/resume; exhausted-run events and retry-run creation; native status, review, finalization and reconstruction; external conversation transitions; and imported blocker relations. Existing service transactions retain outer commit ownership. Savepoint release does not publish work before outer commit. diff --git a/server/src/__tests__/attention-retention-work.test.ts b/server/src/__tests__/attention-retention-work.test.ts index 29a82f866a5..b1e7949a8b7 100644 --- a/server/src/__tests__/attention-retention-work.test.ts +++ b/server/src/__tests__/attention-retention-work.test.ts @@ -1,7 +1,7 @@ import { randomUUID } from "node:crypto"; import { and, eq, sql } from "drizzle-orm"; import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; -import { authUsers, companyMemberships, agents, approvals, budgetIncidents, budgetPolicies, companies, createDb, applyPendingMigrations, decisionArchiveNotificationOutbox, +import { subscribeDatabaseWork, authUsers, companyMemberships, agents, approvals, budgetIncidents, budgetPolicies, companies, createDb, applyPendingMigrations, decisionArchiveNotificationOutbox, decisionRetention, decisions, costEvents, heartbeatRunEvents, heartbeatRuns, issueApprovals, issueRecoveryActions, issueRelations, issueThreadInteractions, issues, type Db } from "@paperclipai/db"; import { attentionRetentionService, registerAttentionRetentionWork } from "../services/attention-retention.js"; @@ -13,6 +13,7 @@ import { companyService } from "../services/companies.js"; import { createDeliveryWorkCoordinator } from "../services/delivery-work-coordinator.js"; import { decisionRetentionService, DEFAULT_DECISION_ARCHIVE_DAYS } from "../services/decision-retention.js"; import { appendHeartbeatRunEvent } from "../services/heartbeat-run-events.js"; +import { buildIssueGraphLivenessIncidentKey, RECOVERY_ORIGIN_KINDS } from "../services/recovery/origins.js"; import { issueThreadInteractionService } from "../services/issue-thread-interactions.js"; import { createAgentLifecycle } from "../services/agent-lifecycle.js"; import { BLOCKER_ATTENTION_MAX_DEPTH, BLOCKER_ATTENTION_MAX_NODES, issueService } from "../services/issues.js"; @@ -32,12 +33,13 @@ const delay = DEFAULT_DECISION_ARCHIVE_DAYS * 86_400_000; let coordinator: ReturnType | undefined; const errors: unknown[] = []; const statements: string[] = []; + const parameterCounts: number[] = []; beforeAll(async () => { if (externalUrl) await applyPendingMigrations(externalUrl); else temporary = await startEmbeddedPostgresTestDatabase("attention-retention-"); db = createDb(externalUrl ?? temporary!.connectionString, { maxConnections: 6, idleTimeoutSeconds: 0, maxLifetimeSeconds: 0 }); db.$client.options.onnotice = () => {}; - db.$client.options.debug = (_connection, query) => statements.push(query); + db.$client.options.debug = (_connection, query, parameters) => { statements.push(query); parameterCounts.push(parameters.length); }; }, 30_000); beforeEach(async () => { companyId = randomUUID(); agentId = randomUUID(); draining = false; errors.length = 0; @@ -311,6 +313,111 @@ const delay = DEFAULT_DECISION_ARCHIVE_DAYS * 86_400_000; } }); + it("bounds every graph query when independent root trees contain over 65535 unfinished nodes", async () => { + await db.execute(sql`insert into issues (id, company_id, title, status, updated_at) + select gen_random_uuid(), ${companyId}::uuid, 'parameter-root-' || n, 'blocked', ${old().toISOString()}::timestamptz + from generate_series(1, 70) n`); + await db.execute(sql`insert into issues (id, company_id, parent_id, title, status, assignee_agent_id, updated_at) + select gen_random_uuid(), ${companyId}::uuid, root.id, 'parameter-leaf-' || n, 'backlog', ${agentId}::uuid, ${old().toISOString()}::timestamptz + from issues root cross join generate_series(1, 999) n + where root.company_id = ${companyId}::uuid and root.status = 'blocked'`); + const roots = await db.select().from(issues).where(and(eq(issues.companyId, companyId), eq(issues.status, "blocked"))); + parameterCounts.length = 0; + const result = await issueService(db).listBlockerAttention(companyId, roots).catch(error => { + throw new Error(`Large graph failed: ${String(error.cause?.message ?? error.message).slice(0, 300)}`); + }); + expect(result.size).toBe(70); + expect([...result.values()].every(item => item.state === "needs_attention" && item.terminalBlockerIssueId)).toBe(true); + expect(Math.max(...parameterCounts)).toBeLessThanOrEqual(1100); + }, 120_000); + + it("canonicalizes wide circular notification variables to the supported identity fields", async () => { + const sourceId = randomUUID(), issueId = randomUUID(), hints: string[] = []; + const wide = { companyId, sourceKind: "review" as const, sourceId, issueId, ignored: {} as unknown }; + wide.ignored = wide; + const unsubscribe = subscribeDatabaseWork(db, DELIVERY_QUEUES.attentionRetention, (event, values) => { + if (event === "settled") hints.push(...values ?? []); + }); + try { await db.transaction(tx => notifyAttentionRetentionWork(tx, wide)); } finally { unsubscribe(); } + expect(hints).toEqual([JSON.stringify({ companyId, sourceKind: "review", sourceId, issueId })]); + }); + + it("keeps held, burst and sustained ordinary issue writes scoped while an unrelated deadline fires", async () => { + const row = await issueService(db).create(companyId, { title: "Active review", status: "in_review", assigneeUserId: "board", createdByAgentId: agentId }); + const urgent = await insertIssue("in_review", { assigneeUserId: "board", updatedAt: new Date(Date.now() - delay + 10_000) }); + const otherCompany = randomUUID(); + await db.insert(companies).values({ id: otherCompany, name: "Unrelated", issuePrefix: "BURST" }); + const foreign = await db.insert(issues).values(Array.from({ length: 125 }, () => ({ companyId: otherCompany, + title: randomUUID(), status: "in_review", assigneeUserId: "board", updatedAt: old() }))).returning(); + await db.insert(decisionRetention).values(foreign.map(issue => ({ companyId: otherCompany, sourceKind: "review", sourceId: issue.id, sourceActivityAt: issue.updatedAt, keep: true }))); + const service = attentionRetentionService(db), original = service.load, load = vi.spyOn(service, "load"); + await start(service); await settle(); load.mockClear(); + let release!: () => void, entered = false; + const gate = new Promise(resolve => { release = resolve; }); + load.mockImplementationOnce(async (...args) => { const page = await original(...args); entered = true; await gate; return page; }); + statements.length = 0; + await issueService(db).update(row.id, { title: "Held mutation" }); + await vi.waitFor(() => expect(entered).toBe(true), { timeout: 15_000 }); + await Promise.all(Array.from({ length: 5 }, (_, index) => issueService(db).update(row.id, { title: `Burst ${index}` }))); + release(); await settle(); + for (let index = 0; index < 8; index++) { + await issueService(db).update(row.id, { title: `Steady ${index}` }); await vi.advanceTimersByTimeAsync(1000); await settle(); + } + await archived("review", urgent.id); + const routine = load.mock.calls.map(([scope]) => scope).filter(scope => !scope?.sourceId); + expect(routine.length).toBeGreaterThan(0); + expect(routine.every(scope => scope?.companyId === companyId && scope.issueId === row.id)).toBe(true); + const scans = statements.filter(query => query.includes("with source_candidates")); + expect(scans.length).toBeGreaterThan(0); + expect(scans.every(query => !query.includes("budget_incidents") && !query.includes("agent_error_alert"))).toBe(true); + expect(await state("review", row.id)).toBeNull(); + }); + + it("wakes all linked issue sources when an ordinary visibility mutation reveals their origin", async () => { + const origin = await insertIssue("in_review", { hiddenAt: old(), assigneeUserId: "board", responsibleUserId: "board" }); + const [run] = await db.insert(heartbeatRuns).values({ companyId, agentId, status: "failed", contextSnapshot: { issueId: origin.id }, createdAt: old(), finishedAt: old() }).returning(); + await db.insert(heartbeatRunEvents).values({ companyId, agentId, runId: run.id, seq: 1, eventType: "lifecycle", message: "Bounded retry exhausted: visibility" }); + const [decision] = await db.insert(decisions).values({ companyId, originAgentId: agentId, originIssueId: origin.id, originRunId: run.id, + title: "Decision", body: "Choose", options: [], signedSpec: "fixture", targetSnapshots: {}, expiresAt: new Date(Date.now() + 86_400_000), updatedAt: old() }).returning(); + const [recovery] = await db.insert(issueRecoveryActions).values({ companyId, sourceIssueId: origin.id, kind: "execution", ownerType: "board", cause: "fixture", fingerprint: "fixture", nextAction: "Choose", updatedAt: old() }).returning(); + const [interaction] = await db.insert(issueThreadInteractions).values({ companyId, issueId: origin.id, kind: "ask_user_questions", payload: { questions: [] }, updatedAt: old() }).returning(); + await start(); await quiet(); + await issueService(db).update(origin.id, { hiddenAt: null }); + for (const [kind, id] of [["decision", decision.id], ["recovery_action", recovery.id], ["failed_run", run.id], ["issue_thread_interaction", interaction.id]]) await archived(kind, id); + await settle(); expect(await state("review", origin.id)).toBeNull(); expect(coordinator!.nextWakeAt()).not.toBeNull(); + }); + + it("invalidates old and new parents, terminal visibility and repointed blocker relations", async () => { + const oldParent = await insertIssue("blocked", { updatedAt: new Date(), responsibleUserId: "board" }); + const newParent = await insertIssue("blocked", { updatedAt: new Date(), responsibleUserId: "board" }); + const terminal = await insertIssue("backlog", { parentId: oldParent.id, hiddenAt: old(), assigneeAgentId: agentId, responsibleUserId: "board" }); + const service = attentionRetentionService(db), load = vi.spyOn(service, "load"); await start(service); await settle(); load.mockClear(); + await issueService(db).update(terminal.id, { parentId: newParent.id }); await settle(); + const ids = load.mock.calls.map(([scope]) => scope?.issueId); + expect(ids).toEqual(expect.arrayContaining([oldParent.id, newParent.id, terminal.id])); + load.mockClear(); + await issueService(db).update(terminal.id, { hiddenAt: null }); await settle(); + expect(await state("blocker_attention", terminal.id)).toBeNull(); + const visible = await service.load({ companyId, issueId: newParent.id }); + expect(visible.rows!.some(entry => entry.sourceId === terminal.id && entry.dueAt > Date.now())).toBe(true); + expect(load.mock.calls.some(([scope]) => scope?.issueId === newParent.id)).toBe(true); + const next = await insertIssue("backlog", { assigneeAgentId: agentId }); + await issueService(db).update(oldParent.id, { blockedByIssueIds: [next.id] }); await archived("blocker_attention", next.id); + }); + + it("cancels and reactivates a blocker deadline through ordinary liveness incident creation and hiding", async () => { + const root = await insertIssue("blocked"); + const terminal = await insertIssue("backlog", { assigneeAgentId: agentId, updatedAt: new Date(Date.now() - delay + 10_000) }); + await db.insert(issueRelations).values({ companyId, issueId: terminal.id, relatedIssueId: root.id, type: "blocks" }); + await start(); await settle(); expect(coordinator!.nextWakeAt()).not.toBeNull(); + const incident = await issueService(db).create(companyId, { title: "Resolve liveness", status: "todo", assigneeUserId: "board", + createdByUserId: "board", originKind: RECOVERY_ORIGIN_KINDS.issueGraphLivenessEscalation, + originId: buildIssueGraphLivenessIncidentKey({ companyId, issueId: root.id, state: "needs_attention", blockerIssueId: terminal.id }) }); + await quiet(); expect(await state("blocker_attention", terminal.id)).toBeNull(); + await issueService(db).update(incident.id, { hiddenAt: new Date() }); + await archived("blocker_attention", terminal.id); await quiet(); + }); + it("scopes ordinary execution-control writes to their company", async () => { const otherCompany = randomUUID(); await db.insert(companies).values({ id: otherCompany, name: "Other", issuePrefix: "OTHER" }); diff --git a/server/src/services/attention-retention.ts b/server/src/services/attention-retention.ts index c72947e5349..27aedba8c47 100644 --- a/server/src/services/attention-retention.ts +++ b/server/src/services/attention-retention.ts @@ -18,8 +18,8 @@ const archiveDelay = DEFAULT_DECISION_ARCHIVE_DAYS * 86_400_000; const kinds = ["decision", "issue_thread_interaction", "recovery_action", "blocker_attention", "review", "failed_run", "budget_alert", "agent_error_alert"] as const; type Kind = typeof kinds[number]; -type Scope = { companyId?: string; sourceKind?: AttentionSourceKind; sourceId?: string }; -type Candidate = { sourceKind: Kind; sourceId: string; companyId: string; activityAt: string; deadlineAt: number }; +type Scope = { companyId?: string; sourceKind?: AttentionSourceKind; sourceId?: string; issueId?: string }; +type Candidate = { sourceKind: Kind; sourceId: string; companyId: string; activityAt: string; deadlineAt: number; issueId: string | null }; type Entry = Candidate & { id: string; dueAt: number; rootId: string; variant?: "owner" | "terminal" }; type Cursor = { sourceKind: string; sourceId: string }; @@ -42,37 +42,39 @@ const sourceTables: Record = { }; const sources: Record = { - decision: sql`select 'decision' as source_kind, d.id::text as source_id, d.company_id, d.updated_at as activity_at + decision: sql`select 'decision' as source_kind, d.id::text as source_id, d.company_id, d.updated_at as activity_at, d.origin_issue_id::text as issue_id from decisions d where d.status = 'open'`, - issue_thread_interaction: sql`select 'issue_thread_interaction', i.id::text, i.company_id, i.updated_at + issue_thread_interaction: sql`select 'issue_thread_interaction', i.id::text, i.company_id, i.updated_at, i.issue_id::text from issue_thread_interactions i where i.status = 'pending' and i.addressee_user_id is null`, - recovery_action: sql`select 'recovery_action', r.id::text, r.company_id, r.updated_at + recovery_action: sql`select 'recovery_action', r.id::text, r.company_id, r.updated_at, r.source_issue_id::text from issue_recovery_actions r where r.status in ('active', 'escalated') and r.owner_type in (${sql.join(HUMAN_RECOVERY_OWNER_TYPES.map(owner => sql`${owner}`), sql`, `)})`, - blocker_attention: sql`select 'blocker_attention', i.id::text, i.company_id, i.updated_at + blocker_attention: sql`select 'blocker_attention', i.id::text, i.company_id, i.updated_at, i.id::text from issues i where i.status = 'blocked' and i.hidden_at is null and i.harness_kind is null`, - review: sql`select 'review', i.id::text, i.company_id, i.updated_at + review: sql`select 'review', i.id::text, i.company_id, i.updated_at, i.id::text from issues i where i.status = 'in_review' and i.hidden_at is null and i.harness_kind is null`, - failed_run: sql`select 'failed_run', r.id::text, r.company_id, coalesce(r.finished_at, r.updated_at, r.created_at) + failed_run: sql`select 'failed_run', r.id::text, r.company_id, coalesce(r.finished_at, r.updated_at, r.created_at), coalesce(r.context_snapshot->>'issueId', r.context_snapshot->>'taskId') from heartbeat_runs r where r.status in ('failed', 'timed_out') and exists ( select 1 from heartbeat_run_events e where e.run_id = r.id and e.company_id = r.company_id and e.event_type = 'lifecycle' and e.message like ${`${RETRY_EXHAUSTION_MESSAGE_PREFIX}%`})`, - budget_alert: sql`select 'budget_alert', b.id::text, b.company_id, b.updated_at from budget_incidents b + budget_alert: sql`select 'budget_alert', b.id::text, b.company_id, b.updated_at, null::text from budget_incidents b where b.status = 'open' and (b.threshold_type = 'hard' or (b.amount_limit > 0 and round(b.amount_observed::numeric / b.amount_limit * 10000) / 100 >= ${ATTENTION_BUDGET_WARNING_PERCENT}))`, - agent_error_alert: sql`select 'agent_error_alert', a.id::text, a.company_id, a.updated_at from agents a where a.status = ${ATTENTION_AGENT_ERROR_STATUS}`, + agent_error_alert: sql`select 'agent_error_alert', a.id::text, a.company_id, a.updated_at, null::text from agents a where a.status = ${ATTENTION_AGENT_ERROR_STATUS}`, }; export function attentionRetentionService(db: Db) { const cursors = new Map(); async function candidates(database: Db, scope: Scope = {}, cursor?: Cursor, sourceId?: string): Promise { - const selected = kinds.filter(kind => !scope.sourceKind || scope.sourceKind === kind); + const selected = kinds.filter(kind => (!scope.sourceKind || scope.sourceKind === kind) + && (!scope.issueId || !["budget_alert", "agent_error_alert"].includes(kind))); if (!selected.length) return []; - const rows = await database.execute(sql`with source_candidates(source_kind, source_id, company_id, activity_at) as (${sql.join(selected.map(kind => sources[kind]), sql` union all `)}) + const rows = await database.execute(sql`with source_candidates(source_kind, source_id, company_id, activity_at, issue_id) as (${sql.join(selected.map(kind => sources[kind]), sql` union all `)}) select s.source_kind as "sourceKind", s.source_id as "sourceId", s.company_id as "companyId", - s.activity_at::text as "activityAt", ceil(extract(epoch from s.activity_at) * 1000 + ${archiveDelay}) as "deadlineAt" + s.issue_id as "issueId", s.activity_at::text as "activityAt", ceil(extract(epoch from s.activity_at) * 1000 + ${archiveDelay}) as "deadlineAt" from source_candidates s inner join companies c on c.id = s.company_id left join decision_retention r on r.company_id = s.company_id and r.source_kind = s.source_kind and r.source_id = s.source_id where c.status = 'active' + and (${scope.issueId ?? null}::text is null or s.issue_id = ${scope.issueId ?? null}) and (${scope.companyId ?? null}::uuid is null or s.company_id = ${scope.companyId ?? null}::uuid) and (${sourceId ?? scope.sourceId ?? null}::text is null or s.source_id = ${sourceId ?? scope.sourceId ?? null}) and (s.source_kind = 'blocker_attention' or ((r.id is null or r.keep = false) and r.archived_at is null)) @@ -302,7 +304,8 @@ export function registerAttentionRetentionWork(coordinator: ReturnType hint === "*" ? undefined : JSON.parse(hint) as Scope, matches: (entry, scope) => (!scope?.companyId || scope.companyId === entry.companyId) && (!scope?.sourceKind || scope.sourceKind === entry.sourceKind) - && (!scope?.sourceId || scope.sourceId === entry.sourceId), + && (!scope?.sourceId || scope.sourceId === entry.sourceId) + && (!scope?.issueId || scope.issueId === entry.issueId), load: service.load, check: service.check, }, canRun); } diff --git a/server/src/services/issue-review-state-notifications.ts b/server/src/services/issue-review-state-notifications.ts index 353114a9bf0..679b7e37475 100644 --- a/server/src/services/issue-review-state-notifications.ts +++ b/server/src/services/issue-review-state-notifications.ts @@ -1,11 +1,9 @@ -import { notifyAttentionRetentionWork } from "./retention-work-notifications.js"; import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js"; import { notifyExternalConversationSettlement } from "./external-conversation-settlement-notifications.js"; import { notifyNativeReviewResponse } from "./native-review-response-notifications.js"; import { notifyNativeStatusReassessment } from "./native-status-reassessment-notifications.js"; export async function notifyIssueReviewState(tx: object, scope: { companyId: string; issueId?: string; runId?: string; agentId?: string }) { - await notifyAttentionRetentionWork(tx, { companyId: scope.companyId }); await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors); await notifyExternalConversationSettlement(tx, scope); await notifyNativeReviewResponse(tx, scope); diff --git a/server/src/services/issues.ts b/server/src/services/issues.ts index fd1fd9ba819..c91a325ad39 100644 --- a/server/src/services/issues.ts +++ b/server/src/services/issues.ts @@ -1,4 +1,4 @@ -import { notifyAttentionRetentionWork } from "./retention-work-notifications.js"; +import { notifyIssueAttentionRetentionWork } from "./retention-work-notifications.js"; import { notifyExternalConversationSettlement } from "./external-conversation-settlement-notifications.js"; import { notifyNativeRecoveryNotices } from "./native-recovery-notice-notifications.js"; import { notifyNativeRunProjection } from "./native-run-projection-notifications.js"; @@ -2538,33 +2538,37 @@ async function listPendingFinalizeBlockerIssueIds( ), ); - if (lockSources) { + const rows: Array<{ issueId: string | null; executionWorkspaceId: string | null; + phase: string; status: string; startedAt: Date }> = []; + for (const chunk of chunkList(executionWorkspaceIds, ISSUE_LIST_RELATED_QUERY_CHUNK_SIZE)) { + if (lockSources) { - await dbOrTx.select({ id: executionWorkspaces.id }).from(executionWorkspaces) - .where(and(eq(executionWorkspaces.companyId, companyId), inArray(executionWorkspaces.id, executionWorkspaceIds))) - .for("update", { noWait: true }); - } - const query = dbOrTx - .select({ - issueId: workspaceOperations.issueId, - executionWorkspaceId: workspaceOperations.executionWorkspaceId, - phase: workspaceOperations.phase, - status: workspaceOperations.status, - startedAt: workspaceOperations.startedAt, - }) - .from(workspaceOperations) - .where( - and( - eq(workspaceOperations.companyId, companyId), - inArray( - workspaceOperations.executionWorkspaceId, - executionWorkspaceIds, + await dbOrTx.select({ id: executionWorkspaces.id }).from(executionWorkspaces) + .where(and(eq(executionWorkspaces.companyId, companyId), inArray(executionWorkspaces.id, chunk))) + .for("update", { noWait: true }); + } + const query = dbOrTx + .select({ + issueId: workspaceOperations.issueId, + executionWorkspaceId: workspaceOperations.executionWorkspaceId, + phase: workspaceOperations.phase, + status: workspaceOperations.status, + startedAt: workspaceOperations.startedAt, + }) + .from(workspaceOperations) + .where( + and( + eq(workspaceOperations.companyId, companyId), + inArray( + workspaceOperations.executionWorkspaceId, + chunk, + ), ), - ), - ); + ); - const rows = await (lockSources ? query.for("share", { noWait: true }) : query); + rows.push(...await (lockSources ? query.for("share", { noWait: true }) : query)); + } const latestAttributedByBlockerWorkspace = new Map< string, @@ -4119,23 +4123,26 @@ async function listIssueBlockerAttentionMap( ownerType: string; ownerAgentId: string | null; ownerUserId: string | null; - }> = await dbOrTx - .select({ - id: issueRecoveryActions.id, - sourceIssueId: issueRecoveryActions.sourceIssueId, - status: issueRecoveryActions.status, - ownerType: issueRecoveryActions.ownerType, - ownerAgentId: issueRecoveryActions.ownerAgentId, - ownerUserId: issueRecoveryActions.ownerUserId, - }) - .from(issueRecoveryActions) - .where( - and( - eq(issueRecoveryActions.companyId, companyId), - inArray(issueRecoveryActions.status, ["active", "escalated"]), - inArray(issueRecoveryActions.sourceIssueId, explicitWaitCandidateIds), - ), - ); + }> = []; + for (const chunk of chunkList(explicitWaitCandidateIds, ISSUE_LIST_RELATED_QUERY_CHUNK_SIZE)) { + recoveryActionRows.push(...await dbOrTx + .select({ + id: issueRecoveryActions.id, + sourceIssueId: issueRecoveryActions.sourceIssueId, + status: issueRecoveryActions.status, + ownerType: issueRecoveryActions.ownerType, + ownerAgentId: issueRecoveryActions.ownerAgentId, + ownerUserId: issueRecoveryActions.ownerUserId, + }) + .from(issueRecoveryActions) + .where( + and( + eq(issueRecoveryActions.companyId, companyId), + inArray(issueRecoveryActions.status, ["active", "escalated"]), + inArray(issueRecoveryActions.sourceIssueId, chunk), + ), + )); + } const recoveryActionIds = recoveryActionRows.map((row) => row.id); const liveRecoveryActionIds = new Set(); for (const chunk of chunkList( @@ -4198,22 +4205,11 @@ async function listIssueBlockerAttentionMap( } } - const agentRows: IssueBlockerAttentionAgentRow[] = - agentIds.size > 0 - ? await dbOrTx - .select({ - id: agents.id, - companyId: agents.companyId, - status: agents.status, - }) - .from(agents) - .where( - and( - eq(agents.companyId, companyId), - inArray(agents.id, [...agentIds]), - ), - ) - : []; + const agentRows: IssueBlockerAttentionAgentRow[] = []; + for (const chunk of chunkList([...agentIds], ISSUE_LIST_RELATED_QUERY_CHUNK_SIZE)) { + agentRows.push(...await dbOrTx.select({ id: agents.id, companyId: agents.companyId, status: agents.status }) + .from(agents).where(and(eq(agents.companyId, companyId), inArray(agents.id, chunk)))); + } const agentsById = new Map(agentRows.map((agent) => [agent.id, agent])); type PathClassification = { @@ -7641,7 +7637,7 @@ export function issueService(db: Db) { await assertNoBlockingCycles(companyId, issueId, deduped, dbOrTx); } - await notifyAttentionRetentionWork(dbOrTx, { companyId }); + await notifyIssueAttentionRetentionWork(dbOrTx, { companyId, issueId }); await dbOrTx .delete(issueRelations) .where( @@ -10627,7 +10623,8 @@ export function issueService(db: Db) { } if (values.monitorNextCheckAt) await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors); await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers); - await notifyAttentionRetentionWork(tx, { companyId }); + values.id ??= randomUUID(); + await notifyIssueAttentionRetentionWork(tx, { companyId, issueId: values.id, parentId: values.parentId, originKind: values.originKind, originId: values.originId }); const [issue] = await tx.insert(issues).values(values).returning(); await attachOwnedDraftImages(tx, issue, issue.description, { userId: issueData.createdByUserId, agentId: issueData.createdByAgentId }); if (issue.visibility === "private" && !inheritedPrivateSubtree && issueData.createdByAgentId) { @@ -11513,7 +11510,7 @@ export function issueService(db: Db) { || issueData.parentId !== undefined || issueData.executionWorkspaceId !== undefined) { await notifyTerminalWorkspaceWork(tx, `issue:${id}`); } - await notifyAttentionRetentionWork(tx, { companyId: receiptExisting.companyId }); + await notifyIssueAttentionRetentionWork(tx, { companyId: receiptExisting.companyId, issueId: id, parentId: patch.parentId, originKind: patch.originKind, originId: patch.originId }); if (patch.status !== undefined || patch.assigneeAgentId !== undefined || patch.assigneeUserId !== undefined || patch.hiddenAt !== undefined || patch.conversationAgentId !== undefined) { await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers); @@ -11975,7 +11972,7 @@ export function issueService(db: Db) { await requestRestorePointDeletion(tx, { sourceIssueId: id }); const [owner] = await tx.select({ companyId: issues.companyId }).from(issues).where(eq(issues.id, id)); if (owner) { - await notifyAttentionRetentionWork(tx, { companyId: owner.companyId }); + await notifyIssueAttentionRetentionWork(tx, { companyId: owner.companyId, issueId: id }); await notifyExternalConversationSettlement(tx, { companyId: owner.companyId }); await notifyApprovalWork(tx, owner.companyId); } @@ -12108,7 +12105,7 @@ export function issueService(db: Db) { ) : isNull(issues.executionRunId); const updated = await db.transaction(async tx => { - await notifyAttentionRetentionWork(tx, { companyId: issueCompany.companyId }); + await notifyIssueAttentionRetentionWork(tx, { companyId: issueCompany.companyId, issueId: id }); await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors); await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers); return tx @@ -12238,7 +12235,7 @@ export function issueService(db: Db) { } const staleExecutionRunId = current.executionRunId; const adopted = await db.transaction(async tx => { - await notifyAttentionRetentionWork(tx, { companyId: issueCompany.companyId }); + await notifyIssueAttentionRetentionWork(tx, { companyId: issueCompany.companyId, issueId: id }); await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors); await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers); return tx @@ -12494,7 +12491,7 @@ export function issueService(db: Db) { existing.status === "in_progress" ? "todo" : existing.status; await notifyTerminalWorkspaceWork(tx, `issue:${id}`); await notifyNativeStatusReassessment(tx, { companyId: existing.companyId, issueId: id }); - await notifyAttentionRetentionWork(tx, { companyId: existing.companyId }); + await notifyIssueAttentionRetentionWork(tx, { companyId: existing.companyId, issueId: id }); await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors); await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers); const updated = await tx @@ -12547,7 +12544,7 @@ export function issueService(db: Db) { patch.assigneeAgentId = null; } - await notifyAttentionRetentionWork(tx, { companyId: existing.companyId }); + await notifyIssueAttentionRetentionWork(tx, { companyId: existing.companyId, issueId: id }); await notifyTerminalWorkspaceWork(tx, `issue:${id}`); if (existing.checkoutRunId || existing.executionRunId || options.clearAssignee) { await notifyExternalConversationSettlement(tx, { companyId: existing.companyId, issueId: id }); diff --git a/server/src/services/native-status-reassessment-notifications.ts b/server/src/services/native-status-reassessment-notifications.ts index 3a5ec8b6c37..30c1bbd098b 100644 --- a/server/src/services/native-status-reassessment-notifications.ts +++ b/server/src/services/native-status-reassessment-notifications.ts @@ -1,10 +1,10 @@ -import { notifyAttentionRetentionWork } from "./retention-work-notifications.js"; +import { notifyIssueAttentionRetentionWork } from "./retention-work-notifications.js"; import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js"; export type NativeStatusReassessmentScope = { companyId: string; issueId?: string; runId?: string; agentId?: string }; export async function notifyNativeStatusReassessment(tx: object, scope: NativeStatusReassessmentScope) { - await notifyAttentionRetentionWork(tx, { companyId: scope.companyId }); + await notifyIssueAttentionRetentionWork(tx, { companyId: scope.companyId, issueId: scope.issueId, runId: scope.runId }); return notifyDeliveryWork(tx, DELIVERY_QUEUES.nativeStatusReassessment, JSON.stringify({ companyId: scope.companyId, issueId: scope.issueId, runId: scope.runId, agentId: scope.agentId })); } diff --git a/server/src/services/retention-work-notifications.ts b/server/src/services/retention-work-notifications.ts index 0331f53f258..cee2486f54c 100644 --- a/server/src/services/retention-work-notifications.ts +++ b/server/src/services/retention-work-notifications.ts @@ -1,10 +1,54 @@ import { notifyApprovalWork } from "./approval-work-notifications.js"; +import { RECOVERY_ORIGIN_KINDS, parseIssueGraphLivenessIncidentKey } from "./recovery/origins.js"; +import { sql } from "drizzle-orm"; import type { Db } from "@paperclipai/db"; import type { AttentionSourceKind } from "@paperclipai/shared"; import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js"; -export async function notifyAttentionRetentionWork(transaction: object, scope: { companyId?: string; sourceKind?: AttentionSourceKind; sourceId?: string } = {}) { - await notifyDeliveryWork(transaction, DELIVERY_QUEUES.attentionRetention, JSON.stringify(scope)); +export async function notifyAttentionRetentionWork(transaction: object, scope: { companyId?: string; sourceKind?: AttentionSourceKind; issueId?: string; sourceId?: string } = {}) { + await notifyDeliveryWork(transaction, DELIVERY_QUEUES.attentionRetention, JSON.stringify({ + companyId: scope.companyId, sourceKind: scope.sourceKind, sourceId: scope.sourceId, issueId: scope.issueId, + })); +} + +export async function notifyIssueAttentionRetentionWork(transaction: object, + scope: { companyId: string; issueId?: string; parentId?: string | null; runId?: string; originKind?: string | null; originId?: string | null }) { + const database = transaction as Db; + const seeds = [scope.issueId, scope.parentId].filter((id): id is string => Boolean(id)); + if (!seeds.length && scope.runId) { + const rows = await database.execute(sql`select issue_id::text as id from heartbeat_runs + where company_id = ${scope.companyId}::uuid and id = ${scope.runId}::uuid and issue_id is not null + union select native_issue_id::text from heartbeat_runs + where company_id = ${scope.companyId}::uuid and id = ${scope.runId}::uuid and native_issue_id is not null + union select coalesce(context_snapshot->>'issueId', context_snapshot->>'taskId') from heartbeat_runs + where company_id = ${scope.companyId}::uuid and id = ${scope.runId}::uuid + and coalesce(context_snapshot->>'issueId', context_snapshot->>'taskId') is not null`); + for (const row of rows) seeds.push(row.id as string); + } + if (!seeds.length) return notifyAttentionRetentionWork(transaction, { companyId: scope.companyId }); + const origins = await database.execute(sql`select id::text as id, origin_kind as "originKind", origin_id as "originId" from issues + where company_id = ${scope.companyId}::uuid + and id in (${sql.join([...new Set(seeds)].map(id => sql`${id}::uuid`), sql`, `)})`); + const current = origins.find(source => source.id === scope.issueId); + origins.push({ originKind: scope.originKind ?? current?.originKind, originId: scope.originId ?? current?.originId }); + for (const source of origins) { + const incident = source.originKind === RECOVERY_ORIGIN_KINDS.issueGraphLivenessEscalation + ? parseIssueGraphLivenessIncidentKey(source.originId as string | null) : null; + if (incident?.companyId === scope.companyId) seeds.push(incident.issueId, incident.leafIssueId); + } + const affected = await database.execute(sql`with recursive affected(id) as ( + select unnest(array[${sql.join([...new Set(seeds)].map(id => sql`${id}::uuid`), sql`, `)}]) + union + select parent.id from affected current cross join lateral ( + select related_issue_id as id from issue_relations where company_id = ${scope.companyId}::uuid + and issue_id = current.id and type = 'blocks' + union select parent_id from issues where company_id = ${scope.companyId}::uuid + and id = current.id and parent_id is not null + ) parent + ) select id::text as "issueId" from affected`); + for (const row of affected) await notifyAttentionRetentionWork(transaction, { + companyId: scope.companyId, issueId: row.issueId as string, + }); } export async function notifyRetentionWork(transaction: object, kind: AttentionSourceKind, scope: { companyId?: string; sourceId?: string } = {}) { From 87ae20ae650de687ea9a2f8ca72e5a8d89050431 Mon Sep 17 00:00:00 2001 From: Devin Foley Date: Fri, 30 Oct 2026 17:04:04 -0700 Subject: [PATCH 60/74] fix(server): reconcile failed sandbox cleanup with signed absence (#18407) Reconcile failed ephemeral Cloud leases through a signed cleanup-only absence observation when local historical inspection is unavailable. Preserve failure outcomes, exact resource bindings, retention and row-version fences; grant no run-continuation authority. All 54 current-head checks/statuses passed. Greptile is 5/5, Medium, with no findings or unresolved threads. The 38 focused tests pass; both new reconciliation cases fail without the fallback. Isolated baseline and branch idle suites each pass all 225 cases. The original broader local teardown timeout is retained in the PR, and the full local run remains in progress. Cloud protocol support is already serving in staging. Exact candidate composition and hosted provider sleep/wake qualification follow this merge. Production has not been changed. Co-Authored-By: Paperclip --- doc/idle-sleep-safety.md | 11 +++ maps/agent-access-map/instance-operations.md | 8 +++ maps/feature-map/instance-operations.md | 9 +++ .../historical-cleanup-verification.test.ts | 54 ++++++++++++++- server/src/services/environment-runtime.ts | 8 ++- .../services/sandbox-absence-proof.test.ts | 52 ++++++++++++-- server/src/services/sandbox-absence-proof.ts | 68 +++++++++++++++---- 7 files changed, 190 insertions(+), 20 deletions(-) diff --git a/doc/idle-sleep-safety.md b/doc/idle-sleep-safety.md index e535edcd569..7b8ef115430 100644 --- a/doc/idle-sleep-safety.md +++ b/doc/idle-sleep-safety.md @@ -101,6 +101,17 @@ The manager permits at most two outstanding inspection RPCs across workers. Caller timeouts retain their slots until the actual worker reply or exit; timeouts cannot accumulate additional provider reads in later sweeps. +If a failed ephemeral lease has no usable local inspection path, a managed +Daytona instance can ask Cloud for a cleanup-only absence receipt. Cloud checks +the stack's historical credential organization and reads the exact allocation +with its manager credential. Only confirmed absence can produce a signed receipt +bound to the stack, company, environment, lease, provider handle and nullable run. +The separate cleanup type and audience cannot authorize run recovery. Older +Cloud servers, unsupported providers, invalid signatures, changed organizations +and failed reads leave the lease unresolved. Existing bounded cleanup batches +and retry deadlines own these reads; no new idle polling is added. An explicit +local provider observation is retained rather than replaced by this fallback. + Host-owned verification metadata binds the company, environment, resource, provider handle, optional run, and exact row version. Concurrent resume, replacement, cleanup, or configuration changes invalidate verification. diff --git a/maps/agent-access-map/instance-operations.md b/maps/agent-access-map/instance-operations.md index 3e21ef72901..f2d4250a4cb 100644 --- a/maps/agent-access-map/instance-operations.md +++ b/maps/agent-access-map/instance-operations.md @@ -62,6 +62,14 @@ authority as company-agent API access. Gaps: Instance start/stop and task workspace start/stop are different capabilities. Local service control is not agent company authority. +Historical Cloud sandbox cleanup remains server-owned. Cleanup-only absence +reads authenticate the tenant host with its server capability; they expose no +new board or agent endpoint. Their signed scope cannot authorize task recovery. +[Runtime and idle-gate regressions](../../server/src/__tests__/historical-cleanup-verification.test.ts) +cover reconciliation, while [receipt checks](../../server/src/services/sandbox-absence-proof.test.ts) +cover signature, identity, freshness and separation from recovery authority. +These checks do not establish a deployed sleep/wake result. + ## `configuration` Capability ID: `instance-operations/configuration`. diff --git a/maps/feature-map/instance-operations.md b/maps/feature-map/instance-operations.md index 7a01c24ebf6..dd43901323e 100644 --- a/maps/feature-map/instance-operations.md +++ b/maps/feature-map/instance-operations.md @@ -61,6 +61,15 @@ served by the admitted application before it finishes. Repeat with a disconnect before headers arrive. Confirm that neither leaves a permanent request hold. An unfinished application write must still prevent sleep after its client leaves. +Automated: [historical cleanup](../../server/src/__tests__/historical-cleanup-verification.test.ts) +exercises the runtime inspection path for failed Cloud sandbox leases, including +verification leases without a run. A signed cleanup-only absence observation +can remove this blocker without changing failed outcomes or granting run +continuation. An unresolved provider resource still prevents idle sleep. +Manual: on an owned Cloud staging workspace, confirm the exact published build, +reconcile failed test allocations, then verify actual provider suspension and +normal hosted entry after wake. Local tests do not prove that hosted journey. + ### `settings-backup` Cloud startup reads nonsecret application choices from a private, versioned diff --git a/server/src/__tests__/historical-cleanup-verification.test.ts b/server/src/__tests__/historical-cleanup-verification.test.ts index 37bc2dd6692..1269c02f1a2 100644 --- a/server/src/__tests__/historical-cleanup-verification.test.ts +++ b/server/src/__tests__/historical-cleanup-verification.test.ts @@ -1,8 +1,9 @@ -import { randomUUID } from "node:crypto"; +import { generateKeyPairSync, randomUUID, sign } from "node:crypto"; import { eq, sql } from "drizzle-orm"; import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest"; import { agents, companies, createDb, environmentCustomImageSetupSessions, environmentLeases, environments, heartbeatRuns } from "@paperclipai/db"; import { historicalCleanupVerificationService } from "../services/historical-cleanup-verification.js"; +import { environmentRuntimeService } from "../services/environment-runtime.js"; import { withoutCleanupVerification } from "../services/cleanup-verification-proof.js"; import { idleWorkSnapshot } from "../services/task-admission.js"; import { readIdleSleepSafety, type IdleSleepDrainStatus } from "../services/idle-sleep-safety.js"; @@ -16,7 +17,7 @@ if (!support.supported) console.warn(`Skipping historical cleanup database tests (support.supported ? describe : describe.skip)("historical cleanup verification", () => { let db: ReturnType, database: Awaited>; beforeAll(async () => { database = await startEmbeddedPostgresTestDatabase("historical-cleanup-"); db = createDb(database.connectionString); }, 60_000); - afterEach(async () => { await db.execute(sql`SET client_min_messages = warning; TRUNCATE companies, plugins, environments CASCADE`); }); + afterEach(async () => { vi.unstubAllEnvs(); vi.unstubAllGlobals(); await db.execute(sql`SET client_min_messages = warning; TRUNCATE companies, plugins, environments CASCADE`); }); afterAll(async () => { await db?.$client.end(); await database?.cleanup(); }); const read = () => readIdleSleepSafety(db, owned, () => now, ownerId, async () => "none", undefined, undefined, true, true); const state = async () => (await read()).backgroundWork; @@ -37,6 +38,55 @@ if (!support.supported) console.warn(`Skipping historical cleanup database tests inspectSetup: async row => ({ providerLeaseId: row.providerLeaseId, state: setupState }), }); } + it.each([false, true])("reconciles a legacy Cloud lease through signed absence and permits idle sleep (ad-hoc %s)", async adHoc => { + const f = await seed(adHoc); + const { privateKey, publicKey } = generateKeyPairSync("ed25519"); + for (const [key, value] of Object.entries({ PAPERCLIP_CLOUD_STACK_ID: "fixture-stack", + PAPERCLIP_CLOUD_API_ORIGIN: "https://cloud.example.test", PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN: "fleet-fixture", + PAPERCLIP_PRIMARY_HOST: "fixture.paperclip.test", PAPERCLIP_AUTH_SECRET: "fixture-secret", + PAPERCLIP_CLOUD_RUNTIME_IDENTITY_JWKS: JSON.stringify({ keys: [{ ...publicKey.export({ format: "jwk" }), kid: "fixture", alg: "EdDSA", use: "sig" }] }) })) vi.stubEnv(key, value); + let absent = false; + const fetchImpl = vi.fn(async (_url: string, init: RequestInit) => { + if (!absent) return Response.json({ absent: false }); + const binding = JSON.parse(init.body as string), seconds = Math.floor(Date.now() / 1000); + expect(binding).toEqual({ purpose: "cleanup", companyId: f.companyId, environmentId: f.environment.id, + leaseId: f.lease.id, heartbeatRunId: adHoc ? null : f.runId, provider: "daytona", providerLeaseId: f.lease.providerLeaseId }); + const header = Buffer.from(JSON.stringify({ alg: "EdDSA", kid: "fixture", typ: "paperclip-sandbox-cleanup-absence+jwt" })).toString("base64url"); + const claims = Buffer.from(JSON.stringify({ v: 1, iss: "paperclip-cloud", aud: "paperclip-sandbox-cleanup-absence/v1", + sub: "fixture-stack", ...binding, absent: true, iat: seconds, exp: seconds + 60 })).toString("base64url"); + return Response.json({ receipt: `${header}.${claims}.${sign(null, Buffer.from(`${header}.${claims}`), privateKey).toString("base64url")}` }); + }); + vi.stubGlobal("fetch", fetchImpl); + const runtime = environmentRuntimeService(db); + const service = historicalCleanupVerificationService(db, { + inspectLease: row => runtime.inspectLeaseResource(row.id), inspectSetup: async () => undefined, + }); + expect(await state()).toBe("present"); + expect(await service.sweep()).toMatchObject({ checked: 1, deferred: 1, verified: 0 }); + expect(await state()).toBe("present"); + absent = true; + expect(await service.sweep()).toMatchObject({ checked: 1, verified: 1, failed: 0 }); + expect(await state()).toBe("none"); + const [current] = await db.select().from(environmentLeases); + expect(current).toMatchObject({ status: "failed", cleanupStatus: "success", heartbeatRunId: adHoc ? null : f.runId }); + expect(current!.metadata).not.toHaveProperty("remoteExecutionTermination"); + expect(await service.sweep()).toMatchObject({ checked: 0 }); + expect(fetchImpl).toHaveBeenCalledTimes(2); + }); + it.each(["active", "stopped", "unknown"] as const)("does not replace an explicit %s provider observation with a Cloud read", async providerState => { + await seed(true); + for (const [key, value] of Object.entries({ PAPERCLIP_CLOUD_STACK_ID: "fixture-stack", + PAPERCLIP_CLOUD_API_ORIGIN: "https://cloud.example.test", PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN: "fleet-fixture", + PAPERCLIP_PRIMARY_HOST: "fixture.paperclip.test", PAPERCLIP_AUTH_SECRET: "fixture-secret" })) vi.stubEnv(key, value); + const fetchImpl = vi.fn(); vi.stubGlobal("fetch", fetchImpl); + const runtime = environmentRuntimeService(db, { drivers: [{ driver: "sandbox", + acquireRunLease: async () => { throw new Error("unexpected acquire"); }, releaseRunLease: async () => null, + inspectLeaseResource: async ({ lease }) => ({ providerLeaseId: lease.providerLeaseId!, state: providerState }), + }] }); + const service = historicalCleanupVerificationService(db, { inspectLease: row => runtime.inspectLeaseResource(row.id), inspectSetup: async () => undefined }); + expect(await service.sweep()).toMatchObject({ checked: 1, verified: 0, deferred: 1 }); + expect(await state()).toBe("present"); expect(fetchImpl).not.toHaveBeenCalled(); + }); it.each([false, true])("accepts exact deletion proof without changing outcomes or granting run authority (ad-hoc %s)", async adHoc => { const f = await seed(adHoc); expect(await state()).toBe("present"); const service = verifier(); expect(await service.sweep()).toMatchObject({ checked: 1, verified: 1, failed: 0 }); diff --git a/server/src/services/environment-runtime.ts b/server/src/services/environment-runtime.ts index 277a5a0254c..33f161457f9 100644 --- a/server/src/services/environment-runtime.ts +++ b/server/src/services/environment-runtime.ts @@ -15,6 +15,7 @@ import { hasStopOnlyCleanup, prepareSandboxStopAndRetain, readStopOnlyCleanup, s import { JsonRpcCallError, readEnvironmentAcquisitionDiagnostic, readEnvironmentCreationCleanupError } from "@paperclipai/plugin-sdk"; import { preserveEnvironmentSyncOutErrorDiagnostic } from "./environment-sync-out-error.js"; import { remoteTerminationReceipt } from "./remote-execution-termination.js"; +import { inspectCloudSandboxCleanupAbsence } from "./sandbox-absence-proof.js"; import { hasNativeWorkspaceExportResume, releaseCompletedNativeWorkspaceExportRetention } from "./native-runtime/native-workspace-export-resume.js"; import { createHash, randomUUID } from "node:crypto"; import { and, eq, inArray, ne, or, sql } from "drizzle-orm"; @@ -4021,7 +4022,12 @@ export function environmentRuntimeService( const lease = await environmentsSvc.getLeaseById(leaseId); if (!lease || lease.status !== "failed" || lease.leasePolicy !== "ephemeral" || !lease.providerLeaseId) return undefined; const environment = lease.environmentId ? await environmentsSvc.getById(lease.environmentId) : null; - return await getDriver(getLeaseDriverKey(lease, environment))?.inspectLeaseResource?.({ environment, lease }); + // Legacy Cloud leases can lack a credential version that permits a + // provider-local observation. Cloud's manager verifies the historical + // organization and signs only exact absence; no replacement key or + // cleanup label is treated as proof. Explicit provider answers win. + return await getDriver(getLeaseDriverKey(lease, environment))?.inspectLeaseResource?.({ environment, lease }) + ?? await inspectCloudSandboxCleanupAbsence(lease); }, async retryPendingSandboxTeardown(input: { diff --git a/server/src/services/sandbox-absence-proof.test.ts b/server/src/services/sandbox-absence-proof.test.ts index d145425137e..82a7a3cfbba 100644 --- a/server/src/services/sandbox-absence-proof.test.ts +++ b/server/src/services/sandbox-absence-proof.test.ts @@ -1,6 +1,7 @@ import { createHmac, generateKeyPairSync, randomUUID, sign } from "node:crypto"; import { describe, expect, it, vi } from "vitest"; -import { requestSandboxAbsenceProof, verifySandboxAbsenceReceipt } from "./sandbox-absence-proof.js"; +import { inspectCloudSandboxCleanupAbsence, requestSandboxAbsenceProof, verifySandboxAbsenceReceipt, + verifySandboxCleanupAbsenceReceipt } from "./sandbox-absence-proof.js"; const { privateKey, publicKey } = generateKeyPairSync("ed25519"); const binding = { companyId: randomUUID(), issueId: randomUUID(), runId: randomUUID(), leaseId: randomUUID(), @@ -9,10 +10,10 @@ const env = { PAPERCLIP_CLOUD_STACK_ID: "fixture-stack", PAPERCLIP_CLOUD_API_ORI PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN: "fleet-token", PAPERCLIP_PRIMARY_HOST: "fixture.paperclip.test", PAPERCLIP_AUTH_SECRET: "fixture-secret", PAPERCLIP_CLOUD_RUNTIME_IDENTITY_JWKS: JSON.stringify({ keys: [{ ...publicKey.export({ format: "jwk" }), kid: "fixture", alg: "EdDSA", use: "sig" }] }) }; const now = new Date("2099-01-01T00:00:00Z"), seconds = now.getTime() / 1000; -function receipt(overrides: Record = {}) { - const header = Buffer.from(JSON.stringify({ alg: "EdDSA", kid: "fixture", typ: "paperclip-sandbox-absence+jwt" })).toString("base64url"); - const payload = Buffer.from(JSON.stringify({ v: 1, iss: "paperclip-cloud", aud: "paperclip-sandbox-absence/v1", sub: "fixture-stack", - ...binding, absent: true, iat: seconds, exp: seconds + 60, ...overrides })).toString("base64url"); +function receipt(overrides: Record = {}, purpose = "sandbox-absence", source: Record = binding) { + const header = Buffer.from(JSON.stringify({ alg: "EdDSA", kid: "fixture", typ: `paperclip-${purpose}+jwt` })).toString("base64url"); + const payload = Buffer.from(JSON.stringify({ v: 1, iss: "paperclip-cloud", aud: `paperclip-${purpose}/v1`, sub: "fixture-stack", + ...source, absent: true, iat: seconds, exp: seconds + 60, ...overrides })).toString("base64url"); return `${header}.${payload}.${sign(null, Buffer.from(`${header}.${payload}`), privateKey).toString("base64url")}`; } describe("sandbox absence proof", () => { @@ -45,3 +46,44 @@ describe("sandbox absence proof", () => { expect(fetchImpl).not.toHaveBeenCalled(); }); }); + +describe("cleanup-only sandbox absence", () => { + const cleanup = { purpose: "cleanup" as const, companyId: binding.companyId, environmentId: randomUUID(), + leaseId: binding.leaseId, heartbeatRunId: null as string | null, provider: binding.provider, providerLeaseId: binding.providerLeaseId }; + const lease = { id: cleanup.leaseId, ...cleanup }; + const signed = (overrides: Record = {}) => receipt(overrides, "sandbox-cleanup-absence", cleanup); + it.each([null, binding.runId])("accepts exact absence with nullable run %s without issuing a run receipt", async heartbeatRunId => { + const source = { ...cleanup, heartbeatRunId }; + const fetchImpl = vi.fn().mockResolvedValue(Response.json({ receipt: receipt({}, "sandbox-cleanup-absence", source) })); + expect(await inspectCloudSandboxCleanupAbsence({ ...lease, heartbeatRunId }, { env, now, fetchImpl })) + .toEqual({ providerLeaseId: cleanup.providerLeaseId, state: "destroyed" }); + const [url, init] = fetchImpl.mock.calls[0]; + expect(url).toBe("https://cloud.example.test/v1/tenant/sandbox-absence"); + expect(JSON.parse(init.body)).toEqual(source); + expect(init.headers.authorization).not.toContain(env.PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN); + expect(init.redirect).toBe("error"); + }); + it("requires every binding, current stack, signature and freshness, and rejects recovery receipts", () => { + expect(verifySandboxCleanupAbsenceReceipt(signed(), cleanup, env, now)).toMatchObject(cleanup); + for (const overrides of [ + ...Object.keys(cleanup).map(key => ({ [key]: randomUUID() })), + { sub: "foreign-stack" }, { absent: false }, { aud: "paperclip-sandbox-absence/v1" }, { iss: "other" }, + { iat: seconds + 31 }, { exp: seconds }, { exp: seconds + 61 }, + ]) expect(verifySandboxCleanupAbsenceReceipt(signed(overrides), cleanup, env, now)).toBeNull(); + expect(verifySandboxCleanupAbsenceReceipt(signed().slice(0, -8) + "tampered", cleanup, env, now)).toBeNull(); + expect(verifySandboxCleanupAbsenceReceipt(receipt(), cleanup, env, now)).toBeNull(); + expect(verifySandboxAbsenceReceipt(signed(), binding, env, now)).toBeNull(); + }); + it("keeps old Clouds, outages, invalid replies and unconfigured instances inconclusive", async () => { + for (const answer of [{ absent: false }, { absent: true }, { receipt: "forged" }, { receipt: signed({ heartbeatRunId: binding.runId }) }]) + expect(await inspectCloudSandboxCleanupAbsence(lease, { env, now, fetchImpl: vi.fn().mockResolvedValue(Response.json(answer)) })).toBeUndefined(); + for (const status of [400, 401, 403, 503]) expect(await inspectCloudSandboxCleanupAbsence(lease, + { env, now, fetchImpl: vi.fn().mockResolvedValue(new Response(null, { status })) })).toBeUndefined(); + expect(await inspectCloudSandboxCleanupAbsence(lease, { env, now, fetchImpl: vi.fn().mockRejectedValue(new Error("offline")) })).toBeUndefined(); + const fetchImpl = vi.fn(); + expect(await inspectCloudSandboxCleanupAbsence(lease, { env: {}, now, fetchImpl })).toBeUndefined(); + for (const change of [{ provider: "other" }, { environmentId: null }, { providerLeaseId: "invalid" }]) + expect(await inspectCloudSandboxCleanupAbsence({ ...lease, ...change }, { env, now, fetchImpl })).toBeUndefined(); + expect(fetchImpl).not.toHaveBeenCalled(); + }); +}); diff --git a/server/src/services/sandbox-absence-proof.ts b/server/src/services/sandbox-absence-proof.ts index 8ae852688ed..ae6044dd0af 100644 --- a/server/src/services/sandbox-absence-proof.ts +++ b/server/src/services/sandbox-absence-proof.ts @@ -7,8 +7,18 @@ const bindingSchema = z.object({ companyId: z.string().uuid(), issueId: z.string().uuid(), runId: z.string().uuid(), leaseId: z.string().uuid(), ownerToken: z.string().uuid(), provider: z.literal("daytona"), providerLeaseId: z.string().uuid(), -}); +}).strict(); +const cleanupBindingSchema = z.object({ + purpose: z.literal("cleanup"), companyId: z.string().uuid(), environmentId: z.string().uuid(), + leaseId: z.string().uuid(), heartbeatRunId: z.string().uuid().nullable(), + provider: z.literal("daytona"), providerLeaseId: z.string().uuid(), +}).strict(); export type SandboxAbsenceBinding = z.infer; +export type SandboxCleanupAbsenceBinding = z.infer; +export type SandboxCleanupAbsenceProof = SandboxCleanupAbsenceBinding & { + v: 1; iss: "paperclip-cloud"; aud: "paperclip-sandbox-cleanup-absence/v1"; sub: string; + absent: true; iat: number; exp: number; +}; export type SandboxAbsenceProof = SandboxAbsenceBinding & { v: 1; iss: "paperclip-cloud"; aud: "paperclip-sandbox-absence/v1"; sub: string; absent: true; iat: number; exp: number; @@ -16,19 +26,30 @@ export type SandboxAbsenceProof = SandboxAbsenceBinding & { export function verifySandboxAbsenceReceipt(receipt: string, binding: SandboxAbsenceBinding, env: NodeJS.ProcessEnv = process.env, now = new Date()): SandboxAbsenceProof | null { + const parsed = bindingSchema.safeParse(binding); + return parsed.success ? verifyReceipt(receipt, parsed.data, "sandbox-absence", env, now) as SandboxAbsenceProof | null : null; +} + +export function verifySandboxCleanupAbsenceReceipt(receipt: string, binding: SandboxCleanupAbsenceBinding, + env: NodeJS.ProcessEnv = process.env, now = new Date()): SandboxCleanupAbsenceProof | null { + const parsed = cleanupBindingSchema.safeParse(binding); + return parsed.success ? verifyReceipt(receipt, parsed.data, "sandbox-cleanup-absence", env, now) as SandboxCleanupAbsenceProof | null : null; +} + +function verifyReceipt(receipt: string, binding: Record, purpose: string, + env: NodeJS.ProcessEnv, now: Date): Record | null { try { - const parsed = bindingSchema.safeParse(binding); - if (receipt.length > 16_000 || !parsed.success) return null; + if (receipt.length > 16_000) return null; const parts = receipt.split("."); if (parts.length !== 3 || parts.some(part => !/^[A-Za-z0-9_-]+$/.test(part))) return null; const header = JSON.parse(Buffer.from(parts[0], "base64url").toString("utf8")); - if (header.alg !== "EdDSA" || header.typ !== "paperclip-sandbox-absence+jwt" || typeof header.kid !== "string") return null; + if (header.alg !== "EdDSA" || header.typ !== `paperclip-${purpose}+jwt` || typeof header.kid !== "string") return null; if (!verify(null, Buffer.from(`${parts[0]}.${parts[1]}`, "ascii"), cloudRuntimePublicKeyForKid(env, header.kid), Buffer.from(parts[2], "base64url"))) return null; - const proof = JSON.parse(Buffer.from(parts[1], "base64url").toString("utf8")) as SandboxAbsenceProof; + const proof = JSON.parse(Buffer.from(parts[1], "base64url").toString("utf8")); const seconds = Math.floor(now.getTime() / 1000); - if (proof.v !== 1 || proof.iss !== "paperclip-cloud" || proof.aud !== "paperclip-sandbox-absence/v1" + if (proof.v !== 1 || proof.iss !== "paperclip-cloud" || proof.aud !== `paperclip-${purpose}/v1` || !env.PAPERCLIP_CLOUD_STACK_ID || proof.sub !== env.PAPERCLIP_CLOUD_STACK_ID || proof.absent !== true - || Object.keys(parsed.data).some(key => proof[key as keyof SandboxAbsenceBinding] !== parsed.data[key as keyof SandboxAbsenceBinding]) + || Object.keys(binding).some(key => proof[key] !== binding[key]) || !Number.isInteger(proof.iat) || !Number.isInteger(proof.exp) || proof.iat > seconds + 30 || proof.exp <= seconds || proof.exp <= proof.iat || proof.exp - proof.iat > 60) return null; return proof; @@ -38,11 +59,35 @@ export function verifySandboxAbsenceReceipt(receipt: string, binding: SandboxAbs export async function requestSandboxAbsenceProof(binding: SandboxAbsenceBinding, options: { env?: NodeJS.ProcessEnv; fetchImpl?: typeof fetch; now?: Date; } = {}): Promise<{ proof: SandboxAbsenceProof; receipt: string } | null> { + const parsed = bindingSchema.safeParse(binding); + if (!parsed.success) return null; + const receipt = await requestReceipt(parsed.data, options); + if (!receipt) return null; + const proof = verifySandboxAbsenceReceipt(receipt, parsed.data, options.env, options.now); + return proof ? { proof, receipt } : null; +} + +/** Cleanup-only observation: never grants continuation authority to a run. */ +export async function inspectCloudSandboxCleanupAbsence(lease: { + id: string; companyId: string; environmentId: string | null; heartbeatRunId: string | null; + provider: string | null; providerLeaseId: string | null; +}, options: { env?: NodeJS.ProcessEnv; fetchImpl?: typeof fetch; now?: Date } = {}) { + const parsed = cleanupBindingSchema.safeParse({ purpose: "cleanup", companyId: lease.companyId, + environmentId: lease.environmentId, leaseId: lease.id, heartbeatRunId: lease.heartbeatRunId, + provider: lease.provider, providerLeaseId: lease.providerLeaseId }); + if (!parsed.success) return undefined; + const receipt = await requestReceipt(parsed.data, options); + if (!receipt || !verifySandboxCleanupAbsenceReceipt(receipt, parsed.data, options.env, options.now)) return undefined; + return { providerLeaseId: parsed.data.providerLeaseId, state: "destroyed" as const }; +} + +async function requestReceipt(binding: Record, options: { + env?: NodeJS.ProcessEnv; fetchImpl?: typeof fetch; +}): Promise { const env = options.env ?? process.env; const context = getCloudStackContext(env); const secret = env.PAPERCLIP_AUTH_SECRET ?? env.BETTER_AUTH_SECRET; - const parsed = bindingSchema.safeParse(binding); - if (!context?.stackId || !context.primaryHost || !context.cloudOrigin || !secret?.trim() || !parsed.success) return null; + if (!context?.stackId || !context.primaryHost || !context.cloudOrigin || !secret?.trim()) return null; try { const origin = new URL(context.cloudOrigin); const host = new URL(`https://${context.primaryHost}`); @@ -51,12 +96,11 @@ export async function requestSandboxAbsenceProof(binding: SandboxAbsenceBinding, const response = await (options.fetchImpl ?? fetch)(`${origin.origin}/v1/tenant/sandbox-absence`, { method: "POST", redirect: "error", signal: AbortSignal.timeout(10_000), headers: { authorization: `Bearer ${capability}`, "x-paperclip-cloud-tenant-host": host.hostname, "content-type": "application/json" }, - body: JSON.stringify(parsed.data), + body: JSON.stringify(binding), }); if (!response.ok) return null; const payload = await response.json() as { receipt?: unknown }; if (typeof payload.receipt !== "string") return null; - const proof = verifySandboxAbsenceReceipt(payload.receipt, binding, env, options.now); - return proof ? { proof, receipt: payload.receipt } : null; + return payload.receipt; } catch { return null; } } From 493e229d93ba7798e80152f4549e2e9177ce50ae Mon Sep 17 00:00:00 2001 From: Nicky Leach Date: Fri, 30 Oct 2026 17:07:32 -0700 Subject: [PATCH 61/74] perf(runner): cache verified native executable digests (#18390) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Thinking Path > - Paperclip manages AI agents and their task execution. > - Runner checks qualified native programs before it starts a provider. > - Installation checks and command opening can hash the same program again. > - A new sidecar process repeats this work on a later warm task. > - This pull request stores the computed digest in a private filesystem cache. > - Every cache hit still checks the independently selected trusted digest and current file metadata. > - An unchanged program can start without another full native file read. ## Linked Issues or Issue Description **What existing behavior does this improve?** Repeated **qualified ACPX** native executable admission in Runner. Claude, ACPX-Codex, and Grok use this verifier in both Core-owned and plugin-backed launches. The ordinary native Codex app-server launcher is a different path and is not changed. A retained provider that opens no new native command has no saved hash work here. The measured Claude task path does not require a comment backlog. **Current behavior** Installation verification and final command opening each read and hash the native executable. The qualified Linux x64 Claude file is 241,667,256 bytes. A new verification and command lease reads 483,334,512 native bytes. Another Node process repeats both reads. **Proposed behavior** Store the actual computed SHA-256 digest by device, inode, size, nanosecond modification time, and nanosecond change time. Reuse it across Runner and sidecar processes. Compare each hit with the current build-owned digest. Hash the file on a miss or changed identity. This replaces the earlier Claude-only deferral design in this same PR. **Reason and benefit** The first verification and command lease reads the native file once. Later processes read none of its native bytes while its identity stays the same. Related: #16372 established qualified executable admission. This change is independent of other preparation PRs. It does not change provider pins or profile versions. ## What Changed - Add a shared filesystem digest cache under the absolute OS temporary root at `paperclip-runner-/executable-digests-v1`. - Use private directories and files, versioned hash-only filenames, bounded JSON entries, and atomic temporary-file replacement. Cache files contain a version and the computed digest. They contain no credentials or source paths. - Fall back to full hashing when the cache is missing, unreadable, malformed, or unusable. Keep current expected-digest selection outside the cache. - Keep cache data outside task homes. Neither `HOME` nor task cwd selects the path. The OS temporary-root selection honors absolute platform `TMPDIR`, `TMP`, or `TEMP` overrides. A relative root disables caching. Hosts must keep that root outside task workspaces. Temp cleanup causes a later rehash. - Keep the no-follow executable open, regular-file and size checks, execute permission, descriptor ownership, before/after identity checks, and preparation-to-launch identity comparison. Cache publication occurs before the final identity checks. - Use this path for standalone installation checks and final command opening. Remove the earlier pending-digest type and Claude-only launch helper. Runtime host and profile selection keep their original structure. - Test changed content, all fingerprint fields, symlinks, mode changes, inode replacement, mutation during lookup/publication, concurrent cache writers, wrong trusted pins, and cache failures. - Include a deliberate trust-boundary test: a process running as the same OS user can plant a matching cached digest for changed bytes. It opens a test descriptor and executes nothing. In plain language: Runner remembers the hash for the same file. It still checks which file it opened and which hash the build requires. The cache trusts the OS user who can write it. ## Verification From `packages/paperclip-runner`: - `../../node_modules/.bin/vitest run src/drivers/acpx/executable-digest-cache.test.ts src/drivers/acpx/installation-integrity.test.ts src/drivers/acpx/runtime-host.test.ts` — 177 passed, 6 platform-specific skips. - `../../node_modules/.bin/tsc -p tsconfig.json --noEmit` — passed. - `git diff --check` and the added-line secret, PII, private-address, and code-comment scan — passed. The offline comparison uses baseline `45a3a92e6dc52d20923b9aa34a22fee2ec5443f9` and treatment `c4356df12032448de90516ae329d8f939af10984`. Each sample starts a different Node process. Each normal sample runs the actual qualified installation verifier and opens a command lease. Seven samples per case follow one warmup. The cache is removed before each cold case. The next process reuses its entry. Baseline order alternates around the treatment pair. Command digest and package paths match in every pair. | Verifier plus command opening | Baseline | Cold cache | Later process hit | | --- | ---: | ---: | ---: | | Full native reads | 2 | 1 | 0 | | Native bytes read | 483,334,512 | 241,667,256 | 0 | | Local median, ms | 1041.26 | 602.70 | 130.17 | | Local range, ms | 994.20–1133.11 | 580.27–999.83 | 121.96–179.45 | | Median whole child process, ms | 1379.96 | 988.75 | 523.01 | Raw verifier-plus-open samples in ms: - Baseline: 1133.11, 1074.40, 1061.99, 1041.26, 994.20, 1036.12, 1037.70. - Cold: 647.13, 646.33, 999.83, 584.83, 580.27, 583.37, 602.70. - Later process hit: 179.45, 138.37, 134.39, 130.17, 121.96, 122.33, 126.68. Separate direct-opener fault probes use a disposable copy of that same native file: | Change | Native bytes read | Result | Local open, ms | | --- | ---: | --- | ---: | | First copied file | 241,667,256 | Accepted | 526.14 | | Next process, different HOME and task cwd | 0 | Accepted | 24.91 | | Changed timestamps | 241,667,256 | Accepted after hashing | 531.40 | | New inode, identical bytes | 241,667,256 | Accepted after hashing | 536.70 | | Corrupt byte, same size and exactly restored mtime | 241,667,256 | Rejected | Whole process: 841.36 | | Different independent trusted pin | 0 | Rejected | Whole process: 374.76 | The corruption probe keeps `mtimeNs` exactly equal before and after, at `1000000000000000000`. Its changed `ctimeNs` invalidates the entry. Cache expiry is not required. The changed-pin probe rejects the old actual digest without a native read. The cached image is `sha256:99ce80690a0c13328f41fd84156cc73e1ac562eaed3120fd41d04d7acdfb325d`. The Node binary is `/opt/paperclip/runner/libexec/node`, version `v24.20.0`. The artifact is Claude SDK `0.3.286`, with expected digest `sha256:fe503f65c6289d59c23e5b21ae44f03583f997dd33a2cbfc75ab4f96fb8fc73f`. The container used a read-only root, a disposable writable tmpfs, and disabled network. No provider process, model request, credential, or deployment was used. **These elapsed values are diagnostics from Linux x64 CPU emulation on an ARM64 Docker host. They are not production latency or TTFT estimates.** The exact byte counts, cross-process reuse, output equality, and invalidation results are the portable evidence. The saved phase 0 exports have no directly comparable native executable hashing span. Their broad `runner.transport.attach` spans include other work. A production comparison within 20% is therefore unavailable for this seam. The earlier local 919 ms control was also an emulation diagnostic; it is not a production baseline. The initial Docker attempt was denied access to the local socket by the sandbox. The same offline command then ran with local socket permission. An initial fault probe restored mtime through a `Date`, which did not prove exact nanosecond equality. Its raw report is retained as historical; the final probe sets and verifies an exact mtime before and after corruption. The prior Claude-only implementation and its checks and measurements remain historical evidence. They do not qualify this replacement. The superseded home-directory cache head had one browser recovery test timeout in its GitHub workflow. It expected a saved title after session recovery. The failure is retained. No browser code was changed; the current head reruns CI after moving the cache outside task homes. No full local CI suite was run, as requested. GitHub CI supplies full regression coverage. Current head `c4356df12032448de90516ae329d8f939af10984` passed the complete [GitHub workflow](https://github.com/paperclipai/paperclip/actions/runs/41145223331), including all browser shards. Its latest checks have 56 successes and 2 intended skips, including both security scans started when the PR became ready. Standard Greptile reviewed this exact head at 5/5 with no open findings. Apex then reviewed the same head at 5/5 under the documented OS-user trust policy. It reported no new findings. There are no unresolved review threads. The PR is ready for review and remains unmerged. ## Risks - This deliberately weakens protection against the same OS user. That user can write the cache and can plant a chosen digest for a file fingerprint. A cache hit trusts that record; metadata identity is not a fresh content hash. The explicit fault test documents this accepted policy. - A read-only administrator-owned Runner can use this user-writable cache. The cache has no signature, secret, or daemon. It does not require a Launcher or ROM change. - Other users cannot use normal cache permissions to write these entries. Cache errors fall back to hashing. Removing the cache directory restores full hashing. Old fingerprints leave small unused entries until the cache is removed or OS temp cleanup evicts it. Temp eviction costs one later full hash. - A changed file normally rehashes. A digest that differs from the independent current pin still fails admission. Existing command, package, credential, process-lifetime, and final dispatch checks remain. - No schema, public protocol, package dependency, profile version, or provider pin changes. ## Model Used OpenAI Codex, GPT-6 family, with reasoning, code editing, test execution, and tool use. This session does not expose a more precise serving model ID or context-window capacity. ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used with the details available to this session - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above - [x] I have described the issue in-PR following the enhancement template - [x] I have not referenced internal or instance-local Paperclip issues or links - [x] My branch name describes the change and contains no internal ticket ID - [x] The targeted local tests pass; the full local suite was not run as requested - [x] I have added or updated tests where applicable - [x] I have updated relevant documentation - [x] I have considered and documented risks - [x] All Paperclip CI gates are green for the replacement head - [x] Standard Greptile is 5/5 with no open findings for the replacement head - [x] Apex is 5/5 on the current head with no unresolved findings - [x] I will address review findings before requesting merge --------- Co-authored-by: Paperclip --- .../paperclip-runner/docs/durable-recovery.md | 29 ++++ .../acpx/executable-digest-cache.test.ts | 160 ++++++++++++++++++ .../drivers/acpx/installation-integrity.ts | 52 +++--- .../src/executable-digest-cache.ts | 70 ++++++++ 4 files changed, 286 insertions(+), 25 deletions(-) create mode 100644 packages/paperclip-runner/src/drivers/acpx/executable-digest-cache.test.ts create mode 100644 packages/paperclip-runner/src/executable-digest-cache.ts diff --git a/packages/paperclip-runner/docs/durable-recovery.md b/packages/paperclip-runner/docs/durable-recovery.md index 0fbef29f508..f04aab08a23 100644 --- a/packages/paperclip-runner/docs/durable-recovery.md +++ b/packages/paperclip-runner/docs/durable-recovery.md @@ -98,6 +98,35 @@ digest, and refuse the rest. A profile identity does not establish hosted harness qualification. [`rules/acpx-profile-identity.mdc`](../../../rules/acpx-profile-identity.mdc) summarizes these practices for contributors. +Qualified ACPX native executables use Runner's persistent digest cache. The cache +stores a computed SHA-256 digest under a key made from device, inode, size, +nanosecond modification time, and nanosecond change time. Each check compares +that digest with the independently selected build-owned pin. A changed file +identity, missing entry, or unusable cache causes a full descriptor read and +hash. A hit reuses the earlier digest; metadata identity is not a fresh content +hash. Claude, Codex, and Grok use the same native executable admission path. + +Cache entries live in the operating-system temporary directory under +`paperclip-runner-/executable-digests-v1`, outside task workspaces such as +`/home/agent`. Neither `HOME` nor the task working directory selects this path. +Node's `os.tmpdir()` selects the root and honors the platform's absolute +`TMPDIR`, `TMP`, or `TEMP` override. Hosts must keep that root outside task +workspaces; a relative or unusable root disables caching. Temporary-directory +eviction removes the optimization until a later full hash fills it again. The +directory and files are private to that user. Writes use a temporary file and +atomic rename, so later Runner or sidecar processes can reuse complete entries. +Cache errors fall back to hashing. The cache stores no credentials or file paths. + +Every admission still opens the executable without following its final symlink, +checks its regular-file type, bounds, execute permission, and before/after file +identity, and retains the descriptor for launch. Package qualification and the +preparation-to-launch identity check remain. The cache deliberately trusts the +operating-system user: a process with that user's access can plant a digest for +a chosen fingerprint. It is not authenticated with a secret or signature. +An administrator-owned read-only Runner can therefore use a user-writable cache, +but this provides weaker protection against that same user than hashing every +executable on each admission. Removing the cache restores full hashing. + ## External controller checkpoints The optional controller persistence callback stores recovery control state, not diff --git a/packages/paperclip-runner/src/drivers/acpx/executable-digest-cache.test.ts b/packages/paperclip-runner/src/drivers/acpx/executable-digest-cache.test.ts new file mode 100644 index 00000000000..f33eb4fd39f --- /dev/null +++ b/packages/paperclip-runner/src/drivers/acpx/executable-digest-cache.test.ts @@ -0,0 +1,160 @@ +import { createHash } from "node:crypto"; +import { chmod, lstat, mkdir, mkdtemp, readFile, readdir, rename, rm, symlink, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { createExecutableDigestCache, executableDigestCache, type ExecutableFileIdentity } from "../../executable-digest-cache.js"; +import { openVerifiedRuntimeExecutable } from "./installation-integrity.js"; + +const temporary: string[] = []; +const identity: ExecutableFileIdentity = { device: "1", inode: "2", size: "3", modifiedNanoseconds: "4", changedNanoseconds: "5" }; +const digest = `sha256:${"a".repeat(64)}`; +afterEach(async () => { + vi.restoreAllMocks(); + await Promise.all(temporary.splice(0).map(path => rm(path, { recursive: true, force: true }))); +}); + +async function fixture() { + const root = await mkdtemp(join(tmpdir(), "runner-executable-digest-cache-")); + temporary.push(root); + const directory = join(root, "cache"); + const cache = createExecutableDigestCache(directory); + return { root, directory, cache }; +} + +describe("persistent executable digest cache", () => { + it("stores the actual digest in a private, versioned entry and survives a new cache instance", async () => { + const f = await fixture(); + expect(await f.cache.read(identity)).toBeNull(); + await f.cache.write(identity, digest); + expect(await createExecutableDigestCache(f.directory).read(identity)).toBe(digest); + const [name] = await readdir(f.directory); + expect(name).toMatch(/^v1-[a-f0-9]{64}\.json$/); + expect(JSON.parse(await readFile(join(f.directory, name), "utf8"))).toEqual({ version: 1, digest }); + expect((await lstat(f.directory)).mode & 0o777).toBe(0o700); + expect((await lstat(join(f.directory, name))).mode & 0o777).toBe(0o600); + }); + + it.each(Object.keys(identity) as (keyof ExecutableFileIdentity)[])("invalidates a changed %s", async key => { + const f = await fixture(); + await f.cache.write(identity, digest); + expect(await f.cache.read({ ...identity, [key]: "changed" })).toBeNull(); + }); + + it.each(["malformed", "version", "digest", "oversized", "permissions", "symlink"] as const)( + "treats an unusable %s entry as a miss", async kind => { + const f = await fixture(); + await f.cache.write(identity, digest); + const [name] = await readdir(f.directory), file = join(f.directory, name); + if (kind === "malformed") await writeFile(file, "{"); + else if (kind === "version") await writeFile(file, JSON.stringify({ version: 2, digest })); + else if (kind === "digest") await writeFile(file, JSON.stringify({ version: 1, digest: "expected" })); + else if (kind === "oversized") await writeFile(file, "x".repeat(1025)); + else if (kind === "permissions") await chmod(file, 0o644); + else { await rename(file, `${file}.target`); await symlink(`${file}.target`, file); } + expect(await f.cache.read(identity)).toBeNull(); + }, + ); + + it("falls back when the cache directory cannot be used and leaves existing permissions unchanged", async () => { + const f = await fixture(); + await writeFile(f.directory, "not a directory"); + await expect(f.cache.write(identity, digest)).resolves.toBeUndefined(); + expect(await f.cache.read(identity)).toBeNull(); + await rm(f.directory); + await mkdir(f.directory, { mode: 0o755 }); + await f.cache.write(identity, digest); + expect(await f.cache.read(identity)).toBeNull(); + expect((await lstat(f.directory)).mode & 0o777).toBe(0o755); + }); + + it("does not resolve a relative cache directory through the task working directory", async () => { + const cache = createExecutableDigestCache("relative-task-cache"); + await cache.write(identity, digest); + expect(await cache.read(identity)).toBeNull(); + }); + + it("publishes complete entries under concurrent writers without leftover temporary files", async () => { + const f = await fixture(); + await Promise.all(Array.from({ length: 8 }, () => f.cache.write(identity, digest))); + expect(await f.cache.read(identity)).toBe(digest); + expect(await readdir(f.directory)).toHaveLength(1); + }); +}); + +describe.skipIf(!["linux", "darwin"].includes(process.platform))("descriptor admission with cached digests", () => { + async function executable() { + const f = await fixture(), path = join(f.root, "native"), bytes = Buffer.from("qualified native fixture"); + await writeFile(path, bytes, { mode: 0o700 }); + vi.spyOn(executableDigestCache, "read").mockImplementation(f.cache.read); + vi.spyOn(executableDigestCache, "write").mockImplementation(f.cache.write); + return { ...f, path, digest: `sha256:${createHash("sha256").update(bytes).digest("hex")}` }; + } + + it("hashes a miss, reuses the actual digest, and still rejects a different independent pin", async () => { + const f = await executable(); + await (await openVerifiedRuntimeExecutable(f.path, f.digest, "fixture")).handle.close(); + expect(executableDigestCache.write).toHaveBeenCalledOnce(); + await (await openVerifiedRuntimeExecutable(f.path, f.digest, "fixture")).handle.close(); + expect(executableDigestCache.write).toHaveBeenCalledOnce(); + await expect(openVerifiedRuntimeExecutable(f.path, digest, "fixture")).rejects.toThrow("digest mismatch"); + }); + + it("rehashes changed contents and rejects corruption despite a previous valid entry", async () => { + const f = await executable(); + await (await openVerifiedRuntimeExecutable(f.path, f.digest, "fixture")).handle.close(); + await writeFile(f.path, "corrupted native fixture"); + await expect(openVerifiedRuntimeExecutable(f.path, f.digest, "fixture")).rejects.toThrow("digest mismatch"); + }); + + it.each(["permissions", "symlink", "replacement"] as const)("retains admission checks after cached %s changes", async kind => { + const f = await executable(); + await (await openVerifiedRuntimeExecutable(f.path, f.digest, "fixture")).handle.close(); + if (kind === "permissions") { + await chmod(f.path, 0o600); + await expect(openVerifiedRuntimeExecutable(f.path, f.digest, "fixture")).rejects.toThrow("bounded executable file"); + } else if (kind === "symlink") { + await rename(f.path, `${f.path}.original`); await symlink(`${f.path}.original`, f.path); + await expect(openVerifiedRuntimeExecutable(f.path, f.digest, "fixture")).rejects.toThrow("real regular file"); + } else { + await rename(f.path, `${f.path}.original`); await writeFile(f.path, "new invalid executable", { mode: 0o700 }); + await expect(openVerifiedRuntimeExecutable(f.path, f.digest, "fixture")).rejects.toThrow("digest mismatch"); + } + }); + + it("rejects mutation while a cache hit is being read", async () => { + const f = await executable(); + await (await openVerifiedRuntimeExecutable(f.path, f.digest, "fixture")).handle.close(); + vi.mocked(executableDigestCache.read).mockImplementationOnce(async () => { + await writeFile(f.path, "mutation during lookup"); + return f.digest; + }); + await expect(openVerifiedRuntimeExecutable(f.path, f.digest, "fixture")).rejects.toThrow("changed while it was verified"); + }); + + it("checks executable identity after publishing a cold cache entry", async () => { + const f = await executable(); + vi.mocked(executableDigestCache.write).mockImplementationOnce(async (identity, digest) => { + await f.cache.write(identity, digest); + await writeFile(f.path, "mutation during cache publication"); + }); + await expect(openVerifiedRuntimeExecutable(f.path, f.digest, "fixture")).rejects.toThrow("changed while it was verified"); + }); + + it("falls back to hashing when the cache is unavailable", async () => { + const f = await executable(); + await writeFile(f.directory, "unavailable cache"); + await (await openVerifiedRuntimeExecutable(f.path, f.digest, "fixture")).handle.close(); + await writeFile(f.path, "invalid native fixture"); + await expect(openVerifiedRuntimeExecutable(f.path, f.digest, "fixture")).rejects.toThrow("digest mismatch"); + }); + + it("trusts a digest planted by the same operating-system user for current metadata", async () => { + const f = await executable(); + await writeFile(f.path, "content changed by the same user"); + const metadata = await lstat(f.path, { bigint: true }); + await f.cache.write({ device: metadata.dev.toString(), inode: metadata.ino.toString(), size: metadata.size.toString(), + modifiedNanoseconds: metadata.mtimeNs.toString(), changedNanoseconds: metadata.ctimeNs.toString() }, f.digest); + await (await openVerifiedRuntimeExecutable(f.path, f.digest, "fixture")).handle.close(); + }); +}); diff --git a/packages/paperclip-runner/src/drivers/acpx/installation-integrity.ts b/packages/paperclip-runner/src/drivers/acpx/installation-integrity.ts index fbf39064e18..1fab0150997 100644 --- a/packages/paperclip-runner/src/drivers/acpx/installation-integrity.ts +++ b/packages/paperclip-runner/src/drivers/acpx/installation-integrity.ts @@ -2,6 +2,7 @@ import { MAX_ACPX_RUNTIME_EXECUTABLE_BYTES, ACPX_PRIVATE_SNAPSHOT_ENV, createAcp import { createNativeAcpxDistributionSnapshot, NATIVE_ACPX_BOOTSTRAP_NAME, readNativeAcpxDistributionEntries, type NativeAcpxDistributionInput } from "./native-distribution-integrity.js"; export type { NativeAcpxDistributionInput } from "./native-distribution-integrity.js"; import { createHash } from "node:crypto"; +import { executableDigestCache, type ExecutableFileIdentity } from "../../executable-digest-cache.js"; import { spawn as spawnChildProcess, type ChildProcess, @@ -606,13 +607,7 @@ export async function awaitVerifiedAcpxProviderExit( await exitProof; } -interface VerifiedAcpxCommandIdentity { - device: string; - inode: string; - size: string; - modifiedNanoseconds: string; - changedNanoseconds: string; -} +type VerifiedAcpxCommandIdentity = ExecutableFileIdentity; interface VerifiedAcpxRuntimeExecutable { path: string; @@ -1204,7 +1199,7 @@ async function inspectCommand( } } -async function openVerifiedRuntimeExecutable( +export async function openVerifiedRuntimeExecutable( executablePath: string, expectedDigest: string, agent: string, @@ -1246,23 +1241,31 @@ async function openVerifiedRuntimeExecutable( `ACPX ${agent} runtime executable must be a bounded executable file`, ); } - const hash = createHash("sha256"); - const buffer = Buffer.alloc(1024 * 1024); + const cachedDigest = await executableDigestCache.read(fileIdentity(before)); + let digest = cachedDigest; let position = 0; - try { - while (position < Number(before.size)) { - const { bytesRead } = await handle.read( - buffer, - 0, - Math.min(buffer.length, Number(before.size) - position), - position, - ); - if (bytesRead === 0) break; - hash.update(buffer.subarray(0, bytesRead)); - position += bytesRead; + if (digest === null) { + const hash = createHash("sha256"); + const buffer = Buffer.alloc(1024 * 1024); + try { + while (position < Number(before.size)) { + const { bytesRead } = await handle.read( + buffer, + 0, + Math.min(buffer.length, Number(before.size) - position), + position, + ); + if (bytesRead === 0) break; + hash.update(buffer.subarray(0, bytesRead)); + position += bytesRead; + } + } finally { + buffer.fill(0); } - } finally { - buffer.fill(0); + digest = `sha256:${hash.digest("hex")}`; + } + if (cachedDigest === null && position === Number(before.size) && digest === expectedDigest) { + await executableDigestCache.write(fileIdentity(before), digest); } const after = await handle.stat({ bigint: true }); const lexicalAfter = await lstat(executablePath, { bigint: true }).catch( @@ -1271,7 +1274,7 @@ async function openVerifiedRuntimeExecutable( const beforeIdentity = fileIdentity(before); const afterIdentity = fileIdentity(after); if ( - position !== Number(before.size) || + (cachedDigest === null && position !== Number(before.size)) || lexicalAfter === null || lexicalAfter.isSymbolicLink() || !lexicalAfter.isFile() || @@ -1283,7 +1286,6 @@ async function openVerifiedRuntimeExecutable( `ACPX ${agent} runtime executable changed while it was verified`, ); } - const digest = `sha256:${hash.digest("hex")}`; if (digest !== expectedDigest) { throw new Error(`ACPX ${agent} runtime executable digest mismatch`); } diff --git a/packages/paperclip-runner/src/executable-digest-cache.ts b/packages/paperclip-runner/src/executable-digest-cache.ts new file mode 100644 index 00000000000..b50b54d9059 --- /dev/null +++ b/packages/paperclip-runner/src/executable-digest-cache.ts @@ -0,0 +1,70 @@ +import { createHash, randomUUID } from "node:crypto"; +import { constants } from "node:fs"; +import { lstat, mkdir, open, rename, unlink } from "node:fs/promises"; +import { tmpdir, userInfo } from "node:os"; +import { isAbsolute, join } from "node:path"; + +export interface ExecutableFileIdentity { + device: string; + inode: string; + size: string; + modifiedNanoseconds: string; + changedNanoseconds: string; +} + +export function createExecutableDigestCache(directory?: string) { + async function location(identity: ExecutableFileIdentity, create: boolean): Promise { + const user = userInfo(); + const temporaryRoot = tmpdir(); + if (!isAbsolute(directory ?? temporaryRoot)) throw new Error("Executable digest cache directory is unavailable"); + const base = join(temporaryRoot, `paperclip-runner-${user.uid}`); + const directories = directory ? [directory] : [base, join(base, "executable-digests-v1")]; + for (const [index, path] of directories.entries()) { + if (create) await mkdir(path, { mode: 0o700 }).catch(error => { + if ((error as NodeJS.ErrnoException).code !== "EEXIST") throw error; + }); + const metadata = await lstat(path); + if (!metadata.isDirectory() || metadata.isSymbolicLink() || metadata.uid !== user.uid || + (metadata.mode & (index === directories.length - 1 ? 0o077 : 0o022)) !== 0) { + throw new Error("Executable digest cache directory is unavailable"); + } + } + const key = createHash("sha256").update(JSON.stringify([1, identity.device, identity.inode, identity.size, + identity.modifiedNanoseconds, identity.changedNanoseconds])).digest("hex"); + return join(directories.at(-1)!, `v1-${key}.json`); + } + + return { + async read(identity: ExecutableFileIdentity): Promise { + try { + const path = await location(identity, false); + const handle = await open(path, constants.O_RDONLY | constants.O_NOFOLLOW); + try { + const metadata = await handle.stat(); + if (!metadata.isFile() || metadata.uid !== userInfo().uid || (metadata.mode & 0o077) !== 0 || + metadata.size < 1 || metadata.size > 1024) return null; + const entry: unknown = JSON.parse(await handle.readFile("utf8")); + if (typeof entry !== "object" || entry === null || !("version" in entry) || entry.version !== 1 || + !("digest" in entry) || typeof entry.digest !== "string" || !/^sha256:[a-f0-9]{64}$/.test(entry.digest)) return null; + return entry.digest; + } finally { await handle.close(); } + } catch { return null; } + }, + async write(identity: ExecutableFileIdentity, digest: string): Promise { + let temporary: string | undefined; + try { + if (!/^sha256:[a-f0-9]{64}$/.test(digest)) return; + const path = await location(identity, true); + temporary = `${path}.${randomUUID()}.tmp`; + const handle = await open(temporary, constants.O_WRONLY | constants.O_CREAT | constants.O_EXCL | constants.O_NOFOLLOW, 0o600); + try { await handle.writeFile(JSON.stringify({ version: 1, digest })); } + finally { await handle.close(); } + await rename(temporary, path); + } catch { } finally { + if (temporary) await unlink(temporary).catch(() => undefined); + } + }, + }; +} + +export const executableDigestCache = createExecutableDigestCache(); From 4ef018cfe65e96b93c4310e15e479fd4425fad8f Mon Sep 17 00:00:00 2001 From: Dotta Date: Fri, 30 Oct 2026 19:17:14 -0500 Subject: [PATCH 62/74] test: model retention reads in native cleanup transaction fixture --- .../native-runtime/native-session-executor.test.ts | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/server/src/services/native-runtime/native-session-executor.test.ts b/server/src/services/native-runtime/native-session-executor.test.ts index 4dabeda6954..6fb2e52c777 100644 --- a/server/src/services/native-runtime/native-session-executor.test.ts +++ b/server/src/services/native-runtime/native-session-executor.test.ts @@ -4302,6 +4302,17 @@ describe("retained native cleanup activation", () => { releaseCommit = release; }); const db = { + execute: async (query: SQL) => { + // Retention hints read the fixture's run/issue relationship inside the + // same cleanup transaction; this fixture has one issue and no ancestors. + const statement = new PgDialect().sqlToQuery(query); + expect(transactionOpen).toBe(true); + expect(statement.params).toContain(run.companyId); + if (statement.sql.includes("from heartbeat_runs")) return [{ id: run.nativeIssueId }]; + if (statement.sql.includes("origin_kind")) return [{ id: run.nativeIssueId, originKind: null, originId: null }]; + if (statement.sql.includes("with recursive affected")) return [{ issueId: run.nativeIssueId }]; + throw new Error(`Unexpected cleanup fixture query: ${statement.sql}`); + }, select: () => ({ from: (table: unknown) => { const rows = From f201195a92a3f6baa6e4411aa52da39f93f07d83 Mon Sep 17 00:00:00 2001 From: Dotta Date: Fri, 30 Oct 2026 19:20:10 -0500 Subject: [PATCH 63/74] test: await committed recovery and adapter admission outcomes --- .../src/__tests__/chat-channels.integration.test.ts | 12 +++++++++--- .../heartbeat-stale-queue-invalidation.test.ts | 8 ++++++++ 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/server/src/__tests__/chat-channels.integration.test.ts b/server/src/__tests__/chat-channels.integration.test.ts index fcaf642aa84..8b10fe2eec7 100644 --- a/server/src/__tests__/chat-channels.integration.test.ts +++ b/server/src/__tests__/chat-channels.integration.test.ts @@ -16341,9 +16341,15 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => { await tx.update(toolConnections).set({ credentialSecretRefs: connection.credentialSecretRefs.map(ref => ({ ...ref, versionSelector: ref.versionSelector === "1" ? "latest" : "1" })) }).where(eq(toolConnections.id, connection.id)); }); - await expect.poll(async () => ((await f.checkpoint()).value as Record).credentialFingerprint, - { timeout: 5000 }).not.toBe(oldWindow.credentialFingerprint); - expect((await f.checkpoint()).value).toMatchObject({ generation: oldWindow.generation, outcome: "epoch_initialized" }); + // Credential synchronization and epoch initialization commit separately. + // Require the final epoch receipt, not just the intermediate fingerprint. + await expect.poll(async () => { + const value = (await f.checkpoint()).value as Record; + return { ...value, fingerprintChanged: typeof value.credentialFingerprint === "string" + && value.credentialFingerprint !== oldWindow.credentialFingerprint }; + }, { timeout: 5000 }).toMatchObject({ + fingerprintChanged: true, generation: oldWindow.generation, outcome: "epoch_initialized", + }); expect(f.requests).toEqual([]); } finally { await coordinator.stop(); } }); diff --git a/server/src/__tests__/heartbeat-stale-queue-invalidation.test.ts b/server/src/__tests__/heartbeat-stale-queue-invalidation.test.ts index 65191c24d2f..c4f0772ad47 100644 --- a/server/src/__tests__/heartbeat-stale-queue-invalidation.test.ts +++ b/server/src/__tests__/heartbeat-stale-queue-invalidation.test.ts @@ -262,6 +262,14 @@ describeEmbeddedPostgres("heartbeat stale queued-run invalidation", () => { await heartbeat.resumeQueuedRuns(); const [delivered] = await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, wake.id)); expect(delivered.runId).toBeTruthy(); + // The receipt can commit before execution enters liveRunExecutions. + // Observe actual adapter admission before awaiting its drain. + await vi.waitFor(async () => { + const current = await heartbeat.getRun(delivered.runId!); + expect(countExecuteCallsForRun(delivered.runId!), JSON.stringify({ + status: current?.status, errorCode: current?.errorCode, + })).toBe(1); + }, { timeout: 5_000 }); await heartbeat.waitForRunExecutionDrain(delivered.runId!); expect(countExecuteCallsForRun(delivered.runId!)).toBe(1); expect((await heartbeat.getRun(delivered.runId!))?.contextSnapshot).toMatchObject({ wakeCommentIds: [comment.id] }); From 71d7b9f09e5a1a0c5c38bbd4084802dfd52f313d Mon Sep 17 00:00:00 2001 From: Dotta Date: Fri, 30 Oct 2026 19:35:20 -0500 Subject: [PATCH 64/74] fix(server): bound blocker facts and preserve origin authority Resolve liveness/stranded references through trusted company issue rows, including valid none sentinels. Bound shared blocker nodes, edges and query results; return private deferred-root completeness so retention immediately performs finite exact source checks. Preserve already-observed live work, completed-root classification and true deadlines. Cover historical producer metadata, moved recovery tasks, many-heavy graph limits, quiet rechecks and faithful transaction fixtures. Co-Authored-By: Paperclip --- .../2026-10-30-attention-retention-sleep.md | 2 +- .../attention-retention-work.test.ts | 126 ++++++++++++++++-- server/src/__tests__/issues-service.test.ts | 2 +- server/src/__tests__/work-products.test.ts | 28 +++- server/src/services/attention-retention.ts | 11 +- .../issue-thread-interactions.test.ts | 19 ++- server/src/services/issues.ts | 120 +++++++++-------- .../services/retention-work-notifications.ts | 19 ++- 8 files changed, 250 insertions(+), 77 deletions(-) diff --git a/doc/plans/2026-10-30-attention-retention-sleep.md b/doc/plans/2026-10-30-attention-retention-sleep.md index 24212b4b98a..aa78f1bb746 100644 --- a/doc/plans/2026-10-30-attention-retention-sleep.md +++ b/doc/plans/2026-10-30-attention-retention-sleep.md @@ -25,7 +25,7 @@ The feed and worker share confirmation collapse, newer-run probes, issue visibil ## Existing mutation boundaries reactivate work -Committed hints identify a company, source kind, source ID or changed issue where those values are known. Routine issue writers pass identity to the retention domain helper before their mutation. The helper captures old and new parent paths, reverse blocker ancestors, run-linked issues and liveness-origin references; issue-scoped source queries then select reviews, blockers, interactions, decisions, recovery actions and failed runs by their issue identity. Budget incidents and agent errors remain tied to their own source changes. Ordinary issue writes do not reconstruct the unrelated company feed. Queue keys explicitly pick supported identity fields. Retention's own writes use exact source hints; blocker retention changes invalidate the company/source because multiple blocked roots can project one terminal subject. Reconstruction pages and due batches remain bounded. Review attention and interaction invokability are computed in company batches per candidate page; a full 100-candidate review page uses at most 20 statements and 100 confirmations use at most six. Decisions, recovery source links, runs, eligible agents, issue visibility, newer-run probes and blocker graphs also load in company batches per page. A 100-source decision or recovery page uses at most three statements; 100 failed runs use at most six; 100 blocker roots use at most 30 graph/source statements plus retention-state reads for their emitted entries. These evidence maps exist only for one load. The blocker owner shares fetched node and edge facts while tracking depth, node count, expanded nodes and live-work reachability independently for each root. An unrelated graph reaching its limit cannot change a shallow root’s terminal subject or deadline. All later graph queries, including recovery source IDs, agent IDs and workspace-finalize inputs, use bounded chunks. A SQL-generated 70,000-node graph proves PostgreSQL parameter limits are respected at the driver boundary. Archive checks still lock and freshly re-read sources, terminal subjects and related eligibility. Ordinary execution, decision and budget mutations pass their affected company; exact decision sources are included. +Committed hints identify a company, source kind, source ID or changed issue where those values are known. Routine issue writers pass identity to the retention domain helper before their mutation. The helper captures old and new parent paths, reverse blocker ancestors, run-linked issues and liveness-origin references; issue-scoped source queries then select reviews, blockers, interactions, decisions, recovery actions and failed runs by their issue identity. Budget incidents and agent errors remain tied to their own source changes. Ordinary issue writes do not reconstruct the unrelated company feed. Opaque liveness and stranded-recovery origin references resolve through company-scoped issue rows before entering UUID queries, so valid none sentinels do not roll back writes and moved recovery tasks still invalidate their original review. Queue keys explicitly pick supported identity fields. Retention's own writes use exact source hints; blocker retention changes invalidate the company/source because multiple blocked roots can project one terminal subject. Reconstruction pages and due batches remain bounded. Review attention and interaction invokability are computed in company batches per candidate page; a full 100-candidate review page uses at most 20 statements and 100 confirmations use at most six. Decisions, recovery source links, runs, eligible agents, issue visibility, newer-run probes and blocker graphs also load in company batches per page. A 100-source decision or recovery page uses at most three statements; 100 failed runs use at most six; 100 blocker roots use at most 30 graph/source statements plus retention-state reads for their emitted entries. These evidence maps exist only for one load. The blocker owner shares fetched node and edge facts while tracking depth, node count, expanded nodes and live-work reachability independently for each root. The shared graph fetch stores at most 2,000 nodes and 4,000 edges, and bounds each frontier query before materializing rows. An internal batch result identifies roots whose evaluation is incomplete; the legacy map API exposes no extra transport fields. Retention schedules only those roots for an immediate fresh single-root check, which disarms or saves the actual source deadline. It never archives a partial intermediary or adds a recurring incomplete-root poll. Fully evaluated roots preserve their exact classification. All later graph queries, including recovery source IDs, agent IDs and workspace-finalize inputs, use bounded chunks. A SQL-generated 70,000-node graph proves PostgreSQL parameter limits are respected at the driver boundary. Archive checks still lock and freshly re-read sources, terminal subjects and related eligibility. Ordinary execution, decision and budget mutations pass their affected company; exact decision sources are included. Producer coverage includes decision creation and changes; interaction creation, resolution, expiry and supersession; issue creation, updates, blocker relations, checkout, adoption and release; agent hire, configuration, lifecycle, error clearing and deletion; budget incidents and company pause/resume; exhausted-run events and retry-run creation; native status, review, finalization and reconstruction; external conversation transitions; and imported blocker relations. Existing service transactions retain outer commit ownership. Savepoint release does not publish work before outer commit. diff --git a/server/src/__tests__/attention-retention-work.test.ts b/server/src/__tests__/attention-retention-work.test.ts index b1e7949a8b7..275b4f64570 100644 --- a/server/src/__tests__/attention-retention-work.test.ts +++ b/server/src/__tests__/attention-retention-work.test.ts @@ -17,7 +17,7 @@ import { buildIssueGraphLivenessIncidentKey, RECOVERY_ORIGIN_KINDS } from "../se import { issueThreadInteractionService } from "../services/issue-thread-interactions.js"; import { createAgentLifecycle } from "../services/agent-lifecycle.js"; import { BLOCKER_ATTENTION_MAX_DEPTH, BLOCKER_ATTENTION_MAX_NODES, issueService } from "../services/issues.js"; -import { notifyAttentionRetentionWork } from "../services/retention-work-notifications.js"; +import { notifyAttentionRetentionWork, notifyIssueAttentionRetentionWork } from "../services/retention-work-notifications.js"; import { idleWorkSnapshot } from "../services/task-admission.js"; import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; @@ -278,12 +278,16 @@ const delay = DEFAULT_DECISION_ARCHIVE_DAYS * 86_400_000; expect(expected.terminalBlockerIssueId).toBe(terminal.id); expect(limited.terminalBlockerIssueId).toBe(limited.directBlockerIssueId); for (const input of [[root, heavy], [heavy, root]]) { - const batch = await owner.listBlockerAttention(companyId, input); - expect(batch.get(root.id)).toEqual(expected); expect(batch.get(heavy.id)).toEqual(limited); + const batch = await owner.listBlockerAttentionBatch(companyId, input); + if (!batch.deferredRootIds.has(root.id)) expect(batch.attention.get(root.id)).toEqual(expected); + if (!batch.deferredRootIds.has(heavy.id)) expect(batch.attention.get(heavy.id)).toEqual(limited); + expect(batch.nodeCount).toBeLessThanOrEqual(BLOCKER_ATTENTION_MAX_NODES); + if (limit === "depth") expect(batch.deferredRootIds.has(root.id)).toBe(false); } const page = await attentionRetentionService(db).load({ companyId, sourceKind: "blocker_attention" }); const entry = page.rows!.find(item => item.rootId === root.id && item.variant === "terminal")!; - expect(entry.sourceId).toBe(terminal.id); expect(entry.dueAt).toBeLessThan(Date.now()); + expect(entry.dueAt).toBeLessThanOrEqual(Date.now()); + if (limit === "depth") expect(entry.sourceId).toBe(terminal.id); await attentionRetentionService(db).check(entry); expect((await state("blocker_attention", terminal.id))?.archivedAt).toBeInstanceOf(Date); expect(await db.select().from(decisionArchiveNotificationOutbox).where(eq(decisionArchiveNotificationOutbox.sourceId, terminal.id))).toHaveLength(1); @@ -317,20 +321,79 @@ const delay = DEFAULT_DECISION_ARCHIVE_DAYS * 86_400_000; await db.execute(sql`insert into issues (id, company_id, title, status, updated_at) select gen_random_uuid(), ${companyId}::uuid, 'parameter-root-' || n, 'blocked', ${old().toISOString()}::timestamptz from generate_series(1, 70) n`); - await db.execute(sql`insert into issues (id, company_id, parent_id, title, status, assignee_agent_id, updated_at) - select gen_random_uuid(), ${companyId}::uuid, root.id, 'parameter-leaf-' || n, 'backlog', ${agentId}::uuid, ${old().toISOString()}::timestamptz - from issues root cross join generate_series(1, 999) n + await db.execute(sql`insert into issues (id, company_id, parent_id, title, status, updated_at) + select gen_random_uuid(), ${companyId}::uuid, root.id, 'parameter-branch-' || n, 'todo', ${old().toISOString()}::timestamptz + from issues root cross join generate_series(1, 10) n where root.company_id = ${companyId}::uuid and root.status = 'blocked'`); + await db.execute(sql`insert into issues (id, company_id, parent_id, title, status, assignee_agent_id, updated_at) + select gen_random_uuid(), ${companyId}::uuid, branch.id, 'parameter-leaf-' || n, 'backlog', ${agentId}::uuid, ${old().toISOString()}::timestamptz + from issues branch cross join generate_series(1, 99) n + where branch.company_id = ${companyId}::uuid and branch.title like 'parameter-branch-%'`); + const completed = await insertIssue("blocked"); + const shallow = await insertIssue("blocked", { updatedAt: new Date() }); + const middle = await insertIssue("todo", { updatedAt: new Date() }); + const overdue = await insertIssue("backlog", { assigneeAgentId: agentId }); + await db.insert(issueRelations).values([ + { companyId, issueId: middle.id, relatedIssueId: shallow.id, type: "blocks" }, + { companyId, issueId: overdue.id, relatedIssueId: middle.id, type: "blocks" }, + ]); const roots = await db.select().from(issues).where(and(eq(issues.companyId, companyId), eq(issues.status, "blocked"))); parameterCounts.length = 0; - const result = await issueService(db).listBlockerAttention(companyId, roots).catch(error => { + const result = await issueService(db).listBlockerAttentionBatch(companyId, roots).catch(error => { throw new Error(`Large graph failed: ${String(error.cause?.message ?? error.message).slice(0, 300)}`); }); - expect(result.size).toBe(70); - expect([...result.values()].every(item => item.state === "needs_attention" && item.terminalBlockerIssueId)).toBe(true); + expect(result.attention.size).toBe(72); + expect(result.deferredRootIds.has(completed.id)).toBe(false); + expect(result.attention.get(completed.id)?.terminalBlockerIssueId).toBe(completed.id); + expect([...result.attention.values()].every(item => item.state === "needs_attention" && item.terminalBlockerIssueId)).toBe(true); expect(Math.max(...parameterCounts)).toBeLessThanOrEqual(1100); + expect(parameterCounts.length).toBeLessThanOrEqual(100); + expect(result.nodeCount).toBeLessThanOrEqual(BLOCKER_ATTENTION_MAX_NODES); + expect(result.edgeCount).toBeLessThanOrEqual(BLOCKER_ATTENTION_MAX_NODES * 2); + expect(result.fetchedRows).toBeLessThanOrEqual(BLOCKER_ATTENTION_MAX_NODES * 6); + expect(result.deferredRootIds.has(shallow.id)).toBe(true); + const service = attentionRetentionService(db); + const page = await service.load({ companyId, sourceKind: "blocker_attention" }); + const immediate = page.rows!.find(entry => entry.rootId === shallow.id && entry.variant === "terminal")!; + expect(immediate.dueAt).toBeLessThanOrEqual(Date.now()); + await service.check(immediate); + expect((await state("blocker_attention", overdue.id))?.archivedAt).toBeInstanceOf(Date); }, 120_000); + it("preserves observed live work when a single large root exhausts shared fetch capacity", async () => { + const root = await insertIssue("blocked"); + const live = await insertIssue("in_progress", { assigneeAgentId: agentId }); + await db.insert(issueRelations).values({ companyId, issueId: live.id, relatedIssueId: root.id, type: "blocks" }); + await db.execute(sql`insert into issues (id, company_id, parent_id, title, status, assignee_agent_id, updated_at) + select gen_random_uuid(), ${companyId}::uuid, ${root.id}::uuid, 'bounded-leaf-' || n, 'backlog', ${agentId}::uuid, ${old().toISOString()}::timestamptz + from generate_series(1, 3000) n`); + const batch = await issueService(db).listBlockerAttentionBatch(companyId, [root]); + expect(batch.deferredRootIds.has(root.id)).toBe(true); + expect(batch.attention.get(root.id)?.blockingTreeLive).toBe(true); + const service = attentionRetentionService(db), page = await service.load({ companyId, sourceKind: "blocker_attention" }); + const entry = page.rows!.find(item => item.rootId === root.id && item.variant === "terminal")!; + expect(await service.check(entry)).toBeNull(); + expect(await state("blocker_attention", root.id)).toBeNull(); + expect(await db.select().from(decisionArchiveNotificationOutbox)).toHaveLength(0); + }); + + it("disarms finite deferred-root checks when large retained graphs have no archive obligation", async () => { + await db.execute(sql`insert into issues (id, company_id, title, status, updated_at) + select gen_random_uuid(), ${companyId}::uuid, 'held-root-' || n, 'blocked', ${old().toISOString()}::timestamptz + from generate_series(1, 4) n`); + await db.execute(sql`insert into issues (id, company_id, parent_id, title, status, assignee_agent_id, updated_at) + select gen_random_uuid(), ${companyId}::uuid, root.id, 'held-leaf-' || n, 'backlog', ${agentId}::uuid, ${old().toISOString()}::timestamptz + from issues root cross join generate_series(1, 600) n + where root.company_id = ${companyId}::uuid and root.status = 'blocked'`); + await db.execute(sql`insert into decision_retention (company_id, source_kind, source_id, source_activity_at, keep) + select company_id, 'blocker_attention', id::text, updated_at, true from issues + where company_id = ${companyId}::uuid and status = 'backlog'`); + const service = attentionRetentionService(db), check = vi.spyOn(service, "check"); + await start(service); await quiet(); + expect(check).toHaveBeenCalledTimes(4); + expect(await db.select().from(decisionArchiveNotificationOutbox)).toHaveLength(0); + }); + it("canonicalizes wide circular notification variables to the supported identity fields", async () => { const sourceId = randomUUID(), issueId = randomUUID(), hints: string[] = []; const wide = { companyId, sourceKind: "review" as const, sourceId, issueId, ignored: {} as unknown }; @@ -418,6 +481,49 @@ const delay = DEFAULT_DECISION_ARCHIVE_DAYS * 86_400_000; await archived("blocker_attention", terminal.id); await quiet(); }); + it("resolves opaque run issue references through trusted company issue rows", async () => { + const issue = await insertIssue(); + const [valid, opaque] = await db.insert(heartbeatRuns).values([ + { companyId, agentId, status: "succeeded", contextSnapshot: { issueId: issue.id } }, + { companyId, agentId, status: "succeeded", contextSnapshot: { issueId: "none" } }, + ]).returning(); + const hints: string[] = []; + const unsubscribe = subscribeDatabaseWork(db, DELIVERY_QUEUES.attentionRetention, (event, values) => { + if (event === "settled") hints.push(...values ?? []); + }); + try { + await db.transaction(tx => notifyIssueAttentionRetentionWork(tx, { companyId, runId: valid.id })); + await db.transaction(tx => notifyIssueAttentionRetentionWork(tx, { companyId, runId: opaque.id })); + } finally { unsubscribe(); } + expect(hints).toEqual([JSON.stringify({ companyId, issueId: issue.id }), JSON.stringify({ companyId })]); + }); + + it("accepts ordinary liveness incident creation and completion with a none leaf sentinel", async () => { + const source = await insertIssue("blocked", { responsibleUserId: "board" }); + const service = issueService(db); + const incident = await service.create(companyId, { title: "Invalid participant", status: "todo", assigneeUserId: "board", + createdByUserId: "board", originKind: RECOVERY_ORIGIN_KINDS.issueGraphLivenessEscalation, + originId: `harness_liveness:${companyId}:${source.id}:invalid_review_participant:none` }); + await service.update(source.id, { blockedByIssueIds: [incident.id] }); + await service.update(incident.id, { status: "done" }); + expect((await service.getById(incident.id))?.status).toBe("done"); + expect((await service.getRelationSummaries(source.id)).blockedBy).toEqual([]); + }); + + it.each(["hidden", "done"] as const)("reactivates an older review when moved stranded recovery is %s", async transition => { + const review = await insertIssue("in_review", { assigneeAgentId: agentId, responsibleUserId: "board" }); + const parent = await insertIssue("todo", { responsibleUserId: "board" }); + const recovery = await insertIssue("todo", { parentId: review.id, originKind: RECOVERY_ORIGIN_KINDS.strandedIssueRecovery, + originId: review.id, assigneeUserId: "board", responsibleUserId: "board" }); + await issueService(db).update(recovery.id, { parentId: parent.id }); + expect((await issueService(db).listReviewAttention(companyId, [review])).get(review.id)?.state).not.toBe("stalled"); + await start(); await quiet(); + await issueService(db).update(recovery.id, transition === "hidden" ? { hiddenAt: new Date() } : { status: "done" }); + await settle(); + expect((await state("review", review.id))?.archivedAt).toBeInstanceOf(Date); + expect(await db.select().from(decisionArchiveNotificationOutbox).where(eq(decisionArchiveNotificationOutbox.sourceId, review.id))).toHaveLength(1); + }); + it("scopes ordinary execution-control writes to their company", async () => { const otherCompany = randomUUID(); await db.insert(companies).values({ id: otherCompany, name: "Other", issuePrefix: "OTHER" }); diff --git a/server/src/__tests__/issues-service.test.ts b/server/src/__tests__/issues-service.test.ts index fb13ff43e82..0d28858b95c 100644 --- a/server/src/__tests__/issues-service.test.ts +++ b/server/src/__tests__/issues-service.test.ts @@ -4151,7 +4151,7 @@ const dependencyTestDatabaseUrl = resolveExternalTestDatabaseUrl("PAPERCLIP_ISSU beforeAll(async () => { if (!dependencyTestDatabaseUrl) tempDb = await startEmbeddedPostgresTestDatabase("paperclip-issues-blockers-"); - db = createDb(dependencyTestDatabaseUrl ?? tempDb!.connectionString); + db = createDb(dependencyTestDatabaseUrl ?? tempDb!.connectionString, { maxConnections: 6 }); svc = issueService(db); await ensureIssueRelationsTable(db); }, 20_000); diff --git a/server/src/__tests__/work-products.test.ts b/server/src/__tests__/work-products.test.ts index 476e1156184..cef395409fd 100644 --- a/server/src/__tests__/work-products.test.ts +++ b/server/src/__tests__/work-products.test.ts @@ -1,3 +1,5 @@ +import { PgDialect } from "drizzle-orm/pg-core"; +import type { SQL } from "drizzle-orm"; import { describe, expect, it, vi } from "vitest"; import { enrichWorkProductMetadataWithDiff, @@ -33,6 +35,18 @@ function createWorkProductRow(overrides: Partial> = {}) }; } +function retentionReads(insideTransaction: () => boolean) { + return vi.fn(async (query: SQL) => { + const statement = new PgDialect().sqlToQuery(query); + expect(insideTransaction()).toBe(true); + expect(statement.params).toContain("company-1"); + expect(statement.params).toContain("issue-1"); + if (statement.sql.includes("origin_kind")) return [{ id: "issue-1", originKind: null, originId: null }]; + if (statement.sql.includes("with recursive affected")) return [{ issueId: "issue-1" }]; + throw new Error(`Unexpected work-product fixture query: ${statement.sql}`); + }); +} + describe("workProductService", () => { it("extracts runner totals and enriches work-product metadata", () => { const summary = workProductDiffSummaryFromEventPayload({ @@ -158,11 +172,16 @@ describe("workProductService", () => { const insertValues = vi.fn(() => ({ returning: insertReturning })); const txInsert = vi.fn(() => ({ values: insertValues })); + let transactionOpen = false; const tx = { + execute: retentionReads(() => transactionOpen), update: txUpdate, insert: txInsert, }; - const transaction = vi.fn(async (callback: (input: typeof tx) => Promise) => await callback(tx)); + const transaction = vi.fn(async (callback: (input: typeof tx) => Promise) => { + transactionOpen = true; + try { return await callback(tx); } finally { transactionOpen = false; } + }); const svc = workProductService({ transaction } as any); const result = await svc.createForIssue("issue-1", "company-1", { @@ -194,11 +213,16 @@ describe("workProductService", () => { const updateSet = vi.fn(() => ({ where: updateWhere })); const txUpdate = vi.fn(() => ({ set: updateSet })); + let transactionOpen = false; const tx = { + execute: retentionReads(() => transactionOpen), select: txSelect, update: txUpdate, }; - const transaction = vi.fn(async (callback: (input: typeof tx) => Promise) => await callback(tx)); + const transaction = vi.fn(async (callback: (input: typeof tx) => Promise) => { + transactionOpen = true; + try { return await callback(tx); } finally { transactionOpen = false; } + }); const svc = workProductService({ transaction } as any); const result = await svc.update("work-product-1", { diff --git a/server/src/services/attention-retention.ts b/server/src/services/attention-retention.ts index 27aedba8c47..ebe91416f52 100644 --- a/server/src/services/attention-retention.ts +++ b/server/src/services/attention-retention.ts @@ -32,6 +32,7 @@ type PageEvidence = { agents: Set; visibleIssues: Map; newerRuns: Set; + deferredRoots: Set; blockers: Awaited["listBlockerAttention"]>>; }; @@ -184,7 +185,9 @@ export function attentionRetentionService(db: Db) { const visibleRows = await readIssues(issueIds); const visible = new Map(visibleRows.map(row => [row.id, row])); const blockerRoots = visibleRows.filter(row => blockerIds.includes(row.id)); - const blockers: PageEvidence["blockers"] = blockerRoots.length ? await issueService(database).listBlockerAttention(companyId, blockerRoots) : new Map(); + const blockerBatch = blockerRoots.length ? await issueService(database).listBlockerAttentionBatch(companyId, blockerRoots) + : { attention: new Map(), deferredRootIds: new Set() }; + const blockers: PageEvidence["blockers"] = blockerBatch.attention; const terminalIds = [...blockers.values()].filter(terminalBlockerRequiresAttention) .map(attention => attention.terminalBlockerIssueId).filter((id): id is string => id != null && !visible.has(id)); for (const row of await readIssues(terminalIds)) visible.set(row.id, row); @@ -192,13 +195,17 @@ export function attentionRetentionService(db: Db) { decisions: new Map(decisionRows.map(row => [row.id, row])), recoveries: new Map(recoveryRows.map(row => [row.id, row])), runs: new Map(runRows.map(row => [row.id, row])), agents: new Set(runAgents.map(row => row.id)), visibleIssues: visible, newerRuns: await listFailedRunsWithNewerRun(database, companyId, - runRows.map(run => ({ ...run, issueId: readRunIssueId(run.contextSnapshot) }))), blockers, + runRows.map(run => ({ ...run, issueId: readRunIssueId(run.contextSnapshot) }))), blockers, deferredRoots: blockerBatch.deferredRootIds, }); } return result; } async function eligible(database: Db, candidate: Candidate, variant?: "owner" | "terminal", lock = false, reviewAttention?: Map, interactionAttention?: Map, evidence?: PageEvidence): Promise { + if (candidate.sourceKind === "blocker_attention" && variant === "terminal" && evidence?.deferredRoots.has(candidate.sourceId)) { + return { ...candidate, rootId: candidate.sourceId, variant, + id: `${candidate.sourceKind}:${candidate.sourceId}:terminal`, dueAt: Date.now() }; + } let sourceId = candidate.sourceId; let activityAt = candidate.activityAt; let dueAt = candidate.deadlineAt; diff --git a/server/src/services/issue-thread-interactions.test.ts b/server/src/services/issue-thread-interactions.test.ts index 24e1489006f..5ad905a8a10 100644 --- a/server/src/services/issue-thread-interactions.test.ts +++ b/server/src/services/issue-thread-interactions.test.ts @@ -1,5 +1,6 @@ import { beforeEach, describe, expect, it, vi } from "vitest"; -import { getTableName } from "drizzle-orm"; +import { PgDialect } from "drizzle-orm/pg-core"; +import { getTableName, type SQL } from "drizzle-orm"; const mockCreateChild = vi.fn(); @@ -48,8 +49,18 @@ function createFakeDb(args: { const toolActionRequestUpdates: Array> = []; const inserts: Array<{ table: string; values: Record }> = []; let selectCallCount = 0; + let transactionDepth = 0; const db: any = { + execute: vi.fn(async (query: SQL) => { + const statement = new PgDialect().sqlToQuery(query); + expect(transactionDepth).toBeGreaterThan(0); + expect(statement.params).toContain(interactionRow.companyId); + expect(statement.params).toContain(interactionRow.issueId); + if (statement.sql.includes("origin_kind")) return [{ id: interactionRow.issueId, originKind: null, originId: null }]; + if (statement.sql.includes("with recursive affected")) return [{ issueId: interactionRow.issueId }]; + throw new Error(`Unexpected interaction fixture query: ${statement.sql}`); + }), select: vi.fn(() => { selectCallCount += 1; const rows = selectCallCount === 1 ? [interactionRow] : (args.parentRows ?? []); @@ -84,7 +95,10 @@ function createFakeDb(args: { inserts.push({ table: getTableName(table as never), values }); }, })), - transaction: async (callback: (tx: typeof db) => Promise) => callback(db), + transaction: async (callback: (tx: typeof db) => Promise) => { + transactionDepth++; + try { return await callback(db); } finally { transactionDepth--; } + }, }; return { @@ -177,6 +191,7 @@ describe("issueThreadInteractionService", () => { }; let selectCallCount = 0; + let transactionDepth = 0; const db: any = { select: vi.fn(() => { selectCallCount += 1; diff --git a/server/src/services/issues.ts b/server/src/services/issues.ts index c91a325ad39..4d5fd5f85f6 100644 --- a/server/src/services/issues.ts +++ b/server/src/services/issues.ts @@ -3436,6 +3436,7 @@ type IssueBlockerAttentionEdge = { blockerIssueId: string; }; type IssueBlockerAttentionQueryRow = IssueBlockerAttentionNode & { + executionWorkspaceId?: string | null; issueId: string | null; blockerIssueId: string; }; @@ -3824,11 +3825,15 @@ async function terminalExplicitBlockersByRoot( return terminalByRoot; } -async function listIssueBlockerAttentionMap( +async function listIssueBlockerAttentionMap(dbOrTx: any, companyId: string, issueRows: IssueBlockerAttentionInputNode[]) { + return (await readIssueBlockerAttentionBatch(dbOrTx, companyId, issueRows)).attention; +} + +async function readIssueBlockerAttentionBatch( dbOrTx: any, companyId: string, issueRows: IssueBlockerAttentionInputNode[], -): Promise> { +) { const roots = issueRows.filter( (row) => row.companyId === companyId && row.status === "blocked", ); @@ -3838,37 +3843,38 @@ async function listIssueBlockerAttentionMap( attentionMap.set(row.id, createIssueBlockerAttention()); } } - if (roots.length === 0) return attentionMap; + const deferredRootIds = new Set(roots.slice(BLOCKER_ATTENTION_MAX_NODES).map(root => root.id)); + let edgeCount = 0, fetchedRows = 0; + const result = () => ({ attention: attentionMap, deferredRootIds, nodeCount: nodesById.size, edgeCount, fetchedRows }); + if (roots.length === 0) return { attention: attentionMap, deferredRootIds, nodeCount: 0, edgeCount, fetchedRows }; const nodesById = new Map(); const edgesByIssueId = new Map(); - for (const root of roots) nodesById.set(root.id, { ...root }); + const admittedRoots = roots.slice(0, BLOCKER_ATTENTION_MAX_NODES); + for (const root of admittedRoots) nodesById.set(root.id, { ...root }); - type Traversal = { nodes: Set; expanded: Set; frontier: string[]; truncated: boolean }; - const traversals = new Map(roots.map(root => [root.id, { - nodes: new Set([root.id]), expanded: new Set(), frontier: [root.id], truncated: false, + type Traversal = { rootId: string; nodes: Set; expanded: Set; frontier: string[]; truncated: boolean }; + const traversals = new Map(admittedRoots.map(root => [root.id, { + rootId: root.id, nodes: new Set([root.id]), expanded: new Set(), frontier: [root.id], truncated: false, }])); const expanded = new Set(); + const queried = new Set(); const pendingFinalizeBlockerIssueIds = new Set(); for (let depth = 0; depth < BLOCKER_ATTENTION_MAX_DEPTH; depth += 1) { const active = [...traversals.values()].filter(traversal => !traversal.truncated && traversal.frontier.length > 0); if (active.length === 0) break; - const frontier = [...new Set(active.flatMap(traversal => traversal.frontier))].filter(id => !expanded.has(id)); + const unread = new Set(); + for (const traversal of active) for (const id of traversal.frontier) if (!expanded.has(id)) unread.add(id); + const frontier = [...unread]; for (const chunk of chunkList( frontier, ISSUE_LIST_RELATED_QUERY_CHUNK_SIZE, )) { - const readinessByIssueId = await listIssueDependencyReadinessMap( - dbOrTx, - companyId, - chunk, - ); - for (const readiness of readinessByIssueId.values()) { - for (const blockerIssueId of readiness.pendingFinalizeBlockerIssueIds) { - pendingFinalizeBlockerIssueIds.add(blockerIssueId); - } - } + const remaining = Math.min(BLOCKER_ATTENTION_MAX_NODES - nodesById.size, + BLOCKER_ATTENTION_MAX_NODES * 2 - edgeCount); + if (remaining <= 0) break; + const rowLimit = remaining + 1; const explicitBlockerRowsPromise: Promise< IssueBlockerAttentionQueryRow[] > = dbOrTx @@ -3882,6 +3888,7 @@ async function listIssueBlockerAttentionMap( title: issues.title, status: issues.status, executionRunId: issues.executionRunId, + executionWorkspaceId: issues.executionWorkspaceId, assigneeAgentId: issues.assigneeAgentId, assigneeUserId: issues.assigneeUserId, }) @@ -3894,7 +3901,7 @@ async function listIssueBlockerAttentionMap( inArray(issueRelations.relatedIssueId, chunk), eq(issues.companyId, companyId), ), - ); + ).limit(rowLimit); const childRowsPromise: Promise = dbOrTx .select({ issueId: issues.parentId, @@ -3919,56 +3926,56 @@ async function listIssueBlockerAttentionMap( BLOCKER_ATTENTION_CHILD_TERMINAL_STATUSES, ), ), - ); + ).limit(rowLimit); const [explicitBlockerRows, childRows] = await Promise.all([ explicitBlockerRowsPromise, childRowsPromise, ]); + for (const id of chunk) queried.add(id); + fetchedRows += explicitBlockerRows.length + childRows.length; + const pendingFinalize = await listPendingFinalizeBlockerIssueIds(dbOrTx, companyId, + explicitBlockerRows.flatMap(row => row.status === "done" && row.executionWorkspaceId + ? [{ blockerIssueId: row.blockerIssueId, executionWorkspaceId: row.executionWorkspaceId }] : [])); + for (const id of pendingFinalize) pendingFinalizeBlockerIssueIds.add(id); + let complete = explicitBlockerRows.length < rowLimit && childRows.length < rowLimit; const unresolvedExplicitBlockerRows = explicitBlockerRows.filter( (row) => row.status !== "done" || pendingFinalizeBlockerIssueIds.has(row.blockerIssueId), ); - appendBlockerAttentionEdges(edgesByIssueId, [ - ...unresolvedExplicitBlockerRows - .filter( - (row): row is IssueBlockerAttentionQueryRow & { issueId: string } => - row.issueId !== null, - ) - .map((row) => ({ - issueId: row.issueId, - blockerIssueId: row.blockerIssueId, - })), - ...childRows - .filter( - (row): row is IssueBlockerAttentionQueryRow & { issueId: string } => - row.issueId !== null, - ) - .map((row) => ({ - issueId: row.issueId, - blockerIssueId: row.blockerIssueId, - })), - ]); - for (const row of [...unresolvedExplicitBlockerRows, ...childRows]) { - if (!row.issueId || nodesById.has(row.blockerIssueId)) continue; - nodesById.set(row.blockerIssueId, { - id: row.blockerIssueId, - companyId: row.companyId, - parentId: row.parentId, - identifier: row.identifier, - title: row.title, - status: row.status, - executionRunId: row.executionRunId, - assigneeAgentId: row.assigneeAgentId, - assigneeUserId: row.assigneeUserId, - }); + if (!row.issueId) continue; + if (!nodesById.has(row.blockerIssueId)) { + if (nodesById.size >= BLOCKER_ATTENTION_MAX_NODES) { complete = false; continue; } + nodesById.set(row.blockerIssueId, { + id: row.blockerIssueId, companyId: row.companyId, parentId: row.parentId, identifier: row.identifier, + title: row.title, status: row.status, executionRunId: row.executionRunId, + assigneeAgentId: row.assigneeAgentId, assigneeUserId: row.assigneeUserId, + }); + } + const edges = edgesByIssueId.get(row.issueId) ?? []; + if (edges.some(edge => edge.blockerIssueId === row.blockerIssueId)) continue; + if (edgeCount >= BLOCKER_ATTENTION_MAX_NODES * 2) { complete = false; continue; } + edges.push({ issueId: row.issueId, blockerIssueId: row.blockerIssueId }); + edgesByIssueId.set(row.issueId, edges); + edgeCount++; } - for (const id of chunk) expanded.add(id); + if (complete) for (const id of chunk) expanded.add(id); + } for (const traversal of active) { + if (traversal.frontier.some(id => !expanded.has(id))) { + for (const id of traversal.frontier) if (queried.has(id)) { + traversal.expanded.add(id); + for (const edge of edgesByIssueId.get(id) ?? []) if (nodesById.has(edge.blockerIssueId)) traversal.nodes.add(edge.blockerIssueId); + } + traversal.truncated = true; + traversal.frontier = []; + deferredRootIds.add(traversal.rootId); + continue; + } const nextFrontier = new Set(); for (const id of traversal.frontier) { traversal.expanded.add(id); @@ -4548,7 +4555,7 @@ async function listIssueBlockerAttentionMap( ); } - return attentionMap; + return result(); } type IssueReviewAttentionInput = Pick; @@ -9383,6 +9390,9 @@ export function issueService(db: Db) { return listIssueDependencyReadinessMap(dbOrTx, companyId, issueIds); }, + listBlockerAttentionBatch: async (companyId: string, issueRows: IssueBlockerAttentionInputNode[], dbOrTx: any = db) => + readIssueBlockerAttentionBatch(dbOrTx, companyId, issueRows), + listBlockerAttention: async ( companyId: string, issueRows: IssueBlockerAttentionInputNode[], diff --git a/server/src/services/retention-work-notifications.ts b/server/src/services/retention-work-notifications.ts index cee2486f54c..ba0379afee1 100644 --- a/server/src/services/retention-work-notifications.ts +++ b/server/src/services/retention-work-notifications.ts @@ -20,9 +20,10 @@ export async function notifyIssueAttentionRetentionWork(transaction: object, where company_id = ${scope.companyId}::uuid and id = ${scope.runId}::uuid and issue_id is not null union select native_issue_id::text from heartbeat_runs where company_id = ${scope.companyId}::uuid and id = ${scope.runId}::uuid and native_issue_id is not null - union select coalesce(context_snapshot->>'issueId', context_snapshot->>'taskId') from heartbeat_runs - where company_id = ${scope.companyId}::uuid and id = ${scope.runId}::uuid - and coalesce(context_snapshot->>'issueId', context_snapshot->>'taskId') is not null`); + union select issue.id::text from heartbeat_runs run inner join issues issue + on issue.company_id = run.company_id + and issue.id::text = coalesce(run.context_snapshot->>'issueId', run.context_snapshot->>'taskId') + where run.company_id = ${scope.companyId}::uuid and run.id = ${scope.runId}::uuid`); for (const row of rows) seeds.push(row.id as string); } if (!seeds.length) return notifyAttentionRetentionWork(transaction, { companyId: scope.companyId }); @@ -31,10 +32,20 @@ export async function notifyIssueAttentionRetentionWork(transaction: object, and id in (${sql.join([...new Set(seeds)].map(id => sql`${id}::uuid`), sql`, `)})`); const current = origins.find(source => source.id === scope.issueId); origins.push({ originKind: scope.originKind ?? current?.originKind, originId: scope.originId ?? current?.originId }); + const references: string[] = []; for (const source of origins) { + if (source.originKind === RECOVERY_ORIGIN_KINDS.strandedIssueRecovery && typeof source.originId === "string") { + references.push(source.originId); + } const incident = source.originKind === RECOVERY_ORIGIN_KINDS.issueGraphLivenessEscalation ? parseIssueGraphLivenessIncidentKey(source.originId as string | null) : null; - if (incident?.companyId === scope.companyId) seeds.push(incident.issueId, incident.leafIssueId); + if (incident?.companyId === scope.companyId) references.push(incident.issueId, incident.leafIssueId); + } + if (references.length) { + const trusted = await database.execute(sql`select id::text as id from issues + where company_id = ${scope.companyId}::uuid + and id::text in (${sql.join([...new Set(references)].map(id => sql`${id}`), sql`, `)})`); + for (const row of trusted) seeds.push(row.id as string); } const affected = await database.execute(sql`with recursive affected(id) as ( select unnest(array[${sql.join([...new Set(seeds)].map(id => sql`${id}::uuid`), sql`, `)}]) From 6469daec07fed5afc68d8d441f4a0009dcea9673 Mon Sep 17 00:00:00 2001 From: Dotta Date: Fri, 30 Oct 2026 19:41:19 -0500 Subject: [PATCH 65/74] fix: bound status-card active-hours calculation work Reuse bounded timezone formatters and upcoming openings only within their exact searched closed ranges. Short-circuit empty windows while preserving the existing cold DST search and precise instant eligibility. Verify cost bounds, cache eviction, timezone transitions and twenty-card ordinary source delivery. Co-Authored-By: Paperclip --- doc/plans/2026-10-30-status-cards-sleep.md | 14 ++ .../status-card-evaluation-work.test.ts | 47 +++++++ .../status-card-update-engine.test.ts | 124 +++++++++++++++++- .../src/services/status-card-update-engine.ts | 53 ++++++-- 4 files changed, 225 insertions(+), 13 deletions(-) diff --git a/doc/plans/2026-10-30-status-cards-sleep.md b/doc/plans/2026-10-30-status-cards-sleep.md index 61d451caed5..3f3b8933cbc 100644 --- a/doc/plans/2026-10-30-status-cards-sleep.md +++ b/doc/plans/2026-10-30-status-cards-sleep.md @@ -141,3 +141,17 @@ deadline-reconstruction hint. Every card mutation requires a company ID; only instance-level settings retain wildcard ingress. Deleting an already-missing card emits no hint. The real two-company regression proves deletion does not re-run unchanged reactive searches or wake either summarizer, then stays quiet. + +Active-hours eligibility keeps the existing exact UTC-minute search across DST +changes. A bounded 32-entry timezone formatter cache removes repeated ICU +construction. A bounded 256-entry cache reuses the next opening for the exact +(timezone,start,end) policy only while the candidate instant stays within its +previously searched closed range. The precise instant is checked first, so +historical second-based offsets, the opening itself, and millisecond deadlines +keep their behavior. Clock rollback and policy changes cannot reuse the wrong +range. Empty windows return without formatter or search work. A cold nonempty +policy can still perform at most 2881 conversions including its first check; warm +closed-range evaluations perform one conversion, not another minute walk. +Both caches use bounded LRU retention and hold no company/card data. Tests prove +eviction, skipped/repeated hours, half-hour DST, date-line changes, rollback, +strict boundaries, and actual twenty-card source delivery followed by quiet. diff --git a/server/src/__tests__/status-card-evaluation-work.test.ts b/server/src/__tests__/status-card-evaluation-work.test.ts index e4588b16be1..d012a239efd 100644 --- a/server/src/__tests__/status-card-evaluation-work.test.ts +++ b/server/src/__tests__/status-card-evaluation-work.test.ts @@ -272,6 +272,53 @@ const support = external ? { supported: true } : await getEmbeddedPostgresTestSu await quiet(); expect(errors).toEqual([]); }, 30_000); + it("bounds active-hours conversion work across twenty cards and repeated ordinary company writes", async () => { + vi.setSystemTime(new Date("2026-11-02T18:00:00Z")); + const hoursPolicy = { ...policy(), activeHours: { start: "09:00", end: "17:00", timezone: "UTC" } }; + const cards = []; + for (let i = 0; i < 20; i++) cards.push(await card("reactive", hoursPolicy)); + await start(); await quiet(); + const RealFormatter = Intl.DateTimeFormat; + const actualParts = RealFormatter.prototype.formatToParts; + let constructions = 0, conversions = 0; + const constructor = vi.spyOn(Intl, "DateTimeFormat").mockImplementation(function (...args: ConstructorParameters) { + if (args[0] === "en-GB" && args[1]?.hourCycle === "h23") constructions++; + return new RealFormatter(...args); + } as unknown as typeof Intl.DateTimeFormat); + const parts = vi.spyOn(RealFormatter.prototype, "formatToParts").mockImplementation(function (this: Intl.DateTimeFormat, date) { + if (this.resolvedOptions().locale === "en-GB" && this.resolvedOptions().hourCycle === "h23") conversions++; + return actualParts.call(this, date); + }); + const target = await issueService(db).create(companyId, { title: "Closed-hours blocker", status: "blocked", priority: "medium" }); + await settled(); + expect(constructions).toBeLessThanOrEqual(1); + expect(conversions).toBeLessThanOrEqual(1000); + const cold = { constructions, conversions }; + constructions = 0; conversions = 0; + for (let i = 0; i < 10; i++) { + await issueService(db).addComment(target.id, `Closed-hours human update ${i}`, { userId: "board-user" }); + await settled(); + } + expect(constructions).toBe(0); + expect(conversions).toBeGreaterThanOrEqual(200); + expect(conversions).toBeLessThanOrEqual(800); + expect(wakeup).not.toHaveBeenCalled(); + const opening = new Date("2026-11-03T09:00:00Z"); + for (const created of cards) expect((await statusCardService(db).getById(created.id))) + .toMatchObject({ state: "paused_hours", pendingChangeCount: 1, nextEvalAt: opening }); + process.stdout.write(JSON.stringify({ measurement: "status-card company active-hours workload", cards: 20, + sourceWrites: 11, cold, warm: { constructions, conversions } }) + "\n"); + constructor.mockRestore(); parts.mockRestore(); + await vi.advanceTimersByTimeAsync(opening.getTime() - Date.now()); await settled(); + expect(wakeup).toHaveBeenCalledTimes(20); + for (const created of cards) { + const generated = (await statusCardService(db).getById(created.id))!; + expect(generated.generatingIssueId).toBeTruthy(); + await writeback(created.id, generated.generatingIssueId!); + } + await quiet(); expect(errors).toEqual([]); + }, 60_000); + it("cancels archived or disabled future work and replaces an earlier interval deadline", async () => { await start(); let created = await card("interval"); await settled(); const old = coordinator!.nextWakeAt(); diff --git a/server/src/__tests__/status-card-update-engine.test.ts b/server/src/__tests__/status-card-update-engine.test.ts index b8309415770..9fdf18584e9 100644 --- a/server/src/__tests__/status-card-update-engine.test.ts +++ b/server/src/__tests__/status-card-update-engine.test.ts @@ -1,4 +1,4 @@ -import { describe, expect, it } from "vitest"; +import { afterEach, describe, expect, it, vi } from "vitest"; import { statusCardRefreshPolicySchema } from "@paperclipai/shared"; import { chooseStatusCardUpdateKind, @@ -13,6 +13,7 @@ import { } from "../services/status-card-update-engine.js"; describe("status card update engine", () => { + afterEach(() => vi.restoreAllMocks()); const defaultPolicy = statusCardRefreshPolicySchema.parse({ mode: "interval", intervalMinutes: 15 }); it("retains non-terminal and terminal status transitions plus membership changes", () => { @@ -101,6 +102,127 @@ describe("status card update engine", () => { expect(nextStatusCardPolicyEligibility({ policy, now, lastChangeAt: now, completedUpdateTimes: [], tokensToday: 0 })).toBeNull(); }); + it("bounds empty-window formatter work before any timezone conversion", () => { + const policy = statusCardRefreshPolicySchema.parse({ mode: "reactive", debounceSeconds: 1, + activeHours: { start: "09:00", end: "09:00", timezone: "Pacific/Chatham" } }); + const formatter = vi.spyOn(Intl, "DateTimeFormat").mockImplementation(function () { + throw new Error("Empty active hours must not construct a formatter"); + }); + const now = new Date("2026-07-23T14:00:00Z"); + expect(isWithinStatusCardActiveHours(policy, now)).toBe(false); + expect(nextStatusCardPolicyEligibility({ policy, now, lastChangeAt: now, completedUpdateTimes: [], tokensToday: 0 })).toBeNull(); + expect(formatter).not.toHaveBeenCalled(); + }); + + it("bounds formatter construction and reuses the next opening across cards and source writes", () => { + const policy = statusCardRefreshPolicySchema.parse({ mode: "reactive", debounceSeconds: 1, + activeHours: { start: "09:00", end: "17:00", timezone: "America/St_Johns" } }); + const RealFormatter = Intl.DateTimeFormat; + let constructions = 0; + vi.spyOn(Intl, "DateTimeFormat").mockImplementation(function (...args: ConstructorParameters) { + if (++constructions > 1) throw new Error("Card evaluation repeatedly constructed timezone formatters"); + return new RealFormatter(...args); + } as unknown as typeof Intl.DateTimeFormat); + const conversions = vi.spyOn(RealFormatter.prototype, "formatToParts"); + const now = new Date("2026-07-23T22:00:00Z"); + const input = { policy, now, lastChangeAt: new Date(now.getTime() - 60_000), completedUpdateTimes: [], tokensToday: 0 }; + const started = performance.now(); + expect(nextStatusCardPolicyEligibility(input)).toEqual(new Date("2026-07-24T11:30:00Z")); + const coldMs = performance.now() - started; + const coldConversions = conversions.mock.calls.length; + expect(coldConversions).toBeLessThanOrEqual(2881); + conversions.mockClear(); + const warmStarted = performance.now(); + for (let write = 0; write < 100; write++) for (let card = 0; card < 20; card++) { + expect(nextStatusCardPolicyEligibility({ ...input, + policy: { ...policy, activeHours: { ...policy.activeHours! } }, + now: new Date(now.getTime() + write * 1000), + })).toEqual(new Date("2026-07-24T11:30:00Z")); + } + expect(constructions).toBeLessThanOrEqual(1); + expect(conversions.mock.calls.length).toBeLessThanOrEqual(2000); + process.stdout.write(JSON.stringify({ measurement: "status-card active-hours cost", coldMs, coldConversions, constructions, + warmMs: performance.now() - warmStarted, warmConversions: conversions.mock.calls.length, evaluations: 2000 }) + "\n"); + }); + + it.each([ + ["America/New_York", "02:15", "03:30", "2026-03-08T06:50:00Z", "2026-03-08T07:00:00Z"], + ["America/New_York", "02:15", "02:45", "2026-03-08T06:50:00Z", "2026-03-09T06:15:00Z"], + ["America/New_York", "02:15", "02:45", "2026-03-07T07:45:00Z", "2026-03-09T06:15:00Z"], + ["America/New_York", "01:15", "01:45", "2026-11-01T05:50:00Z", "2026-11-01T06:15:00Z"], + ["America/New_York", "23:30", "01:30", "2026-11-01T05:40:00Z", "2026-11-01T06:00:00Z"], + ["Australia/Lord_Howe", "02:15", "02:45", "2026-10-03T15:20:00Z", "2026-10-03T15:30:00Z"], + ["Australia/Lord_Howe", "01:35", "01:45", "2026-04-04T14:50:00Z", "2026-04-04T15:05:00Z"], + ["Asia/Kathmandu", "09:00", "17:00", "2026-07-23T12:00:00Z", "2026-07-24T03:15:00Z"], + ["Pacific/Apia", "00:10", "00:20", "2011-12-30T09:50:00Z", "2011-12-30T10:10:00Z"], + ])("preserves the first real opening in %s %s-%s after %s", (timezone, start, end, time, expected) => { + const policy = statusCardRefreshPolicySchema.parse({ mode: "reactive", debounceSeconds: 1, activeHours: { start, end, timezone } }); + const now = new Date(time); + const input = { policy, now, lastChangeAt: new Date(now.getTime() - 60_000), completedUpdateTimes: [], tokensToday: 0 }; + expect(nextStatusCardPolicyEligibility(input)).toEqual(new Date(expected)); + // A later closed instant reuses the opening, while the exact opening itself + // remains eligible without rounding it into the next minute or day. + expect(nextStatusCardPolicyEligibility({ ...input, now: new Date(now.getTime() + 1000) })).toEqual(new Date(expected)); + const opening = new Date(expected); + expect(nextStatusCardPolicyEligibility({ ...input, now: opening })).toEqual(opening); + expect(nextStatusCardPolicyEligibility({ ...input, now: new Date(opening.getTime() + 1) })).toEqual(new Date(opening.getTime() + 1)); + }); + + it("invalidates an upcoming opening for clock rollback and changed policy values", () => { + const policy = statusCardRefreshPolicySchema.parse({ mode: "reactive", debounceSeconds: 1, + activeHours: { start: "09:00", end: "17:00", timezone: "UTC" } }); + const calculate = (time: string, activeHours = policy.activeHours!) => { + const now = new Date(time); + return nextStatusCardPolicyEligibility({ policy: { ...policy, activeHours }, now, + lastChangeAt: new Date(now.getTime() - 60_000), completedUpdateTimes: [], tokensToday: 0 }); + }; + expect(calculate("2026-07-23T18:00:00Z")).toEqual(new Date("2026-07-24T09:00:00Z")); + expect(calculate("2026-07-23T08:00:00Z")).toEqual(new Date("2026-07-23T09:00:00Z")); + expect(calculate("2026-07-23T17:00:00Z")).toEqual(new Date("2026-07-24T09:00:00Z")); + expect(calculate("2026-07-23T18:00:00Z", { ...policy.activeHours!, start: "10:00" })).toEqual(new Date("2026-07-24T10:00:00Z")); + expect(calculate("2026-07-23T18:00:00Z", { ...policy.activeHours!, end: "19:00" })).toEqual(new Date("2026-07-23T18:00:00Z")); + expect(calculate("2026-07-23T18:00:00Z", { ...policy.activeHours!, timezone: "Asia/Tokyo" })).toEqual(new Date("2026-07-24T00:00:00Z")); + }); + + it("checks the precise instant before a cached minute opening for historical second offsets", () => { + const policy = statusCardRefreshPolicySchema.parse({ mode: "reactive", debounceSeconds: 1, + activeHours: { start: "09:00", end: "10:00", timezone: "Africa/Monrovia" } }); + const now = new Date("1970-01-01T08:00:00Z"); + const input = { policy, now, lastChangeAt: new Date(now.getTime() - 60_000), completedUpdateTimes: [], tokensToday: 0 }; + expect(nextStatusCardPolicyEligibility(input)).toEqual(new Date("1970-01-01T09:45:00Z")); + const inside = new Date("1970-01-01T09:44:45Z"); + expect(isWithinStatusCardActiveHours(policy, inside)).toBe(true); + expect(nextStatusCardPolicyEligibility({ ...input, now: inside })).toEqual(inside); + }); + + it("bounds formatter and upcoming-opening cache retention", () => { + const now = new Date("2026-07-25T00:00:00Z"); + const timezonePolicy = (timezone: string) => statusCardRefreshPolicySchema.parse({ mode: "reactive", debounceSeconds: 1, + activeHours: { start: "09:00", end: "17:00", timezone } }); + const zones = Intl.supportedValuesOf("timeZone").slice(0, 34); + const first = timezonePolicy(zones[0]!); + isWithinStatusCardActiveHours(first, now); + for (const zone of zones.slice(1)) isWithinStatusCardActiveHours(timezonePolicy(zone), now); + const RealFormatter = Intl.DateTimeFormat; + const constructor = vi.spyOn(Intl, "DateTimeFormat").mockImplementation(function (...args: ConstructorParameters) { + return new RealFormatter(...args); + } as unknown as typeof Intl.DateTimeFormat); + isWithinStatusCardActiveHours(first, now); + expect(constructor).toHaveBeenCalledTimes(1); + constructor.mockRestore(); + const time = (minute: number) => `${String(Math.floor(minute / 60)).padStart(2, "0")}:${String(minute % 60).padStart(2, "0")}`; + const calculate = (minute: number) => { + const policy = statusCardRefreshPolicySchema.parse({ mode: "reactive", debounceSeconds: 1, + activeHours: { start: time(minute), end: time(minute + 1), timezone: "UTC" } }); + return nextStatusCardPolicyEligibility({ policy, now, lastChangeAt: new Date(now.getTime() - 60_000), completedUpdateTimes: [], tokensToday: 0 }); + }; + calculate(1); + for (let minute = 2; minute <= 258; minute++) calculate(minute); + const conversions = vi.spyOn(Intl.DateTimeFormat.prototype, "formatToParts"); + expect(calculate(1)).toEqual(new Date("2026-07-25T00:01:00Z")); + expect(conversions.mock.calls.length).toBeGreaterThan(1); + }); + it("selects full rebuilds for bounded drift rules and incremental otherwise", () => { const base = { hasDocument: true, changeCount: 2, queryVersion: 3, lastUpdateQueryVersion: 3, incrementalCount: 2, configurationChanged: false }; expect(chooseStatusCardUpdateKind(base)).toBe("incremental"); diff --git a/server/src/services/status-card-update-engine.ts b/server/src/services/status-card-update-engine.ts index 1f734227808..0b1e5325446 100644 --- a/server/src/services/status-card-update-engine.ts +++ b/server/src/services/status-card-update-engine.ts @@ -109,21 +109,36 @@ export function statusCardFingerprintHash(fingerprint: StatusCardFingerprint) { return createHash("sha256").update(JSON.stringify(stable)).digest("hex"); } +// These caches contain only timezone/policy calculations, never card or company +// data. Bound both maps because timezone aliases and hour pairs are user input. +const activeHoursFormatters = new Map(); +const upcomingActiveHours = new Map(); + +function remember(cache: Map, key: K, value: V, limit: number): V { + cache.delete(key); + cache.set(key, value); + if (cache.size > limit) cache.delete(cache.keys().next().value!); + return value; +} + +function activeHoursFormatter(timezone: string) { + const cached = activeHoursFormatters.get(timezone); + return remember(activeHoursFormatters, timezone, cached ?? new Intl.DateTimeFormat("en-GB", { + timeZone: timezone, hour: "2-digit", minute: "2-digit", hourCycle: "h23", + }), 32); +} + export function isWithinStatusCardActiveHours(policy: StatusCardRefreshPolicy, now: Date) { if (!policy.activeHours) return true; - const parts = new Intl.DateTimeFormat("en-GB", { - timeZone: policy.activeHours.timezone, - hour: "2-digit", - minute: "2-digit", - hourCycle: "h23", - }).formatToParts(now); - const hour = Number(parts.find((part) => part.type === "hour")?.value ?? 0); - const minute = Number(parts.find((part) => part.type === "minute")?.value ?? 0); - const current = hour * 60 + minute; const [startHour, startMinute] = policy.activeHours.start.split(":").map(Number); const [endHour, endMinute] = policy.activeHours.end.split(":").map(Number); const start = startHour! * 60 + startMinute!; const end = endHour! * 60 + endMinute!; + if (start === end) return false; + const parts = activeHoursFormatter(policy.activeHours.timezone).formatToParts(now); + const hour = Number(parts.find((part) => part.type === "hour")?.value ?? 0); + const minute = Number(parts.find((part) => part.type === "minute")?.value ?? 0); + const current = hour * 60 + minute; return start <= end ? current >= start && current < end : current >= start || current < end; } @@ -181,6 +196,8 @@ export function nextStatusCardPolicyEligibility(input: { tokensToday: number; }): Date | null { if (input.policy.mode === "manual") return null; + const hours = input.policy.activeHours; + if (hours && hours.start === hours.end) return null; let at = input.now.getTime(); if (input.tokensToday >= (input.policy.dailyTokenCap ?? 100_000)) { const day = new Date(input.now); @@ -194,13 +211,25 @@ export function nextStatusCardPolicyEligibility(input: { .filter(time => time > at - 3_600_000).sort((a, b) => a - b); if (recent.length >= cap) at = Math.max(at, recent[recent.length - cap]! + 3_600_000 + 1); } - if (isWithinStatusCardActiveHours(input.policy, new Date(at))) return new Date(at); + // Check the precise instant before reusing a minute-aligned opening. Some + // historical timezone offsets include seconds, so a window may be open now. + if (!hours || isWithinStatusCardActiveHours(input.policy, new Date(at))) return new Date(at); + const key = JSON.stringify([hours.timezone, hours.start, hours.end]); + const cached = upcomingActiveHours.get(key); + if (cached && at >= cached.closedAt && at < cached.opensAt) { + remember(upcomingActiveHours, key, cached, 256); + return new Date(cached.opensAt); + } + const closedAt = at; // Active-hour precision is one local minute. Walking UTC minutes also honors // skipped/repeated local times at daylight-saving transitions. const end = at + 48 * 3_600_000; for (at = Math.floor(at / 60_000) * 60_000 + 60_000; at <= end; at += 60_000) { - if (isWithinStatusCardActiveHours(input.policy, new Date(at))) return new Date(at); + if (isWithinStatusCardActiveHours(input.policy, new Date(at))) { + remember(upcomingActiveHours, key, { closedAt, opensAt: at }, 256); + return new Date(at); + } } - // Equal start/end describes an empty window; retain changes, awaiting config. + // No opening inside the existing bounded search horizon. Retain changes. return null; } From 8911cca5d0b43a9f8e8adddc8677b333b3f3d2bb Mon Sep 17 00:00:00 2001 From: Dotta Date: Fri, 30 Oct 2026 19:49:06 -0500 Subject: [PATCH 66/74] fix(server): retain live evidence beyond blocker graph admission Keep bounded returned child facts independent of graph node and edge admission, and use the existing live-work predicate for running status, active runs and pending waits. Cover the opposite query ordering with 1,999 explicit blockers, fresh retention checks, no archive/outbox and unchanged SQL/resource bounds. Co-Authored-By: Paperclip --- .../2026-10-30-attention-retention-sleep.md | 2 +- .../attention-retention-work.test.ts | 31 +++++++++++++++++ server/src/services/issues.ts | 34 +++++++++++++------ 3 files changed, 55 insertions(+), 12 deletions(-) diff --git a/doc/plans/2026-10-30-attention-retention-sleep.md b/doc/plans/2026-10-30-attention-retention-sleep.md index aa78f1bb746..2be011ccf9d 100644 --- a/doc/plans/2026-10-30-attention-retention-sleep.md +++ b/doc/plans/2026-10-30-attention-retention-sleep.md @@ -25,7 +25,7 @@ The feed and worker share confirmation collapse, newer-run probes, issue visibil ## Existing mutation boundaries reactivate work -Committed hints identify a company, source kind, source ID or changed issue where those values are known. Routine issue writers pass identity to the retention domain helper before their mutation. The helper captures old and new parent paths, reverse blocker ancestors, run-linked issues and liveness-origin references; issue-scoped source queries then select reviews, blockers, interactions, decisions, recovery actions and failed runs by their issue identity. Budget incidents and agent errors remain tied to their own source changes. Ordinary issue writes do not reconstruct the unrelated company feed. Opaque liveness and stranded-recovery origin references resolve through company-scoped issue rows before entering UUID queries, so valid none sentinels do not roll back writes and moved recovery tasks still invalidate their original review. Queue keys explicitly pick supported identity fields. Retention's own writes use exact source hints; blocker retention changes invalidate the company/source because multiple blocked roots can project one terminal subject. Reconstruction pages and due batches remain bounded. Review attention and interaction invokability are computed in company batches per candidate page; a full 100-candidate review page uses at most 20 statements and 100 confirmations use at most six. Decisions, recovery source links, runs, eligible agents, issue visibility, newer-run probes and blocker graphs also load in company batches per page. A 100-source decision or recovery page uses at most three statements; 100 failed runs use at most six; 100 blocker roots use at most 30 graph/source statements plus retention-state reads for their emitted entries. These evidence maps exist only for one load. The blocker owner shares fetched node and edge facts while tracking depth, node count, expanded nodes and live-work reachability independently for each root. The shared graph fetch stores at most 2,000 nodes and 4,000 edges, and bounds each frontier query before materializing rows. An internal batch result identifies roots whose evaluation is incomplete; the legacy map API exposes no extra transport fields. Retention schedules only those roots for an immediate fresh single-root check, which disarms or saves the actual source deadline. It never archives a partial intermediary or adds a recurring incomplete-root poll. Fully evaluated roots preserve their exact classification. All later graph queries, including recovery source IDs, agent IDs and workspace-finalize inputs, use bounded chunks. A SQL-generated 70,000-node graph proves PostgreSQL parameter limits are respected at the driver boundary. Archive checks still lock and freshly re-read sources, terminal subjects and related eligibility. Ordinary execution, decision and budget mutations pass their affected company; exact decision sources are included. +Committed hints identify a company, source kind, source ID or changed issue where those values are known. Routine issue writers pass identity to the retention domain helper before their mutation. The helper captures old and new parent paths, reverse blocker ancestors, run-linked issues and liveness-origin references; issue-scoped source queries then select reviews, blockers, interactions, decisions, recovery actions and failed runs by their issue identity. Budget incidents and agent errors remain tied to their own source changes. Ordinary issue writes do not reconstruct the unrelated company feed. Opaque liveness and stranded-recovery origin references resolve through company-scoped issue rows before entering UUID queries, so valid none sentinels do not roll back writes and moved recovery tasks still invalidate their original review. Queue keys explicitly pick supported identity fields. Retention's own writes use exact source hints; blocker retention changes invalidate the company/source because multiple blocked roots can project one terminal subject. Reconstruction pages and due batches remain bounded. Review attention and interaction invokability are computed in company batches per candidate page; a full 100-candidate review page uses at most 20 statements and 100 confirmations use at most six. Decisions, recovery source links, runs, eligible agents, issue visibility, newer-run probes and blocker graphs also load in company batches per page. A 100-source decision or recovery page uses at most three statements; 100 failed runs use at most six; 100 blocker roots use at most 30 graph/source statements plus retention-state reads for their emitted entries. These evidence maps exist only for one load. The blocker owner shares fetched node and edge facts while tracking depth, node count, expanded nodes and live-work reachability independently for each root. The shared graph fetch stores at most 2,000 nodes and 4,000 edges, and bounds each frontier query before materializing rows. Positive live-work evidence from the bounded returned rows is retained independently of node/edge admission, including running status, active runs and pending confirmations. Filling the graph with explicit blockers cannot hide a subsequently returned active child. An internal batch result identifies roots whose evaluation is incomplete; the legacy map API exposes no extra transport fields. Retention schedules only those roots for an immediate fresh single-root check, which disarms or saves the actual source deadline. It never archives a partial intermediary or adds a recurring incomplete-root poll. Fully evaluated roots preserve their exact classification. All later graph queries, including recovery source IDs, agent IDs and workspace-finalize inputs, use bounded chunks. A SQL-generated 70,000-node graph proves PostgreSQL parameter limits are respected at the driver boundary. Archive checks still lock and freshly re-read sources, terminal subjects and related eligibility. Ordinary execution, decision and budget mutations pass their affected company; exact decision sources are included. Producer coverage includes decision creation and changes; interaction creation, resolution, expiry and supersession; issue creation, updates, blocker relations, checkout, adoption and release; agent hire, configuration, lifecycle, error clearing and deletion; budget incidents and company pause/resume; exhausted-run events and retry-run creation; native status, review, finalization and reconstruction; external conversation transitions; and imported blocker relations. Existing service transactions retain outer commit ownership. Savepoint release does not publish work before outer commit. diff --git a/server/src/__tests__/attention-retention-work.test.ts b/server/src/__tests__/attention-retention-work.test.ts index 275b4f64570..fb2dca6f6b5 100644 --- a/server/src/__tests__/attention-retention-work.test.ts +++ b/server/src/__tests__/attention-retention-work.test.ts @@ -377,6 +377,37 @@ const delay = DEFAULT_DECISION_ARCHIVE_DAYS * 86_400_000; expect(await db.select().from(decisionArchiveNotificationOutbox)).toHaveLength(0); }); + it.each(["in_progress", "run", "confirmation"])("preserves observed child live evidence (%s) after explicit blockers fill the node budget", async (kind) => { + const root = await insertIssue("blocked"); + await db.execute(sql`with blockers as ( + insert into issues (id, company_id, title, status, assignee_agent_id, updated_at) + select gen_random_uuid(), ${companyId}::uuid, 'explicit-leaf-' || n, 'backlog', ${agentId}::uuid, ${old().toISOString()}::timestamptz + from generate_series(1, 1999) n returning id) + insert into issue_relations (company_id, issue_id, related_issue_id, type) + select ${companyId}::uuid, id, ${root.id}::uuid, 'blocks' from blockers`); + const child = await insertIssue(kind === "in_progress" ? "in_progress" : "todo", { parentId: root.id, assigneeAgentId: agentId }); + if (kind === "run") { + const [run] = await db.insert(heartbeatRuns).values({ companyId, agentId, status: "running" }).returning(); + await db.update(issues).set({ executionRunId: run.id }).where(eq(issues.id, child.id)); + } else if (kind === "confirmation") { + await db.insert(issueThreadInteractions).values({ companyId, issueId: child.id, kind: "request_confirmation", status: "pending", payload: { prompt: "Approve?" } }); + } + parameterCounts.length = 0; + const batch = await issueService(db).listBlockerAttentionBatch(companyId, [root]); + expect(batch.nodeCount).toBeLessThanOrEqual(BLOCKER_ATTENTION_MAX_NODES); + expect(batch.edgeCount).toBeLessThanOrEqual(BLOCKER_ATTENTION_MAX_NODES * 2); + expect(batch.fetchedRows).toBeLessThanOrEqual(BLOCKER_ATTENTION_MAX_NODES * 6); + expect(parameterCounts.length).toBeLessThanOrEqual(100); + expect(Math.max(...parameterCounts)).toBeLessThanOrEqual(1100); + expect(batch.deferredRootIds.has(root.id)).toBe(true); + expect(batch.attention.get(root.id)?.blockingTreeLive).toBe(true); + const service = attentionRetentionService(db), page = await service.load({ companyId, sourceKind: "blocker_attention" }); + const entry = page.rows!.find(item => item.rootId === root.id && item.variant === "terminal")!; + expect(await service.check(entry)).toBeNull(); + expect(await db.select().from(decisionRetention)).toHaveLength(0); + expect(await db.select().from(decisionArchiveNotificationOutbox)).toHaveLength(0); + }); + it("disarms finite deferred-root checks when large retained graphs have no archive obligation", async () => { await db.execute(sql`insert into issues (id, company_id, title, status, updated_at) select gen_random_uuid(), ${companyId}::uuid, 'held-root-' || n, 'blocked', ${old().toISOString()}::timestamptz diff --git a/server/src/services/issues.ts b/server/src/services/issues.ts index 4d5fd5f85f6..b8f78009a59 100644 --- a/server/src/services/issues.ts +++ b/server/src/services/issues.ts @@ -3850,6 +3850,10 @@ async function readIssueBlockerAttentionBatch( const nodesById = new Map(); const edgesByIssueId = new Map(); + // Returned rows are already bounded by the residual fetch budget. Keep their + // positive live-work evidence even when they cannot enter the traversal graph. + const observedNodes = new Map(); + const observedChildrenByParent = new Map>(); const admittedRoots = roots.slice(0, BLOCKER_ATTENTION_MAX_NODES); for (const root of admittedRoots) nodesById.set(root.id, { ...root }); @@ -3946,6 +3950,10 @@ async function readIssueBlockerAttentionBatch( ); for (const row of [...unresolvedExplicitBlockerRows, ...childRows]) { if (!row.issueId) continue; + observedNodes.set(row.blockerIssueId, row); + const observed = observedChildrenByParent.get(row.issueId) ?? new Set(); + observed.add(row.blockerIssueId); + observedChildrenByParent.set(row.issueId, observed); if (!nodesById.has(row.blockerIssueId)) { if (nodesById.size >= BLOCKER_ATTENTION_MAX_NODES) { complete = false; continue; } nodesById.set(row.blockerIssueId, { @@ -3993,11 +4001,12 @@ async function readIssueBlockerAttentionBatch( if (traversal.frontier.length > 0) traversal.truncated = true; } - const nodeIds = [...nodesById.keys()]; + const evidenceNodes = new Map([...nodesById, ...observedNodes]); + const nodeIds = [...evidenceNodes.keys()]; const activeIssueIds = new Set(); const agentIds = new Set(); const issueIdByExecutionRunId = new Map(); - for (const node of nodesById.values()) { + for (const node of evidenceNodes.values()) { if (node.assigneeAgentId) agentIds.add(node.assigneeAgentId); if (node.executionRunId) issueIdByExecutionRunId.set(node.executionRunId, node.id); @@ -4055,7 +4064,7 @@ async function readIssueBlockerAttentionBatch( } } - const explicitWaitCandidateIds = [...nodesById.values()] + const explicitWaitCandidateIds = [...evidenceNodes.values()] .filter((node) => node.status !== "done") .map((node) => node.id); const explicitWaitingIssueIds = new Set(); @@ -4400,19 +4409,22 @@ async function readIssueBlockerAttentionBatch( }; }; + const nodeHasLiveWork = (node: IssueBlockerAttentionNode) => + node.status === "in_progress" || activeIssueIds.has(node.id) || + explicitWaitingIssueIds.has(node.id) || Boolean(node.assigneeUserId); + const observedChildHasLiveWork = (parentId: string) => + [...observedChildrenByParent.get(parentId) ?? []].some(id => { + const node = observedNodes.get(id); + return node?.companyId === companyId && nodeHasLiveWork(node); + }); const pathHasLiveWork = (nodeId: string, seen: Set, traversal: Traversal): boolean => { if (seen.has(nodeId) || !traversal.nodes.has(nodeId)) return false; const node = nodesById.get(nodeId); if (!node || node.companyId !== companyId) return false; - if ( - node.status === "in_progress" || - activeIssueIds.has(node.id) || - explicitWaitingIssueIds.has(node.id) || - Boolean(node.assigneeUserId) - ) - return true; + if (nodeHasLiveWork(node)) return true; if (!traversal.expanded.has(nodeId)) return false; + if (observedChildHasLiveWork(nodeId)) return true; const nextSeen = new Set(seen); nextSeen.add(nodeId); return (edgesByIssueId.get(node.id) ?? []).some((edge) => { @@ -4539,7 +4551,7 @@ async function readIssueBlockerAttentionBatch( stalledEntry?.result.sampleStalledBlockerIdentifier ?? sampleStalledFromChain ?? null, - blockingTreeLive: topLevelEdges.some((edge) => + blockingTreeLive: observedChildHasLiveWork(root.id) || topLevelEdges.some((edge) => pathHasLiveWork(edge.blockerIssueId, new Set([root.id]), traversal), ), directBlockerIssueId: sampleEntry?.edge.blockerIssueId ?? null, From 255713e0cd0505a471cc9f0d264a3891da31a47c Mon Sep 17 00:00:00 2001 From: Dotta Date: Fri, 30 Oct 2026 20:02:36 -0500 Subject: [PATCH 67/74] fix(server): bound and coalesce issue retention invalidations Bound reverse ancestor discovery before materializing adjacency, deduplicate cycles and emit one canonical affected-issue batch per write. Overflow delegates complete company reconstruction to existing paging instead of dropping eligibility or creating per-ancestor worker debt. Prove real high-fanin producer bursts, PostgreSQL execution limits, canonical cyclic batches and far ancestor archive deadlines. Co-Authored-By: Paperclip --- .../2026-10-30-attention-retention-sleep.md | 2 + .../attention-retention-work.test.ts | 107 +++++++++++++++++- server/src/services/attention-retention.ts | 12 +- .../services/retention-work-notifications.ts | 29 +++-- 4 files changed, 134 insertions(+), 16 deletions(-) diff --git a/doc/plans/2026-10-30-attention-retention-sleep.md b/doc/plans/2026-10-30-attention-retention-sleep.md index 2be011ccf9d..92241aefc4f 100644 --- a/doc/plans/2026-10-30-attention-retention-sleep.md +++ b/doc/plans/2026-10-30-attention-retention-sleep.md @@ -46,3 +46,5 @@ The focused suite supports `PAPERCLIP_ATTENTION_RETENTION_TEST_DATABASE_URL`, va Local embedded PostgreSQL could not start on the test host because system semaphores were exhausted. The accepted runs use isolated PostgreSQL 17 databases. A mixed consumer run also exposed fixture contamination from a lifecycle suite that retains native finalization rows; isolated reruns are required and preserve the failed run in the review evidence. Broad Linux CI remains the required full-suite gate. Review the source parity fixtures and producer hints together. The intended outcome is unchanged archive state and notifications with no recurring retention SQL when the sources remain unchanged. No merge or deployment is part of this change. + +Issue producer invalidation has its own finite discovery budget. Reverse parent/blocker traversal returns at most 101 distinct UUIDs in one recursive query, with each relation adjacency capped at 101 unique edges before materialization. At most 100 affected issues produce one sorted canonical batch hint; an overflow sentinel produces one company hint and the existing complete paged reconstruction. UNION deduplication terminates cycles. A PostgreSQL EXPLAIN ANALYZE on 2,100 reverse edges proves the outer limit stops at 101 CTE rows while the indexed relation branch returns only 100 rows. Repeated identical writes coalesce by canonical key. The shared scheduler is unchanged: a broader company hint does not cancel already pending narrower scopes; those remain finite batches of at most 100 IDs, processed at the existing 25-scope pass limit. This permits redundant bounded reconstruction instead of discarding invalidations or introducing cursor/subsumption policy across other owners. A 151-node cyclic chain proves overflow still registers the far ancestor's exact archive deadline and settles quietly afterward. diff --git a/server/src/__tests__/attention-retention-work.test.ts b/server/src/__tests__/attention-retention-work.test.ts index fb2dca6f6b5..f06355e1ae7 100644 --- a/server/src/__tests__/attention-retention-work.test.ts +++ b/server/src/__tests__/attention-retention-work.test.ts @@ -1,5 +1,6 @@ import { randomUUID } from "node:crypto"; -import { and, eq, sql } from "drizzle-orm"; +import { and, eq, sql, type SQL } from "drizzle-orm"; +import { PgDialect } from "drizzle-orm/pg-core"; import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; import { subscribeDatabaseWork, authUsers, companyMemberships, agents, approvals, budgetIncidents, budgetPolicies, companies, createDb, applyPendingMigrations, decisionArchiveNotificationOutbox, decisionRetention, decisions, costEvents, heartbeatRunEvents, heartbeatRuns, issueApprovals, issueRecoveryActions, @@ -425,6 +426,106 @@ const delay = DEFAULT_DECISION_ARCHIVE_DAYS * 86_400_000; expect(await db.select().from(decisionArchiveNotificationOutbox)).toHaveLength(0); }); + it("bounds high-fanin ordinary issue writes to one coalescible hint", async () => { + const service = attentionRetentionService(db), original = service.load, load = vi.spyOn(service, "load"); + await start(service); await settle(); load.mockClear(); + const leaf = await insertIssue("in_review", { updatedAt: new Date(), assigneeUserId: "board" }); + await db.execute(sql`with parents as ( + insert into issues (id, company_id, title, status) + select gen_random_uuid(), ${companyId}::uuid, 'fan-in-' || n, 'todo' + from generate_series(1, 2100) n returning id) + insert into issue_relations (company_id, issue_id, related_issue_id, type) + select ${companyId}::uuid, ${leaf.id}::uuid, id, 'blocks' from parents`); + const hints: string[] = []; + const unsubscribe = subscribeDatabaseWork(db, DELIVERY_QUEUES.attentionRetention, (event, values) => { + if (event === "settled") hints.push(...values ?? []); + }); + let release!: () => void, entered = false; + const gate = new Promise(resolve => { release = resolve; }); + load.mockImplementationOnce(async (...args) => { const page = await original(...args); entered = true; await gate; return page; }); + statements.length = 0; + try { + await issueService(db).update(leaf.id, { title: "High fan-in ordinary edit 0" }); + await vi.waitFor(() => expect(entered).toBe(true), { timeout: 15_000 }); + for (let n = 1; n < 8; n++) await issueService(db).update(leaf.id, { title: `High fan-in ordinary edit ${n}` }); + } finally { release(); unsubscribe(); } + expect(statements.filter(query => query.includes("with recursive affected"))).toHaveLength(8); + expect(hints).toHaveLength(8); + expect(new Set(hints)).toEqual(new Set([JSON.stringify({ companyId })])); + await settle(); + expect(load.mock.calls.length).toBeLessThanOrEqual(3); + expect(load.mock.calls.every(([scope]) => scope?.companyId === companyId && !scope.issueId && !scope.issueIds)).toBe(true); + let query!: Parameters[0], returned = 0; + await db.transaction(async tx => { + const execute = tx.execute.bind(tx); + vi.spyOn(tx, "execute").mockImplementation(async statement => { + const result = await execute(statement); + if (new PgDialect().sqlToQuery(statement as SQL).sql.includes("with recursive affected")) { + query = statement; returned = result.length; + } + return result; + }); + await notifyIssueAttentionRetentionWork(tx, { companyId, issueId: leaf.id }); + }); + expect(returned).toBe(101); + const plans = await db.execute(sql`explain (analyze, format json) ${query as SQL}`); + const plan = (plans[0]["QUERY PLAN"] as any)[0].Plan; + expect(plan["Node Type"]).toBe("Limit"); + expect(plan["Actual Rows"]).toBe(101); + const nodes: any[] = [], collect = (node: any) => { nodes.push(node); for (const child of node.Plans ?? []) collect(child); }; + collect(plan); + const recursive = nodes.filter(node => node["Node Type"] === "Recursive Union"); + const relations = nodes.filter(node => node["Relation Name"] === "issue_relations"); + expect(recursive).toHaveLength(1); + expect(recursive[0]["Actual Rows"]).toBe(101); + expect(relations.length).toBeGreaterThan(0); + expect(relations.every(node => node["Actual Rows"] * node["Actual Loops"] <= 101)).toBe(true); + process.stdout.write(JSON.stringify({ reverseFanInPlan: plan }) + "\n"); + }); + + it("retains far ancestor deadlines when a cyclic chain exceeds the precise hint budget", async () => { + const service = attentionRetentionService(db), load = vi.spyOn(service, "load"); + await start(service); await settle(); load.mockClear(); + await db.execute(sql`insert into issues (id, company_id, title, status, updated_at, assignee_user_id) + select gen_random_uuid(), ${companyId}::uuid, 'chain-' || n, + case when n = 150 then 'in_review' else 'todo' end, + case when n = 150 then ${new Date(Date.now() - delay + 15_000).toISOString()}::timestamptz else now() end, + case when n = 150 then 'board' else null end + from generate_series(0, 150) n`); + await db.execute(sql`update issues child set parent_id = parent.id from issues parent + where child.company_id = ${companyId}::uuid and parent.company_id = child.company_id + and parent.title = 'chain-' || ((substring(child.title from 7)::integer + 1) % 151)`); + const [leaf, far] = await Promise.all([db.select().from(issues).where(and(eq(issues.companyId, companyId), eq(issues.title, "chain-0"))).then(rows => rows[0]), + db.select().from(issues).where(and(eq(issues.companyId, companyId), eq(issues.title, "chain-150"))).then(rows => rows[0])]); + await issueService(db).update(leaf.id, { title: "Ordinary deep chain edit" }); + await settle(); + expect(load.mock.calls.some(([scope]) => scope?.companyId === companyId && !scope.issueId && !scope.issueIds)).toBe(true); + expect(await state("review", far.id)).toBeNull(); + expect(coordinator!.nextWakeAt()).not.toBeNull(); + await archived("review", far.id); + await quiet(); + }); + + it("canonicalizes a complete cyclic affected set into one bounded batch hint", async () => { + const a = await insertIssue(), b = await insertIssue(), c = await insertIssue(); + await db.update(issues).set({ parentId: b.id }).where(eq(issues.id, a.id)); + await db.update(issues).set({ parentId: c.id }).where(eq(issues.id, b.id)); + await db.update(issues).set({ parentId: a.id }).where(eq(issues.id, c.id)); + const hints: string[] = []; + const unsubscribe = subscribeDatabaseWork(db, DELIVERY_QUEUES.attentionRetention, (event, values) => { + if (event === "settled") hints.push(...values ?? []); + }); + try { + await db.transaction(tx => notifyIssueAttentionRetentionWork(tx, { companyId, issueId: a.id })); + await db.transaction(tx => notifyIssueAttentionRetentionWork(tx, { companyId, issueId: c.id })); + const wide = { companyId, issueIds: [b.id, a.id, c.id, a.id], ignored: {} as unknown }; + wide.ignored = wide; + await db.transaction(tx => notifyAttentionRetentionWork(tx, wide)); + } finally { unsubscribe(); } + expect(hints).toHaveLength(3); + expect(new Set(hints)).toEqual(new Set([JSON.stringify({ companyId, issueIds: [a.id, b.id, c.id].sort() })])); + }); + it("canonicalizes wide circular notification variables to the supported identity fields", async () => { const sourceId = randomUUID(), issueId = randomUUID(), hints: string[] = []; const wide = { companyId, sourceKind: "review" as const, sourceId, issueId, ignored: {} as unknown }; @@ -487,14 +588,14 @@ const delay = DEFAULT_DECISION_ARCHIVE_DAYS * 86_400_000; const terminal = await insertIssue("backlog", { parentId: oldParent.id, hiddenAt: old(), assigneeAgentId: agentId, responsibleUserId: "board" }); const service = attentionRetentionService(db), load = vi.spyOn(service, "load"); await start(service); await settle(); load.mockClear(); await issueService(db).update(terminal.id, { parentId: newParent.id }); await settle(); - const ids = load.mock.calls.map(([scope]) => scope?.issueId); + const ids = load.mock.calls.flatMap(([scope]) => scope?.issueIds ?? (scope?.issueId ? [scope.issueId] : [])); expect(ids).toEqual(expect.arrayContaining([oldParent.id, newParent.id, terminal.id])); load.mockClear(); await issueService(db).update(terminal.id, { hiddenAt: null }); await settle(); expect(await state("blocker_attention", terminal.id)).toBeNull(); const visible = await service.load({ companyId, issueId: newParent.id }); expect(visible.rows!.some(entry => entry.sourceId === terminal.id && entry.dueAt > Date.now())).toBe(true); - expect(load.mock.calls.some(([scope]) => scope?.issueId === newParent.id)).toBe(true); + expect(load.mock.calls.some(([scope]) => scope?.issueId === newParent.id || scope?.issueIds?.includes(newParent.id))).toBe(true); const next = await insertIssue("backlog", { assigneeAgentId: agentId }); await issueService(db).update(oldParent.id, { blockedByIssueIds: [next.id] }); await archived("blocker_attention", next.id); }); diff --git a/server/src/services/attention-retention.ts b/server/src/services/attention-retention.ts index ebe91416f52..a0bca4281f5 100644 --- a/server/src/services/attention-retention.ts +++ b/server/src/services/attention-retention.ts @@ -1,7 +1,6 @@ import { and, eq, getTableColumns, inArray, isNull, sql, type SQL } from "drizzle-orm"; import { agents, decisionRetention, decisions, heartbeatRuns, issueApprovals, issueRecoveryActions, issueThreadInteractions, issues, approvals, type Db } from "@paperclipai/db"; -import type { AttentionSourceKind } from "@paperclipai/shared"; import { evaluateAgentInvokability } from "./agent-invokability.js"; import { HUMAN_RECOVERY_OWNER_TYPES, ATTENTION_BUDGET_WARNING_PERCENT, ATTENTION_AGENT_ERROR_STATUS, collapsePendingConfirmationsToNewest, listFailedRunsWithNewerRun, readRunIssueId, requiresReviewAttention, terminalBlockerRequiresAttention } from "./attention-source-eligibility.js"; import { decisionRetentionService, DEFAULT_DECISION_ARCHIVE_DAYS } from "./decision-retention.js"; @@ -13,12 +12,13 @@ import { issueService } from "./issues.js"; import { parseIssueExecutionState } from "./issue-execution-policy.js"; import { isProspectiveBlockedTransition } from "./routable-blocked.js"; import { registerScopedDeadlineWork } from "./scoped-deadline-work.js"; +import type { AttentionRetentionScope } from "./retention-work-notifications.js"; const archiveDelay = DEFAULT_DECISION_ARCHIVE_DAYS * 86_400_000; const kinds = ["decision", "issue_thread_interaction", "recovery_action", "blocker_attention", "review", "failed_run", "budget_alert", "agent_error_alert"] as const; type Kind = typeof kinds[number]; -type Scope = { companyId?: string; sourceKind?: AttentionSourceKind; sourceId?: string; issueId?: string }; +type Scope = AttentionRetentionScope; type Candidate = { sourceKind: Kind; sourceId: string; companyId: string; activityAt: string; deadlineAt: number; issueId: string | null }; type Entry = Candidate & { id: string; dueAt: number; rootId: string; variant?: "owner" | "terminal" }; type Cursor = { sourceKind: string; sourceId: string }; @@ -66,8 +66,9 @@ const sources: Record = { export function attentionRetentionService(db: Db) { const cursors = new Map(); async function candidates(database: Db, scope: Scope = {}, cursor?: Cursor, sourceId?: string): Promise { + const issueIds = scope.issueIds ?? (scope.issueId ? [scope.issueId] : []); const selected = kinds.filter(kind => (!scope.sourceKind || scope.sourceKind === kind) - && (!scope.issueId || !["budget_alert", "agent_error_alert"].includes(kind))); + && (!issueIds.length || !["budget_alert", "agent_error_alert"].includes(kind))); if (!selected.length) return []; const rows = await database.execute(sql`with source_candidates(source_kind, source_id, company_id, activity_at, issue_id) as (${sql.join(selected.map(kind => sources[kind]), sql` union all `)}) select s.source_kind as "sourceKind", s.source_id as "sourceId", s.company_id as "companyId", @@ -75,7 +76,7 @@ export function attentionRetentionService(db: Db) { from source_candidates s inner join companies c on c.id = s.company_id left join decision_retention r on r.company_id = s.company_id and r.source_kind = s.source_kind and r.source_id = s.source_id where c.status = 'active' - and (${scope.issueId ?? null}::text is null or s.issue_id = ${scope.issueId ?? null}) + ${issueIds.length ? sql`and s.issue_id in (${sql.join(issueIds.map(id => sql`${id}`), sql`, `)})` : sql``} and (${scope.companyId ?? null}::uuid is null or s.company_id = ${scope.companyId ?? null}::uuid) and (${sourceId ?? scope.sourceId ?? null}::text is null or s.source_id = ${sourceId ?? scope.sourceId ?? null}) and (s.source_kind = 'blocker_attention' or ((r.id is null or r.keep = false) and r.archived_at is null)) @@ -312,7 +313,8 @@ export function registerAttentionRetentionWork(coordinator: ReturnType (!scope?.companyId || scope.companyId === entry.companyId) && (!scope?.sourceKind || scope.sourceKind === entry.sourceKind) && (!scope?.sourceId || scope.sourceId === entry.sourceId) - && (!scope?.issueId || scope.issueId === entry.issueId), + && (!scope?.issueId || scope.issueId === entry.issueId) + && (!scope?.issueIds?.length || entry.issueId !== null && scope.issueIds.includes(entry.issueId)), load: service.load, check: service.check, }, canRun); } diff --git a/server/src/services/retention-work-notifications.ts b/server/src/services/retention-work-notifications.ts index ba0379afee1..79cffd624a8 100644 --- a/server/src/services/retention-work-notifications.ts +++ b/server/src/services/retention-work-notifications.ts @@ -5,9 +5,16 @@ import type { Db } from "@paperclipai/db"; import type { AttentionSourceKind } from "@paperclipai/shared"; import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js"; -export async function notifyAttentionRetentionWork(transaction: object, scope: { companyId?: string; sourceKind?: AttentionSourceKind; issueId?: string; sourceId?: string } = {}) { +export const ATTENTION_RETENTION_MAX_ISSUE_IDS = 100; +export type AttentionRetentionScope = { companyId?: string; sourceKind?: AttentionSourceKind; + issueId?: string; issueIds?: string[]; sourceId?: string }; + +export async function notifyAttentionRetentionWork(transaction: object, scope: AttentionRetentionScope = {}) { + const issueIds = [...new Set([...(scope.issueIds ?? []), ...(scope.issueId ? [scope.issueId] : [])])].sort(); await notifyDeliveryWork(transaction, DELIVERY_QUEUES.attentionRetention, JSON.stringify({ - companyId: scope.companyId, sourceKind: scope.sourceKind, sourceId: scope.sourceId, issueId: scope.issueId, + companyId: scope.companyId, sourceKind: scope.sourceKind, sourceId: scope.sourceId, + issueId: issueIds.length === 1 ? issueIds[0] : undefined, + issueIds: issueIds.length > 1 && issueIds.length <= ATTENTION_RETENTION_MAX_ISSUE_IDS ? issueIds : undefined, })); } @@ -47,18 +54,24 @@ export async function notifyIssueAttentionRetentionWork(transaction: object, and id::text in (${sql.join([...new Set(references)].map(id => sql`${id}`), sql`, `)})`); for (const row of trusted) seeds.push(row.id as string); } + // UNION deduplicates visited UUIDs (including cycles). Each relation edge is + // unique by schema, so 101 neighbours guarantee overflow of a 100-ID scope. + // Bound that adjacency before materialization and stream at most 101 unique + // IDs to the caller; an inner UNION DISTINCT would consume the full fan-in. const affected = await database.execute(sql`with recursive affected(id) as ( select unnest(array[${sql.join([...new Set(seeds)].map(id => sql`${id}::uuid`), sql`, `)}]) union select parent.id from affected current cross join lateral ( - select related_issue_id as id from issue_relations where company_id = ${scope.companyId}::uuid - and issue_id = current.id and type = 'blocks' - union select parent_id from issues where company_id = ${scope.companyId}::uuid + (select related_issue_id as id from issue_relations where company_id = ${scope.companyId}::uuid + and issue_id = current.id and type = 'blocks' limit ${ATTENTION_RETENTION_MAX_ISSUE_IDS + 1}) + union all select parent_id from issues where company_id = ${scope.companyId}::uuid and id = current.id and parent_id is not null ) parent - ) select id::text as "issueId" from affected`); - for (const row of affected) await notifyAttentionRetentionWork(transaction, { - companyId: scope.companyId, issueId: row.issueId as string, + ) select id::text as "issueId" from affected limit ${ATTENTION_RETENTION_MAX_ISSUE_IDS + 1}`); + // The sentinel means the affected set is incomplete. Reconstruct the whole + // company through the existing paged authority instead of dropping ancestors. + await notifyAttentionRetentionWork(transaction, { companyId: scope.companyId, + issueIds: affected.length <= ATTENTION_RETENTION_MAX_ISSUE_IDS ? affected.map(row => row.issueId as string) : undefined, }); } From d48bb081d090df0a4ec7c0103a8606bedcfa48ff Mon Sep 17 00:00:00 2001 From: Dotta Date: Fri, 30 Oct 2026 20:18:34 -0500 Subject: [PATCH 68/74] fix(ui): retain startup autosave through session recovery Co-Authored-By: Paperclip --- tests/e2e/disconnect-write-recovery.spec.ts | 23 +++++++- ui/src/hooks/useAccountBoundMutation.ts | 60 +-------------------- ui/src/hooks/useDurableAutosave.test.tsx | 45 ++++++++++++++++ ui/src/hooks/useDurableAutosave.ts | 57 +++++++++++++------- ui/src/lib/account-scope.ts | 59 ++++++++++++++++++++ 5 files changed, 167 insertions(+), 77 deletions(-) create mode 100644 ui/src/lib/account-scope.ts diff --git a/tests/e2e/disconnect-write-recovery.spec.ts b/tests/e2e/disconnect-write-recovery.spec.ts index 8f81ee178dc..780f6f27a04 100644 --- a/tests/e2e/disconnect-write-recovery.spec.ts +++ b/tests/e2e/disconnect-write-recovery.spec.ts @@ -178,10 +178,19 @@ test("replaying an accepted comment does not reopen completed work", async ({ pa } }); -test("startup work retries a failed session lookup while other APIs stay healthy", async ({ page, request }) => { +for (const delayWorkspacePreference of [false, true]) test(delayWorkspacePreference + ? "startup title survives session recovery while workspace preferences remount the editor" + : "startup work retries a failed session lookup while other APIs stay healthy", async ({ page, request }) => { const company = await json(await request.post("/api/companies", { data: { name: `Session recovery ${Date.now()}` } })); const session = await json(await request.get("/api/auth/get-session")); let healthy = false, saving = false, failedAfterSave = 0; + let releasePreference!: () => void; + const preferenceGate = new Promise(resolve => { releasePreference = resolve; }); + let sawPreference!: () => void; + const preferenceRequested = new Promise(resolve => { sawPreference = resolve; }); + if (delayWorkspacePreference) await page.route("**/api/workspace-preferences/me", async route => { + sawPreference(); await preferenceGate; await route.continue(); + }); try { const issue = await json(await request.post(`/api/companies/${company.id}/issues`, { data: { title: "Before session recovery", status: "backlog" }, @@ -199,12 +208,24 @@ test("startup work retries a failed session lookup while other APIs stay healthy saving = true; await header.locator("textarea").press("Enter"); await expect.poll(() => failedAfterSave).toBeGreaterThan(0); + if (delayWorkspacePreference) { + await expect(page.getByText("Couldn't update the task", { exact: true })).toBeVisible(); + await page.evaluate(() => new Promise(resolve => requestAnimationFrame(() => resolve(null)))); + } healthy = true; + if (delayWorkspacePreference) { + // Returning to the tab retries session lookup while the saved edit waits. + await page.evaluate(() => window.dispatchEvent(new Event("visibilitychange"))); + await preferenceRequested; + await expect(page.getByTestId("task-chat-history-content")).toBeHidden(); + releasePreference(); + } await expect.poll(async () => (await json(await request.get(`/api/issues/${issue.id}`))).title, { timeout: 15_000 }).toBe("Saved after session recovery"); await page.reload(); await expect(header.getByRole("heading", { name: "Saved after session recovery", exact: true })).toBeVisible(); } finally { + releasePreference(); await request.patch(`/api/companies/${company.id}`, { data: { status: "archived" } }); } }); diff --git a/ui/src/hooks/useAccountBoundMutation.ts b/ui/src/hooks/useAccountBoundMutation.ts index b011b9ec8ad..5cbf156b703 100644 --- a/ui/src/hooks/useAccountBoundMutation.ts +++ b/ui/src/hooks/useAccountBoundMutation.ts @@ -5,68 +5,12 @@ import { QueryClientContext, type DefaultError, type MutateOptions, - type QueryClient, type UseMutationOptions, type UseMutationResult, type MutationFunctionContext, } from "@tanstack/react-query"; -import { queryKeys } from "@/lib/queryKeys"; -import { authApi } from "@/api/auth"; -import { classifySendFailure, PendingSendAccountChangedError } from "@/lib/pending-send"; - -type AccountGeneration = { owner: string | undefined; generation: number }; -const accounts = new WeakMap(); - -function currentOwner(client: QueryClient): string | undefined { - const session = client.getQueryData<{ user?: { id?: string }; session?: { userId?: string } } | null>(queryKeys.auth.session); - return session === undefined ? undefined : session?.user?.id ?? session?.session?.userId ?? "signed-out"; -} - -function bindAccount(client: QueryClient) { - let account = accounts.get(client); - if (!account) { - account = { owner: currentOwner(client), generation: 0 }; - accounts.set(client, account); - const tracked = account; - client.getQueryCache().subscribe(event => { - const owner = currentOwner(client); - const removed = event.type === "removed" && JSON.stringify(event.query.queryKey) === JSON.stringify(queryKeys.auth.session); - if (owner !== tracked.owner || removed) { - if (removed || tracked.owner !== undefined || tracked.generation !== 0) tracked.generation += 1; - tracked.owner = owner; - } - }); - } - const { owner, generation } = account; - const isCurrentScope = () => account.generation === generation && (owner === undefined ? generation === 0 : account.owner === owner); - const isCurrentAccount = () => account.owner !== undefined && isCurrentScope(); - const whenAccountReady = async () => { - if (account.owner !== undefined || !isCurrentScope()) return; - let stopped = false; - let unsubscribe = () => {}; - const ready = new Promise(resolve => { - unsubscribe = client.getQueryCache().subscribe(() => { - if (account.owner !== undefined || !isCurrentScope()) resolve(); - }); - }); - const lookup = async () => { - for (let attempt = 0; attempt < 3; attempt += 1) { - if (stopped || account.owner !== undefined || !isCurrentScope()) return; - try { - await client.fetchQuery({ queryKey: queryKeys.auth.session, queryFn: () => authApi.getSession(), retry: false }); - return; - } catch (error) { - if (stopped || account.owner !== undefined || !isCurrentScope()) return; - if (attempt === 2 || classifySendFailure(error) !== "pending") throw error; - await new Promise(resolve => setTimeout(resolve, 250 * 2 ** attempt)); - } - } - }; - try { await Promise.race([ready, lookup()]); } - finally { stopped = true; unsubscribe(); } - }; - return { isCurrentAccount, isCurrentScope, whenAccountReady }; -} +import { PendingSendAccountChangedError } from "@/lib/pending-send"; +import { bindAccount } from "@/lib/account-scope"; export function useCurrentAccountGuard(): () => boolean { const client = useContext(QueryClientContext); diff --git a/ui/src/hooks/useDurableAutosave.test.tsx b/ui/src/hooks/useDurableAutosave.test.tsx index edad11f04a0..af5860e0a85 100644 --- a/ui/src/hooks/useDurableAutosave.test.tsx +++ b/ui/src/hooks/useDurableAutosave.test.tsx @@ -173,6 +173,51 @@ describe("useDurableAutosave", () => { client.clear(); }); + it.each([false, true])("keeps an unowned startup edit when identity and unmount race (identityFirst=%s)", async identityFirst => { + store.setBrowserOnline(false); + const client = new QueryClient({ defaultOptions: { queries: { staleTime: Infinity, retry: false, enabled: false } } }); + const save = vi.fn().mockResolvedValue(undefined); + act(() => root.render( + + )); + await act(async () => { await current.run("Retained startup title"); }); + expect(current.status).toBe("waiting"); + act(() => { + if (identityFirst) client.setQueryData(queryKeys.auth.session, { user: { id: "user-1" } }); + root.unmount(); root = createRoot(container); + if (!identityFirst) client.setQueryData(queryKeys.auth.session, { user: { id: "user-1" } }); + }); + await act(async () => { store.setBrowserOnline(true); store.reportSuccess(); }); + await advance(2_000); + expect(save).toHaveBeenCalledExactlyOnceWith("Retained startup title"); + expect(store.getSnapshot().pendingWrites).toBe(0); + client.clear(); + }); + + it.each(["different-account", "round-trip", "removed-session", "signed-out"])( + "does not adopt a later identity for a detached startup edit (%s)", async transition => { + store.setBrowserOnline(false); + const client = new QueryClient({ defaultOptions: { queries: { staleTime: Infinity, retry: false, enabled: false } } }); + const save = vi.fn().mockResolvedValue(undefined); + act(() => root.render( + + )); + await act(async () => { await current.run("Private startup title"); }); + act(() => { root.unmount(); root = createRoot(container); }); + act(() => { + if (transition === "removed-session") client.removeQueries({ queryKey: queryKeys.auth.session }); + else client.setQueryData(queryKeys.auth.session, transition === "signed-out" ? null : { user: { id: "user-1" } }); + if (transition === "round-trip") client.setQueryData(queryKeys.auth.session, { user: { id: "user-2" } }); + client.setQueryData(queryKeys.auth.session, { user: { id: transition === "different-account" ? "user-2" : "user-1" } }); + }); + await act(async () => { store.setBrowserOnline(true); store.reportSuccess(); }); + await advance(2_000); + expect(save).not.toHaveBeenCalled(); + expect(store.getSnapshot().pendingWrites).toBe(0); + client.clear(); + }, + ); + it("bounds retries after an editor unmounts", async () => { const save = vi.fn().mockRejectedValue(outage()); render({ save }); diff --git a/ui/src/hooks/useDurableAutosave.ts b/ui/src/hooks/useDurableAutosave.ts index 658d68cd149..0293975cc8b 100644 --- a/ui/src/hooks/useDurableAutosave.ts +++ b/ui/src/hooks/useDurableAutosave.ts @@ -1,4 +1,6 @@ -import { useCallback, useEffect, useId, useRef, useState } from "react"; +import { useCallback, useContext, useEffect, useId, useRef, useState } from "react"; +import { QueryClientContext } from "@tanstack/react-query"; +import { bindAccount } from "@/lib/account-scope"; import { classifyError, describeError, type ErrorDescription } from "@/api/errors"; import { loadStructuredDraft, saveStructuredDraft } from "@/lib/composer-draft"; import { useConnectivityStore } from "@/lib/connectivity"; @@ -108,6 +110,7 @@ const SAVED_LINGER_MS = 1600; * - Errors become readable copy for "Couldn't save (reason)". */ export function useDurableAutosave(options: DurableAutosaveOptions): DurableAutosave { + const client = useContext(QueryClientContext); const owner = useDraftOwner(); const readOwner = useDraftOwnerReader(); const instanceId = useId(); @@ -120,7 +123,7 @@ export function useDurableAutosave(options: DurableAutosaveOptions): Durab const targetKey = options.targetKey ?? options.sourceId; const targetRef = useRef(targetKey); targetRef.current = targetKey; - type Job = { token: symbol; registryKey: string; resends: number; saving?: boolean; id: number; target: string; owner: string | undefined; draft: T; + type Job = { token: symbol; registryKey: string; resends: number; saving?: boolean; id: number; target: string; owner: string | undefined; account: ReturnType | null; draft: T; save: (draft: T) => Promise; matchesServer?: DurableAutosaveOptions["matchesServer"]; onRetryFailed?: DurableAutosaveJob["onRetryFailed"]; storageKey: string | null; action: string }; const jobs = useRef(new Map()); @@ -142,6 +145,22 @@ export function useDurableAutosave(options: DurableAutosaveOptions): Durab entry?.active.delete(job.token); if (entry && !entry.active.size) registry.delete(job.registryKey); }; + // Resolve first identity from the same generation guard used by mutations. + // This also runs after unmount, when React cannot migrate the job for us. + function owned(job: Job) { + if (job.account && !job.account.isCurrentScope()) return false; + const current = readOwner(); + if (job.owner === undefined && current !== undefined) { + const nextKey = JSON.stringify([current, job.target]); + if (registry.get(job.registryKey)?.latest !== job.token || registry.has(nextKey)) return false; + releaseJob(job); + job.owner = current; + job.registryKey = nextKey; + registry.set(nextKey, { latest: job.token, active: new Set([job.token]) }); + } + return current === job.owner && generations.current.get(job.target) === job.id + && registry.get(job.registryKey)?.latest === job.token; + } const mounted = useRef(true); const savingTimer = useRef | null>(null); const savedTimer = useRef | null>(null); @@ -160,17 +179,11 @@ export function useDurableAutosave(options: DurableAutosaveOptions): Durab if (previous === undefined && owner !== undefined) { for (const job of jobs.current.values()) { if (job.owner !== undefined) continue; - const nextKey = JSON.stringify([owner, job.target]); - const superseded = registry.get(job.registryKey)?.latest !== job.token || registry.has(nextKey); - releaseJob(job); - if (superseded) { + if (!owned(job)) { + releaseJob(job); jobs.current.delete(job.target); if (generations.current.get(job.target) === job.id) generations.current.delete(job.target); - continue; } - job.owner = owner; - job.registryKey = nextKey; - registry.set(nextKey, { latest: job.token, active: new Set([job.token]) }); } refresh(value => value + 1); return; @@ -194,12 +207,12 @@ export function useDurableAutosave(options: DurableAutosaveOptions): Durab return stored?.version === 1 && stored.draft !== undefined ? stored.draft : null; }, [storageKey]); const execute = useCallback(async (job: Job, retry = false): Promise => { - const owned = () => readOwner() === job.owner && generations.current.get(job.target) === job.id && registry.get(job.registryKey)?.latest === job.token; - const visible = () => mounted.current && owned() && targetRef.current === job.target; + const stillOwned = () => owned(job); + const visible = () => mounted.current && stillOwned() && targetRef.current === job.target; const publish = (status: DurableAutosaveStatus, error: ErrorDescription | null = null) => { if (visible()) setView({ target: job.target, status, error }); }; - if (!owned()) { + if (!stillOwned()) { if (jobs.current.get(job.target) === job) jobs.current.delete(job.target); releaseJob(job); if (mounted.current) refresh(value => value + 1); @@ -234,7 +247,12 @@ export function useDurableAutosave(options: DurableAutosaveOptions): Durab remove(); return { kind: "saved" }; }; - try { await job.save(job.draft); return done(); } + try { + if (!mounted.current && job.account && !job.account.isCurrentAccount()) await job.account.whenAccountReady(); + if (!stillOwned()) { remove(); return { kind: "waiting" }; } + await job.save(job.draft); + return done(); + } catch (initialCause) { let cause = initialCause; if (classifyError(cause) === "conflict" && job.matchesServer) { @@ -255,7 +273,7 @@ export function useDurableAutosave(options: DurableAutosaveOptions): Durab const current = optionsRef.current; const target = current.targetKey ?? current.sourceId; const admittedOwner = ownerRef.current ?? readOwner(); - const job: Job = { token: Symbol(), registryKey: JSON.stringify([admittedOwner, target]), resends: 0, id: ++sequence.current, target, owner: admittedOwner, draft, + const job: Job = { token: Symbol(), registryKey: JSON.stringify([admittedOwner, target]), resends: 0, id: ++sequence.current, target, owner: admittedOwner, account: client ? bindAccount(client) : null, draft, save: overrides.save ?? current.save, matchesServer: overrides.matchesServer ?? current.matchesServer, onRetryFailed: overrides.onRetryFailed, storageKey: current.storageKey ?? null, action: current.action ?? "save" }; const previous = jobs.current.get(target); @@ -266,7 +284,7 @@ export function useDurableAutosave(options: DurableAutosaveOptions): Durab generations.current.set(target, job.id); jobs.current.set(target, job); refresh(value => value + 1); return execute(job); - }, [execute, readOwner]); + }, [client, execute, readOwner]); const waitingJobs = [...jobs.current.values()].filter(job => !job.saving).length; const auto = useAutoResend({ sourceId: options.sourceId, active: waitingJobs > 0, pendingCount: waitingJobs, resend: async (): Promise => { @@ -281,7 +299,6 @@ export function useDurableAutosave(options: DurableAutosaveOptions): Durab detachedWork.current = () => { if (!jobs.current.size) return; if (auto.stalled) { for (const job of jobs.current.values()) releaseJob(job); jobs.current.clear(); return; } - const admittedOwner = ownerRef.current; const source = `${optionsRef.current.sourceId}:detached:${instanceId}`; let timer: ReturnType | null = null; let disposed = false, running = false; @@ -298,7 +315,11 @@ export function useDurableAutosave(options: DurableAutosaveOptions): Durab }; const wake = async () => { if (disposed || running) return; - if (mounted.current || readOwner() !== admittedOwner || admittedOwner === undefined) { release(); return; } + if (mounted.current) { release(); return; } + for (const job of jobs.current.values()) if (!owned(job)) { + releaseJob(job); + jobs.current.delete(job.target); + } if (!jobs.current.size) { release(); return; } store.setPendingWrites(source, jobs.current.size); if (store.getSnapshot().status !== "online") return; diff --git a/ui/src/lib/account-scope.ts b/ui/src/lib/account-scope.ts new file mode 100644 index 00000000000..0e7d050e502 --- /dev/null +++ b/ui/src/lib/account-scope.ts @@ -0,0 +1,59 @@ +import type { QueryClient } from "@tanstack/react-query"; +import { queryKeys } from "./queryKeys"; +import { authApi } from "@/api/auth"; +import { classifySendFailure } from "./pending-send"; + +type AccountGeneration = { owner: string | undefined; generation: number }; +const accounts = new WeakMap(); + +function currentOwner(client: QueryClient): string | undefined { + const session = client.getQueryData<{ user?: { id?: string }; session?: { userId?: string } } | null>(queryKeys.auth.session); + return session === undefined ? undefined : session?.user?.id ?? session?.session?.userId ?? "signed-out"; +} + +export function bindAccount(client: QueryClient) { + let account = accounts.get(client); + if (!account) { + account = { owner: currentOwner(client), generation: 0 }; + accounts.set(client, account); + const tracked = account; + client.getQueryCache().subscribe(event => { + const owner = currentOwner(client); + const removed = event.type === "removed" && JSON.stringify(event.query.queryKey) === JSON.stringify(queryKeys.auth.session); + if (owner !== tracked.owner || removed) { + if (removed || tracked.owner !== undefined || tracked.generation !== 0) tracked.generation += 1; + tracked.owner = owner; + } + }); + } + const { owner, generation } = account; + const isCurrentScope = () => account.generation === generation && (owner === undefined ? generation === 0 : account.owner === owner); + const isCurrentAccount = () => account.owner !== undefined && isCurrentScope(); + const whenAccountReady = async () => { + if (account.owner !== undefined || !isCurrentScope()) return; + let stopped = false; + let unsubscribe = () => {}; + const ready = new Promise(resolve => { + unsubscribe = client.getQueryCache().subscribe(() => { + if (account.owner !== undefined || !isCurrentScope()) resolve(); + }); + }); + const lookup = async () => { + for (let attempt = 0; attempt < 3; attempt += 1) { + if (stopped || account.owner !== undefined || !isCurrentScope()) return; + try { + await client.fetchQuery({ queryKey: queryKeys.auth.session, queryFn: () => authApi.getSession(), retry: false }); + return; + } catch (error) { + if (stopped || account.owner !== undefined || !isCurrentScope()) return; + if (attempt === 2 || classifySendFailure(error) !== "pending") throw error; + await new Promise(resolve => setTimeout(resolve, 250 * 2 ** attempt)); + } + } + }; + try { await Promise.race([ready, lookup()]); } + finally { stopped = true; unsubscribe(); } + }; + return { isCurrentAccount, isCurrentScope, whenAccountReady }; +} + From 8f58be556a566235779319cb664aa085d8afee1f Mon Sep 17 00:00:00 2001 From: Dotta Date: Fri, 30 Oct 2026 20:18:34 -0500 Subject: [PATCH 69/74] fix(ui): retain startup autosave through session recovery Co-Authored-By: Paperclip --- tests/e2e/disconnect-write-recovery.spec.ts | 23 +++++++- ui/src/hooks/useAccountBoundMutation.ts | 60 +-------------------- ui/src/hooks/useDurableAutosave.test.tsx | 45 ++++++++++++++++ ui/src/hooks/useDurableAutosave.ts | 57 +++++++++++++------- ui/src/lib/account-scope.ts | 59 ++++++++++++++++++++ 5 files changed, 167 insertions(+), 77 deletions(-) create mode 100644 ui/src/lib/account-scope.ts diff --git a/tests/e2e/disconnect-write-recovery.spec.ts b/tests/e2e/disconnect-write-recovery.spec.ts index 8f81ee178dc..780f6f27a04 100644 --- a/tests/e2e/disconnect-write-recovery.spec.ts +++ b/tests/e2e/disconnect-write-recovery.spec.ts @@ -178,10 +178,19 @@ test("replaying an accepted comment does not reopen completed work", async ({ pa } }); -test("startup work retries a failed session lookup while other APIs stay healthy", async ({ page, request }) => { +for (const delayWorkspacePreference of [false, true]) test(delayWorkspacePreference + ? "startup title survives session recovery while workspace preferences remount the editor" + : "startup work retries a failed session lookup while other APIs stay healthy", async ({ page, request }) => { const company = await json(await request.post("/api/companies", { data: { name: `Session recovery ${Date.now()}` } })); const session = await json(await request.get("/api/auth/get-session")); let healthy = false, saving = false, failedAfterSave = 0; + let releasePreference!: () => void; + const preferenceGate = new Promise(resolve => { releasePreference = resolve; }); + let sawPreference!: () => void; + const preferenceRequested = new Promise(resolve => { sawPreference = resolve; }); + if (delayWorkspacePreference) await page.route("**/api/workspace-preferences/me", async route => { + sawPreference(); await preferenceGate; await route.continue(); + }); try { const issue = await json(await request.post(`/api/companies/${company.id}/issues`, { data: { title: "Before session recovery", status: "backlog" }, @@ -199,12 +208,24 @@ test("startup work retries a failed session lookup while other APIs stay healthy saving = true; await header.locator("textarea").press("Enter"); await expect.poll(() => failedAfterSave).toBeGreaterThan(0); + if (delayWorkspacePreference) { + await expect(page.getByText("Couldn't update the task", { exact: true })).toBeVisible(); + await page.evaluate(() => new Promise(resolve => requestAnimationFrame(() => resolve(null)))); + } healthy = true; + if (delayWorkspacePreference) { + // Returning to the tab retries session lookup while the saved edit waits. + await page.evaluate(() => window.dispatchEvent(new Event("visibilitychange"))); + await preferenceRequested; + await expect(page.getByTestId("task-chat-history-content")).toBeHidden(); + releasePreference(); + } await expect.poll(async () => (await json(await request.get(`/api/issues/${issue.id}`))).title, { timeout: 15_000 }).toBe("Saved after session recovery"); await page.reload(); await expect(header.getByRole("heading", { name: "Saved after session recovery", exact: true })).toBeVisible(); } finally { + releasePreference(); await request.patch(`/api/companies/${company.id}`, { data: { status: "archived" } }); } }); diff --git a/ui/src/hooks/useAccountBoundMutation.ts b/ui/src/hooks/useAccountBoundMutation.ts index b011b9ec8ad..5cbf156b703 100644 --- a/ui/src/hooks/useAccountBoundMutation.ts +++ b/ui/src/hooks/useAccountBoundMutation.ts @@ -5,68 +5,12 @@ import { QueryClientContext, type DefaultError, type MutateOptions, - type QueryClient, type UseMutationOptions, type UseMutationResult, type MutationFunctionContext, } from "@tanstack/react-query"; -import { queryKeys } from "@/lib/queryKeys"; -import { authApi } from "@/api/auth"; -import { classifySendFailure, PendingSendAccountChangedError } from "@/lib/pending-send"; - -type AccountGeneration = { owner: string | undefined; generation: number }; -const accounts = new WeakMap(); - -function currentOwner(client: QueryClient): string | undefined { - const session = client.getQueryData<{ user?: { id?: string }; session?: { userId?: string } } | null>(queryKeys.auth.session); - return session === undefined ? undefined : session?.user?.id ?? session?.session?.userId ?? "signed-out"; -} - -function bindAccount(client: QueryClient) { - let account = accounts.get(client); - if (!account) { - account = { owner: currentOwner(client), generation: 0 }; - accounts.set(client, account); - const tracked = account; - client.getQueryCache().subscribe(event => { - const owner = currentOwner(client); - const removed = event.type === "removed" && JSON.stringify(event.query.queryKey) === JSON.stringify(queryKeys.auth.session); - if (owner !== tracked.owner || removed) { - if (removed || tracked.owner !== undefined || tracked.generation !== 0) tracked.generation += 1; - tracked.owner = owner; - } - }); - } - const { owner, generation } = account; - const isCurrentScope = () => account.generation === generation && (owner === undefined ? generation === 0 : account.owner === owner); - const isCurrentAccount = () => account.owner !== undefined && isCurrentScope(); - const whenAccountReady = async () => { - if (account.owner !== undefined || !isCurrentScope()) return; - let stopped = false; - let unsubscribe = () => {}; - const ready = new Promise(resolve => { - unsubscribe = client.getQueryCache().subscribe(() => { - if (account.owner !== undefined || !isCurrentScope()) resolve(); - }); - }); - const lookup = async () => { - for (let attempt = 0; attempt < 3; attempt += 1) { - if (stopped || account.owner !== undefined || !isCurrentScope()) return; - try { - await client.fetchQuery({ queryKey: queryKeys.auth.session, queryFn: () => authApi.getSession(), retry: false }); - return; - } catch (error) { - if (stopped || account.owner !== undefined || !isCurrentScope()) return; - if (attempt === 2 || classifySendFailure(error) !== "pending") throw error; - await new Promise(resolve => setTimeout(resolve, 250 * 2 ** attempt)); - } - } - }; - try { await Promise.race([ready, lookup()]); } - finally { stopped = true; unsubscribe(); } - }; - return { isCurrentAccount, isCurrentScope, whenAccountReady }; -} +import { PendingSendAccountChangedError } from "@/lib/pending-send"; +import { bindAccount } from "@/lib/account-scope"; export function useCurrentAccountGuard(): () => boolean { const client = useContext(QueryClientContext); diff --git a/ui/src/hooks/useDurableAutosave.test.tsx b/ui/src/hooks/useDurableAutosave.test.tsx index edad11f04a0..af5860e0a85 100644 --- a/ui/src/hooks/useDurableAutosave.test.tsx +++ b/ui/src/hooks/useDurableAutosave.test.tsx @@ -173,6 +173,51 @@ describe("useDurableAutosave", () => { client.clear(); }); + it.each([false, true])("keeps an unowned startup edit when identity and unmount race (identityFirst=%s)", async identityFirst => { + store.setBrowserOnline(false); + const client = new QueryClient({ defaultOptions: { queries: { staleTime: Infinity, retry: false, enabled: false } } }); + const save = vi.fn().mockResolvedValue(undefined); + act(() => root.render( + + )); + await act(async () => { await current.run("Retained startup title"); }); + expect(current.status).toBe("waiting"); + act(() => { + if (identityFirst) client.setQueryData(queryKeys.auth.session, { user: { id: "user-1" } }); + root.unmount(); root = createRoot(container); + if (!identityFirst) client.setQueryData(queryKeys.auth.session, { user: { id: "user-1" } }); + }); + await act(async () => { store.setBrowserOnline(true); store.reportSuccess(); }); + await advance(2_000); + expect(save).toHaveBeenCalledExactlyOnceWith("Retained startup title"); + expect(store.getSnapshot().pendingWrites).toBe(0); + client.clear(); + }); + + it.each(["different-account", "round-trip", "removed-session", "signed-out"])( + "does not adopt a later identity for a detached startup edit (%s)", async transition => { + store.setBrowserOnline(false); + const client = new QueryClient({ defaultOptions: { queries: { staleTime: Infinity, retry: false, enabled: false } } }); + const save = vi.fn().mockResolvedValue(undefined); + act(() => root.render( + + )); + await act(async () => { await current.run("Private startup title"); }); + act(() => { root.unmount(); root = createRoot(container); }); + act(() => { + if (transition === "removed-session") client.removeQueries({ queryKey: queryKeys.auth.session }); + else client.setQueryData(queryKeys.auth.session, transition === "signed-out" ? null : { user: { id: "user-1" } }); + if (transition === "round-trip") client.setQueryData(queryKeys.auth.session, { user: { id: "user-2" } }); + client.setQueryData(queryKeys.auth.session, { user: { id: transition === "different-account" ? "user-2" : "user-1" } }); + }); + await act(async () => { store.setBrowserOnline(true); store.reportSuccess(); }); + await advance(2_000); + expect(save).not.toHaveBeenCalled(); + expect(store.getSnapshot().pendingWrites).toBe(0); + client.clear(); + }, + ); + it("bounds retries after an editor unmounts", async () => { const save = vi.fn().mockRejectedValue(outage()); render({ save }); diff --git a/ui/src/hooks/useDurableAutosave.ts b/ui/src/hooks/useDurableAutosave.ts index 658d68cd149..0293975cc8b 100644 --- a/ui/src/hooks/useDurableAutosave.ts +++ b/ui/src/hooks/useDurableAutosave.ts @@ -1,4 +1,6 @@ -import { useCallback, useEffect, useId, useRef, useState } from "react"; +import { useCallback, useContext, useEffect, useId, useRef, useState } from "react"; +import { QueryClientContext } from "@tanstack/react-query"; +import { bindAccount } from "@/lib/account-scope"; import { classifyError, describeError, type ErrorDescription } from "@/api/errors"; import { loadStructuredDraft, saveStructuredDraft } from "@/lib/composer-draft"; import { useConnectivityStore } from "@/lib/connectivity"; @@ -108,6 +110,7 @@ const SAVED_LINGER_MS = 1600; * - Errors become readable copy for "Couldn't save (reason)". */ export function useDurableAutosave(options: DurableAutosaveOptions): DurableAutosave { + const client = useContext(QueryClientContext); const owner = useDraftOwner(); const readOwner = useDraftOwnerReader(); const instanceId = useId(); @@ -120,7 +123,7 @@ export function useDurableAutosave(options: DurableAutosaveOptions): Durab const targetKey = options.targetKey ?? options.sourceId; const targetRef = useRef(targetKey); targetRef.current = targetKey; - type Job = { token: symbol; registryKey: string; resends: number; saving?: boolean; id: number; target: string; owner: string | undefined; draft: T; + type Job = { token: symbol; registryKey: string; resends: number; saving?: boolean; id: number; target: string; owner: string | undefined; account: ReturnType | null; draft: T; save: (draft: T) => Promise; matchesServer?: DurableAutosaveOptions["matchesServer"]; onRetryFailed?: DurableAutosaveJob["onRetryFailed"]; storageKey: string | null; action: string }; const jobs = useRef(new Map()); @@ -142,6 +145,22 @@ export function useDurableAutosave(options: DurableAutosaveOptions): Durab entry?.active.delete(job.token); if (entry && !entry.active.size) registry.delete(job.registryKey); }; + // Resolve first identity from the same generation guard used by mutations. + // This also runs after unmount, when React cannot migrate the job for us. + function owned(job: Job) { + if (job.account && !job.account.isCurrentScope()) return false; + const current = readOwner(); + if (job.owner === undefined && current !== undefined) { + const nextKey = JSON.stringify([current, job.target]); + if (registry.get(job.registryKey)?.latest !== job.token || registry.has(nextKey)) return false; + releaseJob(job); + job.owner = current; + job.registryKey = nextKey; + registry.set(nextKey, { latest: job.token, active: new Set([job.token]) }); + } + return current === job.owner && generations.current.get(job.target) === job.id + && registry.get(job.registryKey)?.latest === job.token; + } const mounted = useRef(true); const savingTimer = useRef | null>(null); const savedTimer = useRef | null>(null); @@ -160,17 +179,11 @@ export function useDurableAutosave(options: DurableAutosaveOptions): Durab if (previous === undefined && owner !== undefined) { for (const job of jobs.current.values()) { if (job.owner !== undefined) continue; - const nextKey = JSON.stringify([owner, job.target]); - const superseded = registry.get(job.registryKey)?.latest !== job.token || registry.has(nextKey); - releaseJob(job); - if (superseded) { + if (!owned(job)) { + releaseJob(job); jobs.current.delete(job.target); if (generations.current.get(job.target) === job.id) generations.current.delete(job.target); - continue; } - job.owner = owner; - job.registryKey = nextKey; - registry.set(nextKey, { latest: job.token, active: new Set([job.token]) }); } refresh(value => value + 1); return; @@ -194,12 +207,12 @@ export function useDurableAutosave(options: DurableAutosaveOptions): Durab return stored?.version === 1 && stored.draft !== undefined ? stored.draft : null; }, [storageKey]); const execute = useCallback(async (job: Job, retry = false): Promise => { - const owned = () => readOwner() === job.owner && generations.current.get(job.target) === job.id && registry.get(job.registryKey)?.latest === job.token; - const visible = () => mounted.current && owned() && targetRef.current === job.target; + const stillOwned = () => owned(job); + const visible = () => mounted.current && stillOwned() && targetRef.current === job.target; const publish = (status: DurableAutosaveStatus, error: ErrorDescription | null = null) => { if (visible()) setView({ target: job.target, status, error }); }; - if (!owned()) { + if (!stillOwned()) { if (jobs.current.get(job.target) === job) jobs.current.delete(job.target); releaseJob(job); if (mounted.current) refresh(value => value + 1); @@ -234,7 +247,12 @@ export function useDurableAutosave(options: DurableAutosaveOptions): Durab remove(); return { kind: "saved" }; }; - try { await job.save(job.draft); return done(); } + try { + if (!mounted.current && job.account && !job.account.isCurrentAccount()) await job.account.whenAccountReady(); + if (!stillOwned()) { remove(); return { kind: "waiting" }; } + await job.save(job.draft); + return done(); + } catch (initialCause) { let cause = initialCause; if (classifyError(cause) === "conflict" && job.matchesServer) { @@ -255,7 +273,7 @@ export function useDurableAutosave(options: DurableAutosaveOptions): Durab const current = optionsRef.current; const target = current.targetKey ?? current.sourceId; const admittedOwner = ownerRef.current ?? readOwner(); - const job: Job = { token: Symbol(), registryKey: JSON.stringify([admittedOwner, target]), resends: 0, id: ++sequence.current, target, owner: admittedOwner, draft, + const job: Job = { token: Symbol(), registryKey: JSON.stringify([admittedOwner, target]), resends: 0, id: ++sequence.current, target, owner: admittedOwner, account: client ? bindAccount(client) : null, draft, save: overrides.save ?? current.save, matchesServer: overrides.matchesServer ?? current.matchesServer, onRetryFailed: overrides.onRetryFailed, storageKey: current.storageKey ?? null, action: current.action ?? "save" }; const previous = jobs.current.get(target); @@ -266,7 +284,7 @@ export function useDurableAutosave(options: DurableAutosaveOptions): Durab generations.current.set(target, job.id); jobs.current.set(target, job); refresh(value => value + 1); return execute(job); - }, [execute, readOwner]); + }, [client, execute, readOwner]); const waitingJobs = [...jobs.current.values()].filter(job => !job.saving).length; const auto = useAutoResend({ sourceId: options.sourceId, active: waitingJobs > 0, pendingCount: waitingJobs, resend: async (): Promise => { @@ -281,7 +299,6 @@ export function useDurableAutosave(options: DurableAutosaveOptions): Durab detachedWork.current = () => { if (!jobs.current.size) return; if (auto.stalled) { for (const job of jobs.current.values()) releaseJob(job); jobs.current.clear(); return; } - const admittedOwner = ownerRef.current; const source = `${optionsRef.current.sourceId}:detached:${instanceId}`; let timer: ReturnType | null = null; let disposed = false, running = false; @@ -298,7 +315,11 @@ export function useDurableAutosave(options: DurableAutosaveOptions): Durab }; const wake = async () => { if (disposed || running) return; - if (mounted.current || readOwner() !== admittedOwner || admittedOwner === undefined) { release(); return; } + if (mounted.current) { release(); return; } + for (const job of jobs.current.values()) if (!owned(job)) { + releaseJob(job); + jobs.current.delete(job.target); + } if (!jobs.current.size) { release(); return; } store.setPendingWrites(source, jobs.current.size); if (store.getSnapshot().status !== "online") return; diff --git a/ui/src/lib/account-scope.ts b/ui/src/lib/account-scope.ts new file mode 100644 index 00000000000..0e7d050e502 --- /dev/null +++ b/ui/src/lib/account-scope.ts @@ -0,0 +1,59 @@ +import type { QueryClient } from "@tanstack/react-query"; +import { queryKeys } from "./queryKeys"; +import { authApi } from "@/api/auth"; +import { classifySendFailure } from "./pending-send"; + +type AccountGeneration = { owner: string | undefined; generation: number }; +const accounts = new WeakMap(); + +function currentOwner(client: QueryClient): string | undefined { + const session = client.getQueryData<{ user?: { id?: string }; session?: { userId?: string } } | null>(queryKeys.auth.session); + return session === undefined ? undefined : session?.user?.id ?? session?.session?.userId ?? "signed-out"; +} + +export function bindAccount(client: QueryClient) { + let account = accounts.get(client); + if (!account) { + account = { owner: currentOwner(client), generation: 0 }; + accounts.set(client, account); + const tracked = account; + client.getQueryCache().subscribe(event => { + const owner = currentOwner(client); + const removed = event.type === "removed" && JSON.stringify(event.query.queryKey) === JSON.stringify(queryKeys.auth.session); + if (owner !== tracked.owner || removed) { + if (removed || tracked.owner !== undefined || tracked.generation !== 0) tracked.generation += 1; + tracked.owner = owner; + } + }); + } + const { owner, generation } = account; + const isCurrentScope = () => account.generation === generation && (owner === undefined ? generation === 0 : account.owner === owner); + const isCurrentAccount = () => account.owner !== undefined && isCurrentScope(); + const whenAccountReady = async () => { + if (account.owner !== undefined || !isCurrentScope()) return; + let stopped = false; + let unsubscribe = () => {}; + const ready = new Promise(resolve => { + unsubscribe = client.getQueryCache().subscribe(() => { + if (account.owner !== undefined || !isCurrentScope()) resolve(); + }); + }); + const lookup = async () => { + for (let attempt = 0; attempt < 3; attempt += 1) { + if (stopped || account.owner !== undefined || !isCurrentScope()) return; + try { + await client.fetchQuery({ queryKey: queryKeys.auth.session, queryFn: () => authApi.getSession(), retry: false }); + return; + } catch (error) { + if (stopped || account.owner !== undefined || !isCurrentScope()) return; + if (attempt === 2 || classifySendFailure(error) !== "pending") throw error; + await new Promise(resolve => setTimeout(resolve, 250 * 2 ** attempt)); + } + } + }; + try { await Promise.race([ready, lookup()]); } + finally { stopped = true; unsubscribe(); } + }; + return { isCurrentAccount, isCurrentScope, whenAccountReady }; +} + From b257ae2014866fe31b96e09c5eba92940fc835ba Mon Sep 17 00:00:00 2001 From: Dotta Date: Fri, 30 Oct 2026 20:29:15 -0500 Subject: [PATCH 70/74] fix(ui): supersede startup jobs when newer edits are admitted Co-Authored-By: Paperclip --- ui/src/hooks/useDurableAutosave.test.tsx | 26 ++++++++++++++++++++++++ ui/src/hooks/useDurableAutosave.ts | 3 +++ 2 files changed, 29 insertions(+) diff --git a/ui/src/hooks/useDurableAutosave.test.tsx b/ui/src/hooks/useDurableAutosave.test.tsx index af5860e0a85..be690a90301 100644 --- a/ui/src/hooks/useDurableAutosave.test.tsx +++ b/ui/src/hooks/useDurableAutosave.test.tsx @@ -120,6 +120,32 @@ describe("useDurableAutosave", () => { expect(store.getSnapshot().pendingWrites).toBe(0); }); + it.each(["same-target", "different-target"])("preserves newer completed saves across first identity resolution (%s)", async target => { + let server = "Original"; + const client = new QueryClient({ defaultOptions: { queries: { staleTime: Infinity, retry: false, enabled: false } } }); + const oldSave = vi.fn().mockRejectedValueOnce(outage()).mockImplementation(async value => { server = value; }); + const show = (sourceId: string, targetKey: string, save: (value: string) => Promise) => act(() => { + root.render( + + ); + }); + show("old-startup-editor", "task-a:title", oldSave); + await act(async () => { expect(await current.run("Older title")).toEqual({ kind: "waiting" }); }); + act(() => { root.unmount(); root = createRoot(container); }); + act(() => client.setQueryData(queryKeys.auth.session, { user: { id: "user-1" } })); + const newSave = vi.fn(async value => { if (target === "same-target") server = value; }); + show("new-known-editor", target === "same-target" ? "task-a:title" : "task-b:title", newSave); + await act(async () => { expect(await current.run("Newer title")).toEqual({ kind: "saved" }); }); + expect(newSave).toHaveBeenCalledExactlyOnceWith("Newer title"); + // The newer save has finished and released its registry entry before the + // detached startup job gets its first chance to adopt the account. + await advance(30_000); + expect(server).toBe(target === "same-target" ? "Newer title" : "Older title"); + expect(oldSave).toHaveBeenCalledTimes(target === "same-target" ? 1 : 2); + expect(store.getSnapshot().pendingWrites).toBe(0); + client.clear(); + }); + it("keeps a detached edit when a reopened editor resets only its local state and storage is unavailable", async () => { const storage = vi.spyOn(Storage.prototype, "setItem").mockImplementation(() => { throw new Error("Storage unavailable"); }); try { diff --git a/ui/src/hooks/useDurableAutosave.ts b/ui/src/hooks/useDurableAutosave.ts index 0293975cc8b..98e062b77e3 100644 --- a/ui/src/hooks/useDurableAutosave.ts +++ b/ui/src/hooks/useDurableAutosave.ts @@ -273,6 +273,9 @@ export function useDurableAutosave(options: DurableAutosaveOptions): Durab const current = optionsRef.current; const target = current.targetKey ?? current.sourceId; const admittedOwner = ownerRef.current ?? readOwner(); + // A newer known-account edit supersedes an older startup job even if this + // save finishes before that job can adopt the account's registry key. + if (admittedOwner !== undefined) registry.delete(JSON.stringify([undefined, target])); const job: Job = { token: Symbol(), registryKey: JSON.stringify([admittedOwner, target]), resends: 0, id: ++sequence.current, target, owner: admittedOwner, account: client ? bindAccount(client) : null, draft, save: overrides.save ?? current.save, matchesServer: overrides.matchesServer ?? current.matchesServer, onRetryFailed: overrides.onRetryFailed, storageKey: current.storageKey ?? null, action: current.action ?? "save" }; From 295f4f82722d92b93d8b5135399bca937116b536 Mon Sep 17 00:00:00 2001 From: Dotta Date: Fri, 30 Oct 2026 20:35:52 -0500 Subject: [PATCH 71/74] test(server): preserve attention feed beyond blocker root budget Co-Authored-By: Paperclip --- .../2026-10-30-attention-retention-sleep.md | 2 ++ .../attention-retention-work.test.ts | 19 +++++++++++++++++++ 2 files changed, 21 insertions(+) diff --git a/doc/plans/2026-10-30-attention-retention-sleep.md b/doc/plans/2026-10-30-attention-retention-sleep.md index 92241aefc4f..b680241adf7 100644 --- a/doc/plans/2026-10-30-attention-retention-sleep.md +++ b/doc/plans/2026-10-30-attention-retention-sleep.md @@ -48,3 +48,5 @@ Local embedded PostgreSQL could not start on the test host because system semaph Review the source parity fixtures and producer hints together. The intended outcome is unchanged archive state and notifications with no recurring retention SQL when the sources remain unchanged. No merge or deployment is part of this change. Issue producer invalidation has its own finite discovery budget. Reverse parent/blocker traversal returns at most 101 distinct UUIDs in one recursive query, with each relation adjacency capped at 101 unique edges before materialization. At most 100 affected issues produce one sorted canonical batch hint; an overflow sentinel produces one company hint and the existing complete paged reconstruction. UNION deduplication terminates cycles. A PostgreSQL EXPLAIN ANALYZE on 2,100 reverse edges proves the outer limit stops at 101 CTE rows while the indexed relation branch returns only 100 rows. Repeated identical writes coalesce by canonical key. The shared scheduler is unchanged: a broader company hint does not cancel already pending narrower scopes; those remain finite batches of at most 100 IDs, processed at the existing 25-scope pass limit. This permits redundant bounded reconstruction instead of discarding invalidations or introducing cursor/subsumption policy across other owners. A 151-node cyclic chain proves overflow still registers the far ancestor's exact archive deadline and settles quietly afterward. + +The ordinary feed retains conservative `needs_attention` entries for roots outside the shared graph budget. The batch finalization visits every input root; a root without explored edges points attention at itself. A real PostgreSQL regression with 2,001 blocked roots verifies the nonempty deferred set, bounded graph size, ordinary issue-list metadata, and all 2,001 attention-feed subject IDs. This is a consumer parity proof; it does not increase graph traversal or change archive eligibility. diff --git a/server/src/__tests__/attention-retention-work.test.ts b/server/src/__tests__/attention-retention-work.test.ts index f06355e1ae7..ce24ac1081b 100644 --- a/server/src/__tests__/attention-retention-work.test.ts +++ b/server/src/__tests__/attention-retention-work.test.ts @@ -318,6 +318,25 @@ const delay = DEFAULT_DECISION_ARCHIVE_DAYS * 86_400_000; } }); + it("keeps deferred roots visible in the ordinary attention feed above the root cap", async () => { + await db.execute(sql`insert into issues (id, company_id, title, status, updated_at) + select gen_random_uuid(), ${companyId}::uuid, 'overflow-root-' || n, 'blocked', ${new Date().toISOString()}::timestamptz + from generate_series(1, ${BLOCKER_ATTENTION_MAX_NODES + 1}) n`); + const roots = await db.select().from(issues).where(eq(issues.companyId, companyId)); + const batch = await issueService(db).listBlockerAttentionBatch(companyId, roots); + expect(batch.deferredRootIds.size).toBeGreaterThan(0); + expect(batch.nodeCount).toBeLessThanOrEqual(BLOCKER_ATTENTION_MAX_NODES); + for (const rootId of batch.deferredRootIds) expect(batch.attention.get(rootId)).toMatchObject({ + state: "needs_attention", reason: "attention_required", terminalBlockerIssueId: rootId, + }); + const listed = await issueService(db).list(companyId, { status: "blocked", includeBlockedBy: true }); + expect(listed).toHaveLength(roots.length); + for (const root of listed) expect(root.blockerAttention).toMatchObject({ state: "needs_attention", terminalBlockerIssueId: root.id }); + const feed = await attentionService(db).list(companyId, { all: true, allowUnscopedAll: true }); + const visible = new Set(feed.items.filter(item => item.sourceKind === "blocker_attention").map(item => item.subject.id)); + expect(visible).toEqual(new Set(roots.map(root => root.id))); + }); + it("bounds every graph query when independent root trees contain over 65535 unfinished nodes", async () => { await db.execute(sql`insert into issues (id, company_id, title, status, updated_at) select gen_random_uuid(), ${companyId}::uuid, 'parameter-root-' || n, 'blocked', ${old().toISOString()}::timestamptz From 19a97bbe57d1ebd3691c70dd822488efe7e5d3e1 Mon Sep 17 00:00:00 2001 From: Dotta Date: Fri, 30 Oct 2026 20:35:57 -0500 Subject: [PATCH 72/74] fix(server): park persisted deferred admission refusals Distinguish a budget or agent refusal saved on the existing receipt from genuine recovery failure. Preserve HTTP409 messages and details, grant each new wait reason its existing finite grace, and retain failure backoff for unsaved or unexpected errors. Prove actual Retry admission,25-minute SQL quiet, independent warm producer-to-process release, missing/cancelled receipts and real PostgreSQL write failure. Co-Authored-By: Paperclip --- .../2026-10-30-deferred-execution-sleep.md | 4 + .../deferred-execution-receipt-work.test.ts | 172 +++++++++++++++++- .../src/services/deferred-execution-wait.ts | 15 +- .../heartbeat/deferred-receipt-work.ts | 7 +- server/src/services/heartbeat/queue.ts | 31 ++-- 5 files changed, 204 insertions(+), 25 deletions(-) diff --git a/doc/plans/2026-10-30-deferred-execution-sleep.md b/doc/plans/2026-10-30-deferred-execution-sleep.md index 56e3611d939..e84d9abf165 100644 --- a/doc/plans/2026-10-30-deferred-execution-sleep.md +++ b/doc/plans/2026-10-30-deferred-execution-sleep.md @@ -21,3 +21,7 @@ C2 remains separate and mandatory: active output silence deadlines, remaining or Review controls cover queue edits that move an already loaded grace deadline. Each due check re-reads current stored grace or claim time before admission. Subtree hold release, subtree restore, and member archival publish eligibility inside their own transactions. Member archival preserves the cancelled Retry tombstone created by an earlier user reassignment. A resume that first records a new execution-wait reason arms its existing 30-second grace once. Writing the same closed-gate reason again does not publish another hint. A clock obligation, an eligibility change, and an infrastructure failure therefore retain separate exits: stored deadline, owning committed producer, and bounded failure backoff. Refreshing a diagnostic timestamp alone does not create permanent polling behind a deliberate pause. + +Persisted budget and agent refusals are normalized by the existing deferred-wait owner. The wait writer reports whether its locked conditional update saved the receipt, and only that successful path can produce the internal `PersistedExecutionWaitRefusal` subtype. It remains an HTTP409 with the original message and details. Receipt recovery parks that outcome instead of creating failure debt; the committed first/changed reason still grants its one stored30-second grace. Unrelated409s, failed writes and unexpected provider/infrastructure errors retain failure backoff. This adds no admission policy and changes no native/task-chat ownership. + +Production-path controls use actual task Retry, heartbeat, PostgreSQL budget/lifecycle services and a child process writing saved stdout. The lifecycle host completion boundary is controlled explicitly. One set of real budget/agent refusal fixtures consumes its finite grace and becomes quiet for25 minutes of timer advancement with zero driver SQL and no execution. Independent warm-worker fixtures keep the real PostgreSQL clock throughout admission, actual gate-producer release, controller authority and child-process exit, proving exactly one saved execution. Long-idle and provider-release evidence are separate; no clock reset or restart is used to claim warm recovery. A PostgreSQL trigger fault proves failed wait persistence retains recovery backoff, and an unrelated unbranded409 remains a failure. diff --git a/server/src/__tests__/deferred-execution-receipt-work.test.ts b/server/src/__tests__/deferred-execution-receipt-work.test.ts index 914f33fe609..18af0f94f97 100644 --- a/server/src/__tests__/deferred-execution-receipt-work.test.ts +++ b/server/src/__tests__/deferred-execution-receipt-work.test.ts @@ -1,7 +1,11 @@ +import { conflict, HttpError } from "../errors.js"; import { setImmediate as immediate } from "node:timers/promises"; import { DELIVERY_QUEUES, subscribeDeliveryWork } from "../services/delivery-work-notifications.js"; +import { instanceSettingsService } from "../services/instance-settings.js"; +import { budgetService } from "../services/budgets.js"; +import { configureAgentLifecycle, createAgentLifecycle } from "../services/agent-lifecycle.js"; import { accessService } from "../services/access.js"; -import { writeDeferredExecutionWait } from "../services/deferred-execution-wait.js"; +import { PersistedExecutionWaitRefusal, writeDeferredExecutionWait } from "../services/deferred-execution-wait.js"; import { createQueuedCommentIssueLockWriter } from "../modules/wake-queue/adapters/queued-comment-postgres.js"; import { createHeartbeatQueue } from "../services/heartbeat/queue.js"; import { createHeartbeatQueueTestDependencies } from "./helpers/heartbeat-queue.js"; @@ -9,7 +13,7 @@ import { issueTreeControlService } from "../services/issue-tree-control.js"; import { randomUUID } from "node:crypto"; import { and, eq, sql } from "drizzle-orm"; import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; -import { agents, agentWakeupRequests, authUsers, chatActions, chatEndpoints, toolApplications, toolConnections, companies, companyMemberships, createDb, reconcileDatabaseWork, heartbeatRuns, issueComments, issues } from "@paperclipai/db"; +import { agents, agentWakeupRequests, authUsers, chatActions, chatEndpoints, toolApplications, toolConnections, companies, companyMemberships, costEvents, createDb, reconcileDatabaseWork, heartbeatRuns, issueComments, issues } from "@paperclipai/db"; import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js"; import { startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; import { createDeliveryWorkCoordinator } from "../services/delivery-work-coordinator.js"; @@ -28,18 +32,23 @@ describe("deferred receipt durable dispatch", () => { let coordinator: ReturnType; let drain = false; const errors: unknown[] = []; + const errorTimes: number[] = []; + const statements: string[] = []; const passes = new Set>(); + const lifecycleWorkers = new Set>(); beforeAll(async () => { const external = resolveExternalTestDatabaseUrl("PAPERCLIP_DEFERRED_RECEIPT_TEST_DATABASE_URL"); if (!external) temporary = await startEmbeddedPostgresTestDatabase("deferred-receipts-"); db = createDb(external ?? temporary!.connectionString, { maxConnections: 6, idleTimeoutSeconds: 0, maxLifetimeSeconds: 0 }); + db.$client.options.onnotice = () => {}; + db.$client.options.debug = (_connection, query) => { statements.push(query); }; await Promise.all(Array.from({ length: 6 }, () => db.execute(sql`select pg_sleep(0.01)`))); }); beforeEach(async () => { - await db.execute(sql`truncate companies cascade`); drain = false; errors.length = 0; + await db.execute(sql`truncate companies cascade`); drain = false; errors.length = 0; errorTimes.length = 0; heartbeat = heartbeatService(db); coordinator = createDeliveryWorkCoordinator({ owner: db, canRun: () => !drain, canReconcile: () => true, - onError: error => errors.push(error) }); + onError: error => { errors.push(error); errorTimes.push(Date.now()); } }); const register = coordinator.register.bind(coordinator); coordinator.register = (queue, task) => register(queue, { ...task, run: signal => { const pass = Promise.resolve(task.run(signal)); passes.add(pass); @@ -48,6 +57,7 @@ describe("deferred receipt durable dispatch", () => { }); afterEach(async () => { await coordinator.stop(); await heartbeat.drainActiveRunExecutions(); + await Promise.all([...lifecycleWorkers].map(worker => worker.stop())); lifecycleWorkers.clear(); vi.useRealTimers(); vi.restoreAllMocks(); expect(errors).toEqual([]); }); afterAll(async () => { await db?.$client.end({ timeout: 1 }); await temporary?.cleanup(); }); @@ -72,7 +82,9 @@ describe("deferred receipt durable dispatch", () => { const reads = vi.spyOn(db, "select"), executes = vi.spyOn(db, "execute"); const selected = reads.mock.calls.length, executed = executes.mock.calls.length; if (!vi.isFakeTimers()) vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"] }); - await vi.advanceTimersByTimeAsync(20 * 60_000); + const count = statements.length; + await vi.advanceTimersByTimeAsync(25 * 60_000); + expect(statements).toHaveLength(count); expect(reads).toHaveBeenCalledTimes(selected); expect(executes).toHaveBeenCalledTimes(executed); } async function finish(issueId: string) { @@ -130,6 +142,156 @@ describe("deferred receipt durable dispatch", () => { expect(await db.select().from(heartbeatRuns).where(sql`${heartbeatRuns.contextSnapshot}->>'taskRetryRequestId' = ${receipt.requestId}`)).toHaveLength(1); await quiet(); }); + async function blockedRetry(gate: "budget" | "paused agent", clocked = false) { + const f = await fixture(), lifecycle = createAgentLifecycle(db), budgets = budgetService(db); + // Host lifecycle completion is controlled; all state, admission and the + // eventual process adapter execution use their production implementations. + const lifecycleWorker = configureAgentLifecycle(db, { requiredPluginIds: async () => [], + runHost: async () => "complete", runPlugin: async () => "complete" }); + lifecycleWorkers.add(lifecycleWorker); + const policy = { scopeType: "agent" as const, scopeId: f.agent.id, amount: 100, + metric: "billed_cents" as const, windowKind: "lifetime" as const, hardStopEnabled: true, notifyEnabled: false }; + if (gate === "budget") { + await db.insert(costEvents).values({ companyId: f.companyId, agentId: f.agent.id, + provider: "test", model: "receipt-proof", costCents: 200, occurredAt: new Date() }); + await budgets.upsertPolicy(f.companyId, policy, f.actorId); + } else await lifecycle.pauseAgent(f.agent.id); + await lifecycleWorker.process(f.agent.id); + if (clocked) vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"] }); + await heartbeat.registerDeferredReceiptWork(coordinator, () => !drain).ready; + await heartbeat.registerRunDispatchWork(coordinator, () => !drain).ready; + await heartbeat.registerScheduledRetryCancellationWork(coordinator, () => !drain).ready; + const request = await requestTaskRetry({ db, companyId: f.companyId, issueId: f.issue.id, actorId: f.actorId, + observedRunId: f.source.id, idempotencyKey: randomUUID() }); + expect(request.accepted).toBe(true); + const read = async () => (await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, request.requestId!)))[0]; + await vi.waitFor(async () => expect((await read()).payload?.executionWait).toMatchObject({ + reason: gate === "budget" ? "budget.blocked" : "agent.not_invokable", + }), { timeout: 10_000 }); + const refusals: unknown[] = []; + await heartbeat.resumeExecutionWaitComments({ companyId: f.companyId, requestId: request.requestId!, onError: error => refusals.push(error) }); + expect(refusals).toHaveLength(1); + expect(refusals[0]).toBeInstanceOf(HttpError); + expect(refusals[0]).toMatchObject({ status: 409, + details: gate === "budget" ? { scopeType: "agent", scopeId: f.agent.id } : { status: "paused", reason: "paused" } }); + return { f, lifecycle, budgets, policy, lifecycleWorker, request, read }; + } + + it.each(["budget", "paused agent"] as const)("parks real task Retry behind unchanged %s without recurring SQL", async gate => { + const { request, read } = await blockedRetry(gate, true); + // The clock was installed before worker registration and every timer. + // No execution exists while we observe actual driver SQL across25min. + for (let n = 0; n < 3; n++) { + await vi.advanceTimersByTimeAsync(31_000); await Promise.allSettled([...passes]); await immediate(); + } + expect(errors).toEqual([]); + await quiet(); + expect(await read()).toMatchObject({ status: "deferred_issue_execution", runId: null }); + expect(await db.select().from(heartbeatRuns).where(sql`${heartbeatRuns.contextSnapshot}->>'taskRetryRequestId' = ${request.requestId}`)).toHaveLength(0); + }); + + it.each(["budget", "paused agent"] as const)("releases real %s Retry into exactly one saved process execution", async gate => { + const { f, lifecycle, budgets, policy, lifecycleWorker, request, read } = await blockedRetry(gate); + // This independent warm-worker fixture stays on the real PostgreSQL clock + // through gate release, controller authority and actual child-process exit. + if (gate === "budget") await budgets.upsertPolicy(f.companyId, { ...policy, amount: 1000 }, f.actorId); + else await lifecycle.resumeAgent(f.agent.id); + await lifecycleWorker.process(f.agent.id); + let runId: string | null = null; + await vi.waitFor(async () => { + const wake = await read(); expect(wake.runId).toBeTruthy(); runId = wake.runId; + }, { timeout: 40_000 }); + await vi.waitFor(async () => { + await heartbeat.drainActiveRunExecutions(); + const [run] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, runId!)); + expect(run.status, JSON.stringify({ error: run.error, code: run.errorCode })).toBe("succeeded"); + expect(run.stdoutExcerpt).toContain("receipt-proof"); + }, { timeout: 15_000 }); + expect(await db.select().from(heartbeatRuns).where(sql`${heartbeatRuns.contextSnapshot}->>'taskRetryRequestId' = ${request.requestId}`)).toHaveLength(1); + await finish(f.issue.id); await quiet(); + }); + + it.each(["missing", "cancelled"] as const)("preserves ordinary409 when a %s receipt cannot save the gate", async status => { + const f = await fixture(), requestId = randomUUID(); + if (status === "cancelled") await db.insert(agentWakeupRequests).values({ id: requestId, companyId: f.companyId, + agentId: f.agent.id, source: "on_demand", status, payload: { issueId: f.issue.id } }); + const deps = createHeartbeatQueueTestDependencies(db), queue = createHeartbeatQueue(db, deps); + deps.getAgent.mockResolvedValue(f.agent); + deps.instanceSettings.getExperimental.mockImplementation(() => instanceSettingsService(db).getExperimental()); + for (const gate of ["budget", "agent"] as const) { + deps.budgets.getInvocationBlock.mockResolvedValue(gate === "budget" + ? { scopeType: "agent", scopeId: f.agent.id, scopeName: f.agent.name, reason: "Budget blocked" } : null); + deps.getAgentInvokability.mockResolvedValue({ invokable: false, message: "Paused", reason: "paused", invalidOrgChain: false }); + const error = await queue.enqueueWakeup(f.agent.id, { source: "on_demand", reason: "task_retry", + requestedByActorType: "user", requestedByActorId: f.actorId, payload: { issueId: f.issue.id }, + contextSnapshot: { issueId: f.issue.id, responsibleUserId: f.actorId } }, requestId).catch(cause => cause); + expect(error).toBeInstanceOf(HttpError); + expect(error).not.toBeInstanceOf(PersistedExecutionWaitRefusal); + expect(error).toMatchObject({ status: 409, message: gate === "budget" ? "Budget blocked" : "Paused" }); + } + const rows = await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, requestId)); + expect(rows).toHaveLength(status === "missing" ? 0 : 1); + if (status === "cancelled") expect(rows[0]).toMatchObject({ status: "cancelled", runId: null, payload: { issueId: f.issue.id } }); + }); + + it("retains failure backoff for a real wait-persistence database error", async () => { + const f = await fixture(); + await createAgentLifecycle(db).pauseAgent(f.agent.id); + const request = await requestTaskRetry({ db, companyId: f.companyId, issueId: f.issue.id, actorId: f.actorId, + observedRunId: f.source.id, idempotencyKey: randomUUID() }); + await db.execute(sql`create function c1_refusal_fault() returns trigger language plpgsql as $$ + begin if new.reason = 'task_retry' and new.payload ? 'executionWait' then + raise exception 'C1 injected wait persistence failure'; end if; return new; end $$`); + await db.execute(sql`create trigger c1_refusal_fault before update on agent_wakeup_requests + for each row execute function c1_refusal_fault()`); + vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"] }); + try { + await heartbeat.registerDeferredReceiptWork(coordinator, () => !drain).ready; + await vi.waitFor(() => expect(errors).toHaveLength(1)); + const failure = errors[0] as AggregateError; + expect(failure).toBeInstanceOf(AggregateError); + expect(String(failure.errors[0]?.errors[0]?.cause?.message)).toContain("C1 injected wait persistence failure"); + const due = errorTimes[0] + 30_000; + await Promise.allSettled([...passes]); await immediate(); + await vi.waitFor(() => expect(coordinator.nextWakeAt()).toBe(due)); + expect((await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, request.requestId!)))[0].payload?.executionWait).toBeUndefined(); + await vi.advanceTimersByTimeAsync(29_999); + expect(errors).toHaveLength(1); + await db.execute(sql`drop trigger c1_refusal_fault on agent_wakeup_requests`); + await db.execute(sql`drop function c1_refusal_fault()`); + errors.length = 0; + for (let n = 0; n < 3; n++) { + await vi.advanceTimersByTimeAsync(31_000); await Promise.allSettled([...passes]); await immediate(); + } + expect((await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, request.requestId!)))[0].payload?.executionWait) + .toMatchObject({ reason: "agent.not_invokable" }); + await quiet(); + } finally { + await db.execute(sql`drop trigger if exists c1_refusal_fault on agent_wakeup_requests`); + await db.execute(sql`drop function if exists c1_refusal_fault()`); + } + }); + + it("does not park an unrelated HTTP409 as a persisted refusal", async () => { + const f = await fixture(); + const [wake] = await db.insert(agentWakeupRequests).values({ companyId: f.companyId, agentId: f.agent.id, + source: "on_demand", reason: "task_retry", status: "deferred_issue_execution" }).returning(); + const failure = conflict("Unexpected admission conflict"); + let fail = true; + const resume = vi.fn(async (scope: { onError?: (error: unknown) => void }) => { if (fail) scope.onError?.(failure); }); + vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"] }); + await registerDeferredReceiptWork(db, coordinator, { resume, settleTaskChat: async () => {}, dispatchNative: async () => {}, + schedulingSuppressed: async () => false }, () => !drain).ready; + expect(errors).toHaveLength(1); + expect((errors[0] as AggregateError).errors[0].errors).toEqual([failure]); + expect(coordinator.nextWakeAt()).toBe(Date.now() + 30_000); + expect((await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, wake.id)))[0].status).toBe("deferred_issue_execution"); + errors.length = 0; fail = false; + await vi.advanceTimersByTimeAsync(30_000); + await vi.waitFor(() => expect(resume).toHaveBeenCalledTimes(2)); + await quiet(); + }); + it.each(["native claim", "execution wait"])("reconstructs an exact fresh %s deadline without probing early", async kind => { const f = await fixture(), updatedAt = new Date(), native = kind === "native claim"; const [wake] = await db.insert(agentWakeupRequests).values({ companyId: f.companyId, agentId: f.agent.id, diff --git a/server/src/services/deferred-execution-wait.ts b/server/src/services/deferred-execution-wait.ts index c4857d5b692..6924cb4e1ae 100644 --- a/server/src/services/deferred-execution-wait.ts +++ b/server/src/services/deferred-execution-wait.ts @@ -1,24 +1,33 @@ +import { HttpError } from "../errors.js"; import { and, eq, sql } from "drizzle-orm"; import { agentWakeupRequests, type Db } from "@paperclipai/db"; import { notifyDeferredExecutionWork } from "./deferred-execution-notifications.js"; +/** Admission has saved this refusal on the caller's durable deferred receipt. + * HTTP callers still receive the existing 409; receipt recovery waits for its + * owning gate producer instead of treating the refusal as infrastructure debt. */ +export class PersistedExecutionWaitRefusal extends HttpError { + constructor(message: string, details?: unknown) { super(409, message, details); } +} + export async function writeDeferredExecutionWait(db: Db, scope: { companyId: string; requestId: string }, condition: Record, options: { replace?: boolean } = {}) { return db.transaction(async tx => { const [current] = await tx.select({ payload: agentWakeupRequests.payload }).from(agentWakeupRequests) .where(and(eq(agentWakeupRequests.companyId, scope.companyId), eq(agentWakeupRequests.id, scope.requestId), eq(agentWakeupRequests.status, "deferred_issue_execution"))).for("update"); - if (!current) return; + if (!current) return false; const previous = current.payload?.executionWait; const previousReason = previous && typeof previous === "object" && !Array.isArray(previous) ? (previous as Record).reason : undefined; if (!previous || previousReason !== condition.reason) await notifyDeferredExecutionWork(tx, scope); - await tx.update(agentWakeupRequests).set({ + const [saved] = await tx.update(agentWakeupRequests).set({ payload: sql`jsonb_set(coalesce(${agentWakeupRequests.payload}, '{}'::jsonb), '{executionWait}', ${options.replace ? sql`'{}'::jsonb` : sql`coalesce(${agentWakeupRequests.payload}->'executionWait', '{}'::jsonb)`} || ${JSON.stringify(condition)}::jsonb)`, updatedAt: new Date(), }).where(and(eq(agentWakeupRequests.companyId, scope.companyId), eq(agentWakeupRequests.id, scope.requestId), - eq(agentWakeupRequests.status, "deferred_issue_execution"))); + eq(agentWakeupRequests.status, "deferred_issue_execution"))).returning({ id: agentWakeupRequests.id }); + return Boolean(saved); }); } diff --git a/server/src/services/heartbeat/deferred-receipt-work.ts b/server/src/services/heartbeat/deferred-receipt-work.ts index 7a69bc3be58..1089b45fe67 100644 --- a/server/src/services/heartbeat/deferred-receipt-work.ts +++ b/server/src/services/heartbeat/deferred-receipt-work.ts @@ -1,3 +1,4 @@ +import { PersistedExecutionWaitRefusal } from "../deferred-execution-wait.js"; import { and, asc, eq, gt, inArray, isNull, or, sql } from "drizzle-orm"; import { agentWakeupRequests, type Db } from "@paperclipai/db"; import type { createDeliveryWorkCoordinator } from "../delivery-work-coordinator.js"; @@ -105,7 +106,11 @@ export function registerDeferredReceiptWork( onError: error => { errors.push(error); }, }); else if (kind === "task_chat") await owner.settleTaskChat(scope); - else await owner.resume({ ...scope, onError: error => { errors.push(error); } }); + else { + const failed = (error: unknown) => { if (!(error instanceof PersistedExecutionWaitRefusal)) errors.push(error); }; + try { await owner.resume({ ...scope, onError: failed }); } + catch (error) { failed(error); } + } if (errors.length) throw new AggregateError(errors, `Deferred receipt ${entry.id} could not settle`); return next === null ? null : { ...entry, dueAt: next }; }, diff --git a/server/src/services/heartbeat/queue.ts b/server/src/services/heartbeat/queue.ts index 962f15f327e..758b808680b 100644 --- a/server/src/services/heartbeat/queue.ts +++ b/server/src/services/heartbeat/queue.ts @@ -1,4 +1,4 @@ -import { writeDeferredExecutionWait } from "../deferred-execution-wait.js"; +import { PersistedExecutionWaitRefusal, writeDeferredExecutionWait } from "../deferred-execution-wait.js"; import { notifyDeferredExecutionWork } from "../deferred-execution-notifications.js"; import type { DeferredReceiptRecoveryOptions } from "../deferred-execution-notifications.js"; import { notifyRunDispatch } from "../run-dispatch-notifications.js"; @@ -2519,17 +2519,17 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende skipReason: string, patch: Partial = {}, waitCondition?: Record, - ) => { + ): Promise<{ created: boolean; waitSaved?: boolean }> => { // A temporary refusal must leave the handoff retryable, without burning // its one durable receipt ID on a skipped wake. if (caseRequest) return { created: false }; if (executionWaitRequestId) { - await writeDeferredExecutionWait(db, { companyId: agent.companyId, requestId: executionWaitRequestId }, { + const waitSaved = await writeDeferredExecutionWait(db, { companyId: agent.companyId, requestId: executionWaitRequestId }, { reason: skipReason, message: patch.error ?? (skipReason === "issue_tree_hold_active" ? "This task is paused. Resume it to send your saved message." : "Waiting for task execution to be enabled. Your message is saved."), }); - return { created: false }; + return { created: false, waitSaved }; } const request = { ...durableReceiptFields, @@ -2801,31 +2801,30 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende }, ); if (budgetBlock) { - await writeSkippedRequest("budget.blocked", { error: budgetBlock.reason }, { + const wait = await writeSkippedRequest("budget.blocked", { error: budgetBlock.reason }, { scopeType: budgetBlock.scopeType, scopeId: budgetBlock.scopeId, }); if (caseRequest) return null; - throw conflict(budgetBlock.reason, { - scopeType: budgetBlock.scopeType, - scopeId: budgetBlock.scopeId, - }); + const details = { scopeType: budgetBlock.scopeType, scopeId: budgetBlock.scopeId }; + throw wait.waitSaved ? new PersistedExecutionWaitRefusal(budgetBlock.reason, details) + : conflict(budgetBlock.reason, details); } // Setup delays execution, but must not discard work accepted by a caller. const invokability = await getAgentInvokability(isAgentAwaitingSetup(agent) ? { ...agent, status: "idle" } : agent); if (!invokability.invokable) { if (caseRequest) return null; + let waitSaved = false; if (opts.requestedByActorType !== "user" || executionWaitRequestId) { - await writeSkippedRequest("agent.not_invokable", { + const wait = await writeSkippedRequest("agent.not_invokable", { error: invokability.message, }, { status: agent.status, reason: invokability.reason }); + waitSaved = wait.waitSaved === true; } - throw conflict(invokability.message, { - status: agent.status, - reason: invokability.reason, - invalidOrgChain: invokability.invalidOrgChain, - ...invokability.details, - }); + const details = { status: agent.status, reason: invokability.reason, + invalidOrgChain: invokability.invalidOrgChain, ...invokability.details }; + throw waitSaved ? new PersistedExecutionWaitRefusal(invokability.message, details) + : conflict(invokability.message, details); } const policy = parseHeartbeatPolicy(agent); From a8783f8acebdb50b30cc8118e9d20da18d04fd3b Mon Sep 17 00:00:00 2001 From: Dotta Date: Fri, 30 Oct 2026 20:44:52 -0500 Subject: [PATCH 73/74] fix(ui): invalidate pending startup work on sign-out Keep session-readiness failures inside the existing bounded retry policy. Co-Authored-By: Paperclip --- ui/src/hooks/useDurableAutosave.test.tsx | 69 ++++++++++++++++++++++++ ui/src/hooks/useDurableAutosave.ts | 10 +++- ui/src/hooks/useSignOut.ts | 3 ++ ui/src/lib/account-scope.ts | 6 +++ 4 files changed, 87 insertions(+), 1 deletion(-) diff --git a/ui/src/hooks/useDurableAutosave.test.tsx b/ui/src/hooks/useDurableAutosave.test.tsx index be690a90301..7f2816593c2 100644 --- a/ui/src/hooks/useDurableAutosave.test.tsx +++ b/ui/src/hooks/useDurableAutosave.test.tsx @@ -4,6 +4,8 @@ import { act } from "react"; import { createRoot, type Root } from "react-dom/client"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { ApiError } from "@/api/client"; +import { authApi } from "@/api/auth"; +import { useSignOut } from "./useSignOut"; import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; import { queryKeys } from "@/lib/queryKeys"; import { @@ -244,6 +246,73 @@ describe("useDurableAutosave", () => { }, ); + it("invalidates an unknown-account detached draft through the real sign-out reset", async () => { + store.setBrowserOnline(false); + const client = new QueryClient({ defaultOptions: { queries: { staleTime: Infinity, retry: false, enabled: false } } }); + client.setQueryData(queryKeys.health, { status: "ok", deploymentMode: "authenticated" }); + const lookup = vi.spyOn(authApi, "getSession").mockReturnValue(new Promise(() => {})); + const signOutApi = vi.spyOn(authApi, "signOut").mockResolvedValue({ success: true }); + const save = vi.fn().mockResolvedValue(undefined); + let signOut!: ReturnType; + function SignOut() { signOut = useSignOut(); return null; } + try { + act(() => root.render( + + )); + await act(async () => { await current.run("Private title before sign-out"); }); + act(() => { root.unmount(); root = createRoot(container); }); + act(() => root.render()); + await act(async () => { await signOut.mutateAsync(); }); + expect(signOutApi).toHaveBeenCalledOnce(); + expect(client.getQueryData(queryKeys.auth.session)).toBeUndefined(); + act(() => client.setQueryData(queryKeys.auth.session, { user: { id: "next-account" } })); + await act(async () => { store.setBrowserOnline(true); store.reportSuccess(); }); + await advance(2_000); + expect(save).not.toHaveBeenCalled(); + expect(store.getSnapshot().pendingWrites).toBe(0); + } finally { client.clear(); lookup.mockRestore(); signOutApi.mockRestore(); } + }); + + it.each([403, 404])("retains detached startup work after a definitive session lookup error (%s)", async status => { + const client = new QueryClient({ defaultOptions: { queries: { staleTime: Infinity, retry: false, enabled: false } } }); + const lookup = vi.spyOn(authApi, "getSession").mockRejectedValue(new ApiError("Session lookup unavailable", status, {})); + const save = vi.fn().mockRejectedValueOnce(outage()).mockResolvedValue(undefined); + try { + act(() => root.render( + + )); + await act(async () => { await current.run("Retained title"); }); + act(() => { root.unmount(); root = createRoot(container); }); + await advance(0); + expect(lookup).toHaveBeenCalled(); + expect(save).toHaveBeenCalledTimes(1); + expect(store.getSnapshot().pendingWrites).toBe(1); + act(() => client.setQueryData(queryKeys.auth.session, { user: { id: "first-account" } })); + await advance(30_000); + expect(save).toHaveBeenCalledTimes(2); + expect(save).toHaveBeenLastCalledWith("Retained title"); + expect(store.getSnapshot().pendingWrites).toBe(0); + } finally { client.clear(); lookup.mockRestore(); } + }); + + it("bounds detached session discovery when the first identity never becomes available", async () => { + const client = new QueryClient({ defaultOptions: { queries: { staleTime: Infinity, retry: false, enabled: false } } }); + const lookup = vi.spyOn(authApi, "getSession").mockRejectedValue(new ApiError("Session unavailable", 403, {})); + const save = vi.fn().mockRejectedValue(outage()); + try { + act(() => root.render( + + )); + await act(async () => { await current.run("Retained title"); }); + act(() => { root.unmount(); root = createRoot(container); }); + for (let attempt = 0; attempt < 10; attempt++) await advance(30_000); + expect(lookup.mock.calls.length).toBeGreaterThan(1); + expect(lookup.mock.calls.length).toBeLessThanOrEqual(9); + expect(save).toHaveBeenCalledTimes(1); + expect(store.getSnapshot().pendingWrites).toBe(0); + } finally { client.clear(); lookup.mockRestore(); } + }); + it("bounds retries after an editor unmounts", async () => { const save = vi.fn().mockRejectedValue(outage()); render({ save }); diff --git a/ui/src/hooks/useDurableAutosave.ts b/ui/src/hooks/useDurableAutosave.ts index 98e062b77e3..85a994d9fd7 100644 --- a/ui/src/hooks/useDurableAutosave.ts +++ b/ui/src/hooks/useDurableAutosave.ts @@ -248,7 +248,15 @@ export function useDurableAutosave(options: DurableAutosaveOptions): Durab return { kind: "saved" }; }; try { - if (!mounted.current && job.account && !job.account.isCurrentAccount()) await job.account.whenAccountReady(); + if (!mounted.current && job.account && !job.account.isCurrentAccount()) { + try { await job.account.whenAccountReady(); } + catch { + // Session discovery has not attempted the write. Keep its draft within + // the existing retry budget unless an account transition invalidated it. + if (!stillOwned()) remove(); + return { kind: "waiting" }; + } + } if (!stillOwned()) { remove(); return { kind: "waiting" }; } await job.save(job.draft); return done(); diff --git a/ui/src/hooks/useSignOut.ts b/ui/src/hooks/useSignOut.ts index 7d54525c406..ff016a8a776 100644 --- a/ui/src/hooks/useSignOut.ts +++ b/ui/src/hooks/useSignOut.ts @@ -2,6 +2,7 @@ import { useMutation, useQueryClient } from "@tanstack/react-query"; import { authApi } from "@/api/auth"; import { navigateTopLevel } from "@/lib/browserNavigation"; import { queryKeys } from "@/lib/queryKeys"; +import { invalidateAccountScope } from "@/lib/account-scope"; import { useCloudInstance } from "./useCloudInstance"; const CLOUD_SIGN_OUT_PATH = "/cloud/logout"; @@ -47,6 +48,7 @@ export function useSignOut({ onSignedOut }: UseSignOutOptions = {}) { return useMutation({ mutationFn: async () => { if (cloud) { + invalidateAccountScope(queryClient); onSignedOut?.(); navigateTopLevel(CLOUD_SIGN_OUT_PATH); return "cloud" as const; @@ -58,6 +60,7 @@ export function useSignOut({ onSignedOut }: UseSignOutOptions = {}) { onSuccess: async (target) => { if (target === "cloud") return; + invalidateAccountScope(queryClient); onSignedOut?.(); // Drop every account-scoped cache entry, rather than invalidating a diff --git a/ui/src/lib/account-scope.ts b/ui/src/lib/account-scope.ts index 0e7d050e502..eeadd51fc4c 100644 --- a/ui/src/lib/account-scope.ts +++ b/ui/src/lib/account-scope.ts @@ -11,6 +11,12 @@ function currentOwner(client: QueryClient): string | undefined { return session === undefined ? undefined : session?.user?.id ?? session?.session?.userId ?? "signed-out"; } +/** A successful sign-out ends pending work even if the first session never loaded. */ +export function invalidateAccountScope(client: QueryClient) { + const account = accounts.get(client); + if (account) account.generation += 1; +} + export function bindAccount(client: QueryClient) { let account = accounts.get(client); if (!account) { From 5e0f8aab7665435f82f9cf022e8b2c5ff0d6ecd7 Mon Sep 17 00:00:00 2001 From: Dotta Date: Fri, 30 Oct 2026 20:54:00 -0500 Subject: [PATCH 74/74] test(ui): wait for task panel activation to settle Co-Authored-By: Paperclip --- ui/src/hooks/useTaskPluginPanels.test.tsx | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/ui/src/hooks/useTaskPluginPanels.test.tsx b/ui/src/hooks/useTaskPluginPanels.test.tsx index c013137a057..bec7ca5b66c 100644 --- a/ui/src/hooks/useTaskPluginPanels.test.tsx +++ b/ui/src/hooks/useTaskPluginPanels.test.tsx @@ -143,9 +143,17 @@ it("discovers a panel started after the task is already open, then hides it on s get.mockRejectedValue(new Error("Access denied")); await refresh(); expect(current).toEqual([]); }); it("clears automatic activation when the run ends even if its view is still ready", async () => { - await render(); expect(current).not.toEqual([]); + await render(); + await vi.waitFor(async () => { + await act(async () => {}); + expect(current).not.toEqual([]); + }); await act(async () => { client.setQueryData(queryKeys.issues.liveRuns("task"), []); }); - await render(); expect(current).toEqual([]); + await render(); + await vi.waitFor(async () => { + await act(async () => {}); + expect(current).toEqual([]); + }); }); it("does not carry an old task's ready panel to a new task or disabled plugin", async () => { await render(); expect(current).not.toEqual([]);