From c55de053f5fd0239f8cbc2f1dfb61f82b82eb2a2 Mon Sep 17 00:00:00 2001 From: Nicky Leach Date: Thu, 19 Nov 2026 07:23:03 -0800 Subject: [PATCH] perf(wake-queue): skip detailed recovery reads on tasks without history (#22290) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Thinking Path > - Paperclip keeps agents from resuming work that still has an execution owner or a recovery hold. > - Ordinary board tasks have no recovery record. > - Admission evaluates two detailed recovery queries for those tasks while it holds the issue row lock. > - The source-issue foreign key prevents another transaction from inserting or moving the first recovery record onto the locked issue. > - Use one indexed existence check there. Continue to check execution ownership and apply the complete policy to tasks with history. ## Linked Issues or Issue Description **What existing behavior does this improve?** Admitting board assignments and saved task comments. **Subsystem affected** Wake-queue’s PostgreSQL adapter for task admission. **Current behavior** Admission evaluates detailed recovery predicates even when a locked board task has no recovery history. **Proposed behavior** For a locked board task that did not require writer preservation, check for any recovery row for that company and task. An empty result skips the recovery predicates for that call. **Additional context** Depends on #22292, which enforces recovery eligibility after wake-queue acquires the task lock. #22159 and #21459 address separate recovery lifecycle decisions. ## What Changed - Preserve the independent ownership and lease check. - Use the existing detailed recovery logic when history exists. Include resolved records in the existence check. - Run the probe after acquiring the issue lock, in its own statement. Cover both orderings of a concurrent recovery insert and a record moved onto the issue. ## Verification At 83d4acd8bf the two query-shape tests failed on the parent and passed on the new implementation. All 425 tests across five suites passed on that revision, including all five focused PostgreSQL regressions. A normal user comment waiting on an uncommitted recovery record creates no run after the writer commits. Its deferred receipt retains the exact comment and recovery action. Server TypeScript and the module dependency check passed. The published head 9783a92ae9 removes six inline comment lines across both layers. All ten affected application and PostgreSQL regression tests passed again on that head. We repeated the whole-admission comparison in both orders after other local tests finished. Each condition had 34 warmed samples per revision. With no recovery history, median times fell from 21.45 to 10.35 ms for a new task, 21.80 to 11.40 ms for one comment, and 44.50 to 22.85 ms for two serial comments. With one resolved record, median times rose from 20.80 to 22.65 ms, 22.00 to 22.90 ms, and 44.30 to 48.05 ms. All runs retained the expected receipts and comment IDs. The isolated blocker read used one fewer statement when history was empty, and one more when history existed. In 100 paired reads per condition, the mean empty-history saving was 10.54 ms and the mean resolved-history cost was 0.18 ms. The latter estimate varied enough to include zero. The separate post-lock statement sees any recovery insert that committed while admission waited for the issue lock. These local timings exclude HTTP and concurrent workload latency. The full stack passed verification and E2E on 9783a92ae9. [CI run](https://github.com/paperclipai/paperclip/actions/runs/44461026015) · [Standard Greptile review, 5/5](https://github.com/paperclipai/paperclip/pull/22290#issuecomment-6999610966) ## Risks Locked board tasks with recovery history pay one additional indexed lookup. The adapter owns the issue lock. The proof of absence depends on the source-issue foreign key and on recovery writers preserving company identity. The regression checks contention on the source-issue foreign key and that the next admission sees the new hold. ## Model Used OpenAI GPT-6.2 Sol (gpt-6.2-sol) implemented the change. OpenAI Codex reviewed and qualified it. ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above - [x] I have either (a) linked existing issues with `Fixes: #` / `Closes #` / `Refs #` OR (b) described the issue in-PR following the relevant issue template - [x] I have not referenced internal/instance-local Paperclip issues or links (only public GitHub `#NNN` / `github.com/paperclipai/paperclip` URLs) - [x] My branch name describes the change (e.g. `docs/...`, `fix/...`) and contains no internal Paperclip ticket id or instance-derived details - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [x] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [x] All Paperclip CI gates are green - [x] Greptile is 5/5 with no open P2s, recommendations, or follow-ups - [x] I will address all Greptile and reviewer comments before requesting merge --- .../wake-queue/adapters/postgres.test.ts | 56 +++++++++++++++++++ .../modules/wake-queue/adapters/postgres.ts | 10 +++- server/src/services/execution-blocker.ts | 9 ++- server/src/services/heartbeat/queue.test.ts | 49 +++++++++++++++- 4 files changed, 120 insertions(+), 4 deletions(-) diff --git a/server/src/modules/wake-queue/adapters/postgres.test.ts b/server/src/modules/wake-queue/adapters/postgres.test.ts index 407a80abe04..58f56f84e42 100644 --- a/server/src/modules/wake-queue/adapters/postgres.test.ts +++ b/server/src/modules/wake-queue/adapters/postgres.test.ts @@ -4,6 +4,7 @@ import { instanceSettingsService } from "../../../services/instance-settings.js" import * as executionBlocker from "../../../services/execution-blocker.js"; import { randomBytes, randomUUID } from "node:crypto"; import { and, eq, sql } from "drizzle-orm"; +import { drizzle } from "drizzle-orm/postgres-js"; import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest"; import type { Db } from "@paperclipai/db"; import { createSystemRecovery } from "../../../services/system-recovery-state.js"; @@ -257,6 +258,61 @@ describeEmbeddedPostgres("wake-queue postgres adapter", () => { } }); + it.each([false, true])("uses one scoped recovery query for a locked task without history (lease held: %s)", async leaseHeld => { + const companyId = await seedCompany(); + const agentId = await seedAgent({ companyId }); + const issueId = await seedIssue({ companyId, assigneeAgentId: agentId }); + const runId = await seedRun({ companyId, agentId, contextSnapshot: { issueId } }); + await db.update(heartbeatRuns).set({ runtimeMode: "legacy", runnerProfileJson: { adapterDispatch: { adapterType: "codex_local" } } }) + .where(eq(heartbeatRuns.id, runId)); + if (leaseHeld) await db.insert(environmentLeases).values({ companyId, heartbeatRunId: runId }); + const queries: string[] = []; + const recorded = drizzle(db.$client, { schema: db._.fullSchema, logger: { logQuery: query => queries.push(query) } }) as unknown as Db; + await createPostgresIssueWakeAdmission(recorded)({ companyId, issueId }, async (_tx, { recovery }) => { + const { blocker } = await recovery.readBlocker({ commentId: null }); + if (leaseHeld) expect(blocker).toMatchObject({ cause: "execution_owner_active", runId, recoveryActionId: null }); + else expect(blocker).toBeNull(); + return { kind: "skipped" }; + }); + expect(queries.filter(query => query.includes('from "issue_recovery_actions"'))).toEqual([ + expect.stringMatching(/^select "id" from "issue_recovery_actions" where .*"company_id".*"source_issue_id".* limit /), + ]); + }); + + it.each(["insert", "reparent"])("serializes a recovery %s and honors the resolved no-replay hold afterward", async operation => { + const companyId = await seedCompany(); + const issueId = await seedIssue({ companyId }); + const action = { + id: randomUUID(), companyId, sourceIssueId: issueId, kind: "active_run_watchdog", + cause: "legacy_execution_requires_reconciliation", fingerprint: randomUUID(), + status: "resolved", nextAction: "Inspect the stopped run.", evidence: { automaticRecovery: { replay: "blocked" } }, + }; + if (operation === "reparent") { + const otherIssueId = await seedIssue({ companyId }); + await db.insert(issueRecoveryActions).values({ ...action, sourceIssueId: otherIssueId }); + } + const writeAction = (conn: Db) => operation === "insert" + ? conn.insert(issueRecoveryActions).values(action) + : conn.update(issueRecoveryActions).set({ sourceIssueId: issueId }).where(eq(issueRecoveryActions.id, action.id)); + const admit = createPostgresIssueWakeAdmission(db); + await admit({ companyId, issueId }, async (_tx, { recovery }) => { + await expect(recovery.readBlocker({ commentId: null })).resolves.toMatchObject({ blocker: null }); + await expect(db.transaction(async writer => { + await writer.execute(sql`set local lock_timeout = '250ms'`); + await writeAction(writer as unknown as Db); + })).rejects.toMatchObject({ cause: { code: "55P03" } }); + return { kind: "skipped" }; + }); + + await writeAction(db); + await admit({ companyId, issueId }, async (_tx, { recovery }) => { + await expect(recovery.readBlocker({ commentId: null })).resolves.toMatchObject({ + blocker: { recoveryActionId: action.id, cause: action.cause }, + }); + return { kind: "skipped" }; + }); + }); + it("checks conversation reset eligibility using the row refreshed under the admission lock", async () => { const companyId = await seedCompany(); const agentId = await seedAgent({ companyId }); diff --git a/server/src/modules/wake-queue/adapters/postgres.ts b/server/src/modules/wake-queue/adapters/postgres.ts index dad4943d701..f853659b4e3 100644 --- a/server/src/modules/wake-queue/adapters/postgres.ts +++ b/server/src/modules/wake-queue/adapters/postgres.ts @@ -14,7 +14,7 @@ import { withRunRedactionHistory } from "../../../services/run-secret-redaction. import { CHAT_COMPLETION_WAKE_REASON, isCompletedOnboardingHandoffWake, settleReportedChatCompletions } from "../../../services/chat-completion-delivery.js"; import { earlierDeclinedOutcomeReply } from "../../../services/chat-interaction-outcome-report.js"; import { currentConversationCommentCondition, isConversation, isConversationReset } from "../../../services/agent-conversations.js"; -import { getExecutionBlocker, hasPendingRunCleanup } from "../../../services/execution-blocker.js"; +import { getExecutionBlocker, getExecutionOwnershipBlocker, hasPendingRunCleanup } from "../../../services/execution-blocker.js"; import { preserveTaskRemoteWorkspaceWriter } from "../../../services/native-runtime/native-failed-workspace-recovery.js"; import type { WakeRecoveryAdmissionStore } from "../application/recovery-admission.js"; import { nativeManualAdmissionHoldsExecution } from "../../../services/native-runtime/native-runner-ownership.js"; @@ -1365,6 +1365,14 @@ export function createPostgresIssueWakeAdmission(db: Db) { const verifiedReset = Boolean(resetComment && isConversationReset(resetComment.body)); const preserveWorkspace = userComment && issue?.executionWorkspaceId; if (preserveWorkspace) await preserveTaskRemoteWorkspaceWriter(tx as unknown as Db, companyId, issueId); + if (!preserveWorkspace && issue?.conversationAgentId === null) { + const [recovery] = await tx.select({ id: issueRecoveryActions.id }).from(issueRecoveryActions) + .where(and(eq(issueRecoveryActions.companyId, companyId), eq(issueRecoveryActions.sourceIssueId, issueId))).limit(1); + if (!recovery) return { + blocker: await getExecutionOwnershipBlocker(tx as unknown as Db, companyId, issueId), + verifiedReset, + }; + } const blocker = await getExecutionBlocker(tx as unknown as Db, companyId, issueId, { conversationResetCommentId: verifiedReset ? userComment : null, lockedIssue: preserveWorkspace ? undefined : issue ?? undefined, diff --git a/server/src/services/execution-blocker.ts b/server/src/services/execution-blocker.ts index ed95748d982..fdc644f4418 100644 --- a/server/src/services/execution-blocker.ts +++ b/server/src/services/execution-blocker.ts @@ -328,6 +328,11 @@ const blockerIssueFields = { executionRunId: issues.executionRunId, checkoutRunId: issues.checkoutRunId, }; +export async function getExecutionOwnershipBlocker(db: Db, companyId: string, issueId: string): Promise { + const ownership = await getConversationOwnershipBlocker(db, companyId, issueId); + return ownership ? { ...ownership, recoveryActionId: null } : null; +} + export async function getExecutionBlocker(db: Db, companyId: string, issueId: string, options?: { conversationResetCommentId?: string | null; /** The current issue row held under this transaction's issue lock. */ @@ -365,8 +370,8 @@ export async function getExecutionBlocker(db: Db, companyId: string, issueId: st eq(issueComments.id, conversation.conversationBoundaryCommentId), )).limit(1) : []; - const ownership = await getConversationOwnershipBlocker(db, companyId, issueId); - if (ownership) return { ...ownership, recoveryActionId: null }; + const ownership = await getExecutionOwnershipBlocker(db, companyId, issueId); + if (ownership) return ownership; const [action] = restoreHold ? [restoreHold] : await db.select().from(issueRecoveryActions).where(and( eq(issueRecoveryActions.companyId, companyId), eq(issueRecoveryActions.sourceIssueId, issueId), diff --git a/server/src/services/heartbeat/queue.test.ts b/server/src/services/heartbeat/queue.test.ts index ecc292533aa..d0cfc04d0a9 100644 --- a/server/src/services/heartbeat/queue.test.ts +++ b/server/src/services/heartbeat/queue.test.ts @@ -2,7 +2,7 @@ import { randomUUID } from "node:crypto"; import { contextCompactionService } from "../context-compaction.js"; import { claimNativeSessionResumptions } from "../native-runtime/native-finalization-reconciler.js"; import type { Db } from "@paperclipai/db"; -import { authUsers, agents, agentTaskSessions, agentWakeupRequests, budgetIncidents, budgetPolicies, caseHandoffs, cases, chatCompletionDeliveries, companies, companyMemberships, costEvents, createDb, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS, environmentLeases, heartbeatRuns, issueComments, issueThreadInteractions, issues, nativeRunFinalizations, routines, routineRuns } from "@paperclipai/db"; +import { authUsers, agents, agentTaskSessions, agentWakeupRequests, budgetIncidents, budgetPolicies, caseHandoffs, cases, chatCompletionDeliveries, companies, companyMemberships, costEvents, createDb, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS, environmentLeases, heartbeatRuns, issueComments, issueRecoveryActions, issueThreadInteractions, issues, nativeRunFinalizations, routines, routineRuns } from "@paperclipai/db"; import { eq, inArray, sql } from "drizzle-orm"; import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; import { resolveExternalTestDatabaseUrl } from "../../__tests__/helpers/external-test-database.js"; @@ -325,6 +325,53 @@ describe.skipIf(!support.supported)("heartbeat queue database wiring", () => { expect(await db.select().from(heartbeatRuns)).toHaveLength(kind === "fresh" ? 0 : 1); }); + it("defers an ordinary user wake behind a recovery hold committed while acquiring the task lock", async () => { + const { user, company, agent, issue, deps, queue } = await waitingConversationFixture(); + await db.update(issues).set({ conversationAgentId: null, conversationUserId: null, conversationState: null, status: "todo" }) + .where(eq(issues.id, issue.id)); + const [comment] = await db.insert(issueComments).values({ + companyId: company.id, issueId: issue.id, authorType: "user", authorUserId: user.id, body: "Please continue.", + }).returning(); + const actionId = randomUUID(); + function enqueue() { + return queue.enqueueWakeup(agent.id, { + source: "assignment", triggerDetail: "system", reason: "issue_commented", + requestedByActorType: "user", requestedByActorId: user.id, payload: { issueId: issue.id, commentId: comment.id }, + }); + } + let pending: ReturnType | undefined; + try { + await db.transaction(async writer => { + const [{ pid }] = await writer.execute<{ pid: number }>(sql`select pg_backend_pid() as pid`); + await writer.insert(issueRecoveryActions).values({ + id: actionId, companyId: company.id, sourceIssueId: issue.id, kind: "active_run_watchdog", + cause: "legacy_execution_requires_reconciliation", fingerprint: actionId, status: "resolved", + evidence: { automaticRecovery: { replay: "blocked" } }, nextAction: "Inspect the stopped run.", + }); + pending = enqueue(); + await vi.waitFor(async () => { + const [waiter] = await db.execute<{ waiting: boolean }>(sql`select exists ( + select 1 from pg_stat_activity + where wait_event_type = 'Lock' and ${pid} = any(pg_blocking_pids(pg_stat_activity.pid)) + and query ilike '%for update%' + ) as waiting`); + expect(waiter.waiting).toBe(true); + }, { timeout: 5_000, interval: 20 }); + }); + } finally { + await pending; + } + + await expect(pending).resolves.toBeNull(); + expect(await db.select().from(heartbeatRuns)).toEqual([]); + expect(await db.select().from(agentWakeupRequests)).toEqual([ + expect.objectContaining({ status: "deferred_issue_execution", runId: null, + payload: expect.objectContaining({ issueId: issue.id, commentId: comment.id, + executionWait: expect.objectContaining({ recoveryActionId: actionId }) }) }), + ]); + expect(deps.executeRun).not.toHaveBeenCalled(); + }); + it("uses issue changes written by durable chat authorization before admission", async () => { const { user, company, agent, issue, queue } = await waitingConversationFixture(); await db.update(issues).set({ conversationAgentId: null, conversationUserId: null, conversationState: null, status: "todo" })