diff --git a/server/src/modules/wake-queue/adapters/postgres.test.ts b/server/src/modules/wake-queue/adapters/postgres.test.ts index 407a80abe04..58f56f84e42 100644 --- a/server/src/modules/wake-queue/adapters/postgres.test.ts +++ b/server/src/modules/wake-queue/adapters/postgres.test.ts @@ -4,6 +4,7 @@ import { instanceSettingsService } from "../../../services/instance-settings.js" import * as executionBlocker from "../../../services/execution-blocker.js"; import { randomBytes, randomUUID } from "node:crypto"; import { and, eq, sql } from "drizzle-orm"; +import { drizzle } from "drizzle-orm/postgres-js"; import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest"; import type { Db } from "@paperclipai/db"; import { createSystemRecovery } from "../../../services/system-recovery-state.js"; @@ -257,6 +258,61 @@ describeEmbeddedPostgres("wake-queue postgres adapter", () => { } }); + it.each([false, true])("uses one scoped recovery query for a locked task without history (lease held: %s)", async leaseHeld => { + const companyId = await seedCompany(); + const agentId = await seedAgent({ companyId }); + const issueId = await seedIssue({ companyId, assigneeAgentId: agentId }); + const runId = await seedRun({ companyId, agentId, contextSnapshot: { issueId } }); + await db.update(heartbeatRuns).set({ runtimeMode: "legacy", runnerProfileJson: { adapterDispatch: { adapterType: "codex_local" } } }) + .where(eq(heartbeatRuns.id, runId)); + if (leaseHeld) await db.insert(environmentLeases).values({ companyId, heartbeatRunId: runId }); + const queries: string[] = []; + const recorded = drizzle(db.$client, { schema: db._.fullSchema, logger: { logQuery: query => queries.push(query) } }) as unknown as Db; + await createPostgresIssueWakeAdmission(recorded)({ companyId, issueId }, async (_tx, { recovery }) => { + const { blocker } = await recovery.readBlocker({ commentId: null }); + if (leaseHeld) expect(blocker).toMatchObject({ cause: "execution_owner_active", runId, recoveryActionId: null }); + else expect(blocker).toBeNull(); + return { kind: "skipped" }; + }); + expect(queries.filter(query => query.includes('from "issue_recovery_actions"'))).toEqual([ + expect.stringMatching(/^select "id" from "issue_recovery_actions" where .*"company_id".*"source_issue_id".* limit /), + ]); + }); + + it.each(["insert", "reparent"])("serializes a recovery %s and honors the resolved no-replay hold afterward", async operation => { + const companyId = await seedCompany(); + const issueId = await seedIssue({ companyId }); + const action = { + id: randomUUID(), companyId, sourceIssueId: issueId, kind: "active_run_watchdog", + cause: "legacy_execution_requires_reconciliation", fingerprint: randomUUID(), + status: "resolved", nextAction: "Inspect the stopped run.", evidence: { automaticRecovery: { replay: "blocked" } }, + }; + if (operation === "reparent") { + const otherIssueId = await seedIssue({ companyId }); + await db.insert(issueRecoveryActions).values({ ...action, sourceIssueId: otherIssueId }); + } + const writeAction = (conn: Db) => operation === "insert" + ? conn.insert(issueRecoveryActions).values(action) + : conn.update(issueRecoveryActions).set({ sourceIssueId: issueId }).where(eq(issueRecoveryActions.id, action.id)); + const admit = createPostgresIssueWakeAdmission(db); + await admit({ companyId, issueId }, async (_tx, { recovery }) => { + await expect(recovery.readBlocker({ commentId: null })).resolves.toMatchObject({ blocker: null }); + await expect(db.transaction(async writer => { + await writer.execute(sql`set local lock_timeout = '250ms'`); + await writeAction(writer as unknown as Db); + })).rejects.toMatchObject({ cause: { code: "55P03" } }); + return { kind: "skipped" }; + }); + + await writeAction(db); + await admit({ companyId, issueId }, async (_tx, { recovery }) => { + await expect(recovery.readBlocker({ commentId: null })).resolves.toMatchObject({ + blocker: { recoveryActionId: action.id, cause: action.cause }, + }); + return { kind: "skipped" }; + }); + }); + it("checks conversation reset eligibility using the row refreshed under the admission lock", async () => { const companyId = await seedCompany(); const agentId = await seedAgent({ companyId }); diff --git a/server/src/modules/wake-queue/adapters/postgres.ts b/server/src/modules/wake-queue/adapters/postgres.ts index dad4943d701..f853659b4e3 100644 --- a/server/src/modules/wake-queue/adapters/postgres.ts +++ b/server/src/modules/wake-queue/adapters/postgres.ts @@ -14,7 +14,7 @@ import { withRunRedactionHistory } from "../../../services/run-secret-redaction. import { CHAT_COMPLETION_WAKE_REASON, isCompletedOnboardingHandoffWake, settleReportedChatCompletions } from "../../../services/chat-completion-delivery.js"; import { earlierDeclinedOutcomeReply } from "../../../services/chat-interaction-outcome-report.js"; import { currentConversationCommentCondition, isConversation, isConversationReset } from "../../../services/agent-conversations.js"; -import { getExecutionBlocker, hasPendingRunCleanup } from "../../../services/execution-blocker.js"; +import { getExecutionBlocker, getExecutionOwnershipBlocker, hasPendingRunCleanup } from "../../../services/execution-blocker.js"; import { preserveTaskRemoteWorkspaceWriter } from "../../../services/native-runtime/native-failed-workspace-recovery.js"; import type { WakeRecoveryAdmissionStore } from "../application/recovery-admission.js"; import { nativeManualAdmissionHoldsExecution } from "../../../services/native-runtime/native-runner-ownership.js"; @@ -1365,6 +1365,14 @@ export function createPostgresIssueWakeAdmission(db: Db) { const verifiedReset = Boolean(resetComment && isConversationReset(resetComment.body)); const preserveWorkspace = userComment && issue?.executionWorkspaceId; if (preserveWorkspace) await preserveTaskRemoteWorkspaceWriter(tx as unknown as Db, companyId, issueId); + if (!preserveWorkspace && issue?.conversationAgentId === null) { + const [recovery] = await tx.select({ id: issueRecoveryActions.id }).from(issueRecoveryActions) + .where(and(eq(issueRecoveryActions.companyId, companyId), eq(issueRecoveryActions.sourceIssueId, issueId))).limit(1); + if (!recovery) return { + blocker: await getExecutionOwnershipBlocker(tx as unknown as Db, companyId, issueId), + verifiedReset, + }; + } const blocker = await getExecutionBlocker(tx as unknown as Db, companyId, issueId, { conversationResetCommentId: verifiedReset ? userComment : null, lockedIssue: preserveWorkspace ? undefined : issue ?? undefined, diff --git a/server/src/services/execution-blocker.ts b/server/src/services/execution-blocker.ts index ed95748d982..fdc644f4418 100644 --- a/server/src/services/execution-blocker.ts +++ b/server/src/services/execution-blocker.ts @@ -328,6 +328,11 @@ const blockerIssueFields = { executionRunId: issues.executionRunId, checkoutRunId: issues.checkoutRunId, }; +export async function getExecutionOwnershipBlocker(db: Db, companyId: string, issueId: string): Promise { + const ownership = await getConversationOwnershipBlocker(db, companyId, issueId); + return ownership ? { ...ownership, recoveryActionId: null } : null; +} + export async function getExecutionBlocker(db: Db, companyId: string, issueId: string, options?: { conversationResetCommentId?: string | null; /** The current issue row held under this transaction's issue lock. */ @@ -365,8 +370,8 @@ export async function getExecutionBlocker(db: Db, companyId: string, issueId: st eq(issueComments.id, conversation.conversationBoundaryCommentId), )).limit(1) : []; - const ownership = await getConversationOwnershipBlocker(db, companyId, issueId); - if (ownership) return { ...ownership, recoveryActionId: null }; + const ownership = await getExecutionOwnershipBlocker(db, companyId, issueId); + if (ownership) return ownership; const [action] = restoreHold ? [restoreHold] : await db.select().from(issueRecoveryActions).where(and( eq(issueRecoveryActions.companyId, companyId), eq(issueRecoveryActions.sourceIssueId, issueId), diff --git a/server/src/services/heartbeat/queue.test.ts b/server/src/services/heartbeat/queue.test.ts index ecc292533aa..d0cfc04d0a9 100644 --- a/server/src/services/heartbeat/queue.test.ts +++ b/server/src/services/heartbeat/queue.test.ts @@ -2,7 +2,7 @@ import { randomUUID } from "node:crypto"; import { contextCompactionService } from "../context-compaction.js"; import { claimNativeSessionResumptions } from "../native-runtime/native-finalization-reconciler.js"; import type { Db } from "@paperclipai/db"; -import { authUsers, agents, agentTaskSessions, agentWakeupRequests, budgetIncidents, budgetPolicies, caseHandoffs, cases, chatCompletionDeliveries, companies, companyMemberships, costEvents, createDb, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS, environmentLeases, heartbeatRuns, issueComments, issueThreadInteractions, issues, nativeRunFinalizations, routines, routineRuns } from "@paperclipai/db"; +import { authUsers, agents, agentTaskSessions, agentWakeupRequests, budgetIncidents, budgetPolicies, caseHandoffs, cases, chatCompletionDeliveries, companies, companyMemberships, costEvents, createDb, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS, environmentLeases, heartbeatRuns, issueComments, issueRecoveryActions, issueThreadInteractions, issues, nativeRunFinalizations, routines, routineRuns } from "@paperclipai/db"; import { eq, inArray, sql } from "drizzle-orm"; import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; import { resolveExternalTestDatabaseUrl } from "../../__tests__/helpers/external-test-database.js"; @@ -325,6 +325,53 @@ describe.skipIf(!support.supported)("heartbeat queue database wiring", () => { expect(await db.select().from(heartbeatRuns)).toHaveLength(kind === "fresh" ? 0 : 1); }); + it("defers an ordinary user wake behind a recovery hold committed while acquiring the task lock", async () => { + const { user, company, agent, issue, deps, queue } = await waitingConversationFixture(); + await db.update(issues).set({ conversationAgentId: null, conversationUserId: null, conversationState: null, status: "todo" }) + .where(eq(issues.id, issue.id)); + const [comment] = await db.insert(issueComments).values({ + companyId: company.id, issueId: issue.id, authorType: "user", authorUserId: user.id, body: "Please continue.", + }).returning(); + const actionId = randomUUID(); + function enqueue() { + return queue.enqueueWakeup(agent.id, { + source: "assignment", triggerDetail: "system", reason: "issue_commented", + requestedByActorType: "user", requestedByActorId: user.id, payload: { issueId: issue.id, commentId: comment.id }, + }); + } + let pending: ReturnType | undefined; + try { + await db.transaction(async writer => { + const [{ pid }] = await writer.execute<{ pid: number }>(sql`select pg_backend_pid() as pid`); + await writer.insert(issueRecoveryActions).values({ + id: actionId, companyId: company.id, sourceIssueId: issue.id, kind: "active_run_watchdog", + cause: "legacy_execution_requires_reconciliation", fingerprint: actionId, status: "resolved", + evidence: { automaticRecovery: { replay: "blocked" } }, nextAction: "Inspect the stopped run.", + }); + pending = enqueue(); + await vi.waitFor(async () => { + const [waiter] = await db.execute<{ waiting: boolean }>(sql`select exists ( + select 1 from pg_stat_activity + where wait_event_type = 'Lock' and ${pid} = any(pg_blocking_pids(pg_stat_activity.pid)) + and query ilike '%for update%' + ) as waiting`); + expect(waiter.waiting).toBe(true); + }, { timeout: 5_000, interval: 20 }); + }); + } finally { + await pending; + } + + await expect(pending).resolves.toBeNull(); + expect(await db.select().from(heartbeatRuns)).toEqual([]); + expect(await db.select().from(agentWakeupRequests)).toEqual([ + expect.objectContaining({ status: "deferred_issue_execution", runId: null, + payload: expect.objectContaining({ issueId: issue.id, commentId: comment.id, + executionWait: expect.objectContaining({ recoveryActionId: actionId }) }) }), + ]); + expect(deps.executeRun).not.toHaveBeenCalled(); + }); + it("uses issue changes written by durable chat authorization before admission", async () => { const { user, company, agent, issue, queue } = await waitingConversationFixture(); await db.update(issues).set({ conversationAgentId: null, conversationUserId: null, conversationState: null, status: "todo" })