From defed64af9255ef9516850828eb5930685355c63 Mon Sep 17 00:00:00 2001 From: Devin Foley Date: Fri, 13 Nov 2026 14:02:04 -0800 Subject: [PATCH] fix(native-runtime): retry a native failure write after a brief lock timeout (#21494) An interrupted native turn records its same-run resume in one transaction under a 1 s lock_timeout. When another transaction briefly held a row that the write needs (for example the agent row that the new recovery action references), the insert failed with 55P03 and the whole write rolled back. The run kept its dead lease, and the task needed reconciliation instead of the recorded automatic resume. Retry the whole failure write on transient lock contention with the existing bounded helper. A rolled-back attempt has no effect, so a fresh attempt is safe. The regression test holds the agent row until one attempt is cancelled by its lock timeout, so only a retry can complete the write. Co-Authored-By: Paperclip Co-Authored-By: Claude Opus 5.5 --- ...-write-lock-contention.integration.test.ts | 111 ++++++++++++++++++ .../native-runtime/native-session-executor.ts | 9 +- 2 files changed, 118 insertions(+), 2 deletions(-) create mode 100644 server/src/services/native-runtime/native-failure-write-lock-contention.integration.test.ts diff --git a/server/src/services/native-runtime/native-failure-write-lock-contention.integration.test.ts b/server/src/services/native-runtime/native-failure-write-lock-contention.integration.test.ts new file mode 100644 index 00000000000..d1b27c39295 --- /dev/null +++ b/server/src/services/native-runtime/native-failure-write-lock-contention.integration.test.ts @@ -0,0 +1,111 @@ +import { randomUUID } from "node:crypto"; +import { and, eq, sql } from "drizzle-orm"; +import { afterAll, beforeAll, describe, expect, it, vi } from "vitest"; +import { agents, companies, companyMemberships, completionContracts, createDb, heartbeatRuns, issueRecoveryActions, issues, + nativeRunFinalizations } from "@paperclipai/db"; +import type { NativeExecutionInputV1 } from "@paperclipai/paperclip-runner"; +import { startEmbeddedPostgresTestDatabase } from "../../__tests__/helpers/embedded-postgres.js"; + +const provider = vi.hoisted(() => ({ execute: vi.fn() })); +vi.mock("../../vendor/paperclip-runner/index.js", async (importOriginal) => ({ + ...(await importOriginal()), + executeNativeSession: provider.execute, +})); + +import { executePaperclipNativeSession } from "./native-session-executor.js"; +import { buildNativeCompletionContract } from "./completion-contracts.js"; +import { prepareNativeHeartbeatRun } from "./prepare-native-run.js"; + +// Staging: an interrupted provider turn wrote its +// same-run resume under a 1 s lock_timeout. Another transaction briefly held +// the agent row, so the recovery action's foreign-key check timed out (55P03), +// the whole failure write rolled back, and a resumable run became a task +// blocked on native_continuation_requires_reconciliation. +describe("native failure write under brief lock contention", () => { + let temporary: Awaited>; + let db: ReturnType; + const companyId = randomUUID(); + const agentId = randomUUID(); + + beforeAll(async () => { + temporary = await startEmbeddedPostgresTestDatabase("native-failure-lock-"); + db = createDb(temporary.connectionString); + await db.insert(companies).values({ id: companyId, name: "Failure lock", issuePrefix: "NFL" }); + await db.insert(agents).values({ id: agentId, companyId, name: "OpenCode agent", status: "active", adapterType: "paperclip_runner", + adapterConfig: { provider: "opencode", model: "openrouter/test-model" } }); + await db.insert(companyMemberships).values({ companyId, principalType: "user", principalId: "board-user", status: "active", membershipRole: "member" }); + }); + afterAll(async () => { await temporary?.cleanup(); }); + + it("records the same-run resume after the competing lock is released", async () => { + const issueId = randomUUID(); + await db.insert(issues).values({ id: issueId, companyId, title: "Write the brief", status: "in_progress", workMode: "standard", + assigneeAgentId: agentId, createdByUserId: "board-user", responsibleUserId: "board-user" }); + const [issue] = await db.select().from(issues).where(eq(issues.id, issueId)); + const runId = randomUUID(); + const [run] = await db.insert(heartbeatRuns).values({ id: runId, companyId, agentId, status: "running", runtimeMode: "native", + nativeIssueId: issueId, responsibleUserId: "board-user", invocationSource: "automation", triggerDetail: "system", + contextSnapshot: { issueId } }).returning(); + await db.update(issues).set({ executionRunId: runId }).where(eq(issues.id, issueId)); + const native = await prepareNativeHeartbeatRun({ db, run, issue, environmentLeaseId: randomUUID() }); + const [contract] = await db.select().from(completionContracts).where(eq(completionContracts.issueId, issueId)); + await db.insert(nativeRunFinalizations).values({ runId, companyId, issueId, phase: "observed" }); + const execution: NativeExecutionInputV1 = { + schema: "paperclip.native-execution-input.v1", + binding: { companyId, runId, issueId, agentId, executionWorkspaceId: runId }, + provider: { kind: "opencode", model: "openrouter/test-model" }, + task: { identifier: issueId, title: issue.title, description: null, prompt: "Write the brief.", workMode: "standard" }, + workspace: { cwd: "/tmp", repoUrl: null, repoRef: null, branchName: null }, + session: { normalizedSessionId: native.normalizedSessionId, driverKind: "opencode_server", protocolVersion: 1, lifecyclePolicy: { mode: "per_turn", idleTimeoutMs: null } }, + completionContract: { id: contract.id, sha256: contract.canonicalSha256, schemaVersion: "paperclip.completion-contract.v1", + contract: buildNativeCompletionContract(issue, { revision: contract.revision }) }, + interactionResponses: [], credentialBindings: [], + }; + await db.update(heartbeatRuns).set({ runnerProfileJson: { nativeExecutionInput: execution, + sessionCheckpoint: { backendKind: "mock", providerSessionId: "opencode-session" } } }).where(eq(heartbeatRuns.id, runId)); + + // Hold the agent row from another connection across the failure write. + // The new recovery action references this agent, so its insert must wait. + // Release the row only after a write attempt was blocked by it and then + // cancelled by its lock timeout, so only a later attempt can succeed. + let holder: Promise | undefined; + let cancelledAttempt = false; + provider.execute.mockReset().mockImplementation(async () => { + let locked!: () => void; + const acquired = new Promise(resolve => { locked = resolve; }); + holder = db.transaction(async tx => { + await tx.select({ id: agents.id }).from(agents).where(eq(agents.id, agentId)).for("update"); + const holderPid = (await tx.execute<{ pid: number }>(sql`select pg_backend_pid() as pid`))[0]!.pid; + locked(); + let blocked = false; + for (let poll = 0; poll < 1_000 && !cancelledAttempt; poll++) { + const rows = (await db.execute( + sql`select 1 from pg_stat_activity where ${holderPid} = any(pg_blocking_pids(pid))`, + )) as unknown as Array; + if (rows[0]) blocked = true; + else if (blocked) cancelledAttempt = true; + await new Promise(resolve => setTimeout(resolve, 10)); + } + }); + await acquired; + throw new Error("PRP command turn.start indeterminate: runner connection closed"); + }); + await executePaperclipNativeSession({ db, execution, runnerInstanceId: native.runnerInstanceId, backend: { + descriptor: async () => { throw new Error("Unexpected provider call"); }, + openSession: async () => { throw new Error("Unexpected provider call"); }, + } }).catch(() => undefined); + await holder; + expect(cancelledAttempt).toBe(true); + + const [coordinator] = await db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, runId)); + expect(coordinator).toMatchObject({ phase: "retryable_failure", failureCode: "native_session_interrupted", + recoveryState: "resuming_session", leaseOwner: null }); + const actions = await db.select().from(issueRecoveryActions).where(and( + eq(issueRecoveryActions.companyId, companyId), eq(issueRecoveryActions.sourceIssueId, issueId))); + expect(actions).toHaveLength(1); + expect(actions[0]).toMatchObject({ status: "active", kind: "active_run_watchdog", cause: "native_session_interrupted", + ownerAgentId: agentId, wakePolicy: expect.objectContaining({ kind: "resume_native_run", runId }) }); + const [failedRun] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, runId)); + expect(failedRun.errorCode).toBe("native_session_interrupted"); + }, 120_000); +}); diff --git a/server/src/services/native-runtime/native-session-executor.ts b/server/src/services/native-runtime/native-session-executor.ts index b1cc715e51f..0878e6b9128 100644 --- a/server/src/services/native-runtime/native-session-executor.ts +++ b/server/src/services/native-runtime/native-session-executor.ts @@ -1,6 +1,7 @@ import { currentControllerProcess } from "../controller-process-identity.js"; import { remoteRecoveryLeaseMismatch } from "./remote-recovery-lease-mismatch.js"; import { isGovernancePauseStop } from "../governance-pause-stop.js"; +import { retryTransientLockContention } from "../transient-lock-contention.js"; import { nativeLeaseHolder, notifyNativeTokenLeaseClaimed } from "./native-token-lease-settlement.js"; import { captureRemoteNativeProviderArtifacts, shareNativeProviderArtifactCapture } from "./native-provider-artifact-capture.js"; import { NativeSessionDiagnostics, nativeSessionDiagnostics, type NativeDiagnosticLease } from "./native-session-diagnostics.js"; @@ -13101,7 +13102,11 @@ async function executePaperclipNativeSessionAttempt( // transitions the run into "failed". Read after the transaction // commits, so a rolled-back write never reports a false failure. let terminalRunToReport: typeof heartbeatRuns.$inferSelect | null = null; - await input.db.transaction(async (tx) => { + // A lock timeout rolls this write back whole, so a fresh attempt is safe. + // Without it a briefly held row strands the run under its dead lease and + // the task needs reconciliation instead of its recorded recovery. + await retryTransientLockContention(() => input.db.transaction(async (tx) => { + terminalRunToReport = null; await tx.execute( sql`select set_config('statement_timeout', '15000', true), set_config('lock_timeout', '1000', true)`, ); @@ -13496,7 +13501,7 @@ async function executePaperclipNativeSessionAttempt( supersedeOnIdentityChange: recoveryProjection.supersedeOnIdentityChange, }); - }).catch(async (failure: unknown) => { + })).catch(async (failure: unknown) => { // Emit after rollback so the diagnostic sink cannot deadlock on the // coordinator transaction. Reporting failure cannot change fencing. if (failure instanceof NativePostResultFailureError) {