diff --git a/packages/db/src/index.ts b/packages/db/src/index.ts index 0c4c0fcd9e8..7e41782d377 100644 --- a/packages/db/src/index.ts +++ b/packages/db/src/index.ts @@ -54,5 +54,5 @@ export { issueRelations } from "./schema/issue_relations.js"; export { issueReferenceMentions } from "./schema/issue_reference_mentions.js"; export * from "./schema/index.js"; -export { databaseWorkScopeKey, signalDatabaseWork, subscribeDatabaseWork, databaseWorkPending, reconcileDatabaseWork } from "./work-signals.js"; +export { databaseWorkScopeKey, primeDatabaseWorkTransactionId, signalDatabaseWork, subscribeDatabaseWork, databaseWorkPending, reconcileDatabaseWork } from "./work-signals.js"; export { withDatabaseOperation } from "./response-capture.js"; diff --git a/packages/db/src/work-signals.test.ts b/packages/db/src/work-signals.test.ts index b98a868af96..704afac48a7 100644 --- a/packages/db/src/work-signals.test.ts +++ b/packages/db/src/work-signals.test.ts @@ -3,7 +3,7 @@ import { drizzle } from "drizzle-orm/postgres-js"; import { afterAll, beforeAll, describe, expect, it, vi } from "vitest"; import { createDb, withDedicatedDbConnection } from "./client.js"; import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./test-embedded-postgres.js"; -import { DATABASE_WORK_CHANNEL, databaseWorkScopeKey, databaseWorkPending, enableDatabaseWorkNotifications, installDatabaseWorkSignals, notifyDatabaseWorkOnCommit, reconcileDatabaseWork, signalDatabaseWork, subscribeDatabaseWork } from "./work-signals.js"; +import { DATABASE_WORK_CHANNEL, databaseWorkScopeKey, databaseWorkPending, enableDatabaseWorkNotifications, installDatabaseWorkSignals, notifyDatabaseWorkOnCommit, primeDatabaseWorkTransactionId, reconcileDatabaseWork, signalDatabaseWork, subscribeDatabaseWork } from "./work-signals.js"; const support = await getEmbeddedPostgresTestSupport(); (support.supported ? describe : describe.skip)("database work signals", () => { @@ -145,6 +145,22 @@ const support = await getEmbeddedPostgresTestSupport(); } finally { release(); await serverDone?.catch(() => {}); unsubscribe(); } }); + it("resolves an uncertain transaction with a transaction id that a statement returned", async () => { + await expect(db.transaction(async tx => { + const [{ xid }] = await tx.execute(sql`select pg_current_xact_id()::text as xid`); + primeDatabaseWorkTransactionId(tx, xid as string); + await signalDatabaseWork(tx, "primed-disconnect"); + await tx.execute(sql`insert into work_signal_test values ('primed-disconnect')`); + const [{ pid }] = await tx.execute(sql`select pg_backend_pid() as pid`); + await db.execute(sql`select pg_terminate_backend(${pid as number})`); + await tx.execute(sql`select 1`); + })).rejects.toThrow(); + expect(databaseWorkPending(db, "primed-disconnect")).toBe(true); + await reconcileDatabaseWork(db, "primed-disconnect"); + expect(databaseWorkPending(db, "primed-disconnect")).toBe(false); + expect(await db.execute(sql`select * from work_signal_test where id = 'primed-disconnect'`)).toHaveLength(0); + }); + it("recovers after a real backend disconnect without replaying the queue write", async () => { await expect(db.transaction(async tx => { await signalDatabaseWork(tx, "disconnect"); @@ -196,6 +212,17 @@ describe("work signal query boundaries", () => { expect(write).not.toHaveBeenCalled(); expect(databaseWorkPending(connection, "failed")).toBe(false); }); + it("reads no transaction id when a statement of the transaction already returned it", async () => { + const execute = vi.fn(); + const connection = installDatabaseWorkSignals({ execute, transaction: async (callback: any) => callback({ execute, transaction: async () => {} }) }); + primeDatabaseWorkTransactionId(connection, "outside"); + await connection.transaction(async (tx: any) => { + primeDatabaseWorkTransactionId(tx, "42"); + await signalDatabaseWork(tx, "primed"); + await signalDatabaseWork(tx, "primed-again"); + }); + expect(execute).not.toHaveBeenCalled(); + }); it("rejects registration on an autocommit root", async () => { const connection = installDatabaseWorkSignals({ execute: vi.fn(), transaction: vi.fn() }); await expect(signalDatabaseWork(connection, "bad")).rejects.toThrow("inside a transaction"); diff --git a/packages/db/src/work-signals.ts b/packages/db/src/work-signals.ts index 05b83a417cf..aa312072cd7 100644 --- a/packages/db/src/work-signals.ts +++ b/packages/db/src/work-signals.ts @@ -88,6 +88,11 @@ export async function signalDatabaseWork(connection: object, topic: string, hint if (context.scope.notify) notifyAtCommit(context, notification(context.scope, topic)); } +export function primeDatabaseWorkTransactionId(connection: object, xid: string): void { + const state = contexts.get(connection)?.transaction; + if (state && !state.xid) state.xid = Promise.resolve(xid); +} + function notifyAtCommit(context: Context, payload: string): void { if (context.ended) throw new Error("Database work was signalled after its transaction or savepoint ended"); const key = `${DATABASE_WORK_CHANNEL}\u0000${payload}`; diff --git a/server/src/__tests__/heartbeat-run-event-atomic-append.test.ts b/server/src/__tests__/heartbeat-run-event-atomic-append.test.ts index b805d99e9a6..9ff9edb5694 100644 --- a/server/src/__tests__/heartbeat-run-event-atomic-append.test.ts +++ b/server/src/__tests__/heartbeat-run-event-atomic-append.test.ts @@ -1,9 +1,11 @@ import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js"; import { randomBytes } from "node:crypto"; -import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { afterAll, beforeAll, describe, expect, it, vi } from "vitest"; import { eq, inArray, sql } from "drizzle-orm"; import { drizzle } from "drizzle-orm/postgres-js"; -import { agents, companies, createDb, heartbeatRunEvents, heartbeatRuns, issues, nativeRunnerCheckpoints } from "@paperclipai/db"; +import { agents, companies, createDb, heartbeatRunEvents, heartbeatRuns, issues, nativeRunnerCheckpoints, databaseWorkPending, + reconcileDatabaseWork, signalDatabaseWork, subscribeDatabaseWork } from "@paperclipai/db"; +import { DELIVERY_QUEUES } from "../services/delivery-work-notifications.js"; import { appendHeartbeatRunEvent, appendHeartbeatRunEventBatch } from "../services/heartbeat-run-events.js"; import { nativeSha256 } from "../services/native-runtime/canonical.js"; import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; @@ -293,6 +295,7 @@ describeEmbeddedPostgres("atomic ordinary run events", () => { locked(); await released; }); + void holder.catch(() => undefined); await lockedPromise; return { release: () => { release(); return holder; } }; }; @@ -485,17 +488,143 @@ describeEmbeddedPostgres("atomic ordinary run events", () => { const runId = await makeRun(); const [issue] = await db.insert(issues).values({ companyId, title: "Progress task" }).returning(); await db.update(heartbeatRuns).set({ nativeIssueId: issue!.id, runtimeMode: "native" }).where(eq(heartbeatRuns.id, runId)); - const first = await sentStatements(db.$client, () => appendHeartbeatRunEvent(db, nativeEvent(runId, 1, "item.completed"))); - // BEGIN, lock, duplicate probe, the wake's transaction id, the append, COMMIT. - expect(first.statements).toHaveLength(6); - const wake = first.statements.findIndex(text => text.includes("pg_current_xact_id")); - expect(wake).toBeGreaterThan(0); - expect(wake).toBeLessThan(first.statements.findIndex(text => text.includes("insert into heartbeat_run_events"))); - expect(first.result).toMatchObject({ disposition: "committed", highestContiguousSourceSeq: 1 }); - const replay = await sentStatements(db.$client, () => appendHeartbeatRunEvent(db, nativeEvent(runId, 1, "item.completed"))); - expect(replay.statements.some(text => text.includes("pg_current_xact_id"))).toBe(false); - expect(replay.result).toEqual({ row: first.result.row, disposition: "duplicate", highestContiguousSourceSeq: 1 }); - expect(await readEvents(runId)).toEqual([first.result.row]); + const wakes: string[] = []; + const unsubscribe = subscribeDatabaseWork(db, DELIVERY_QUEUES.chatNativeProgress, event => { wakes.push(event); }); + try { + const first = await sentStatements(db.$client, () => appendHeartbeatRunEvent(db, nativeEvent(runId, 1, "item.completed"))); + expect(first.statements).toHaveLength(4); + expect(first.statements[0]).toMatch(/^begin/i); + expect(first.statements[1]).toMatch(/^with "snapshot_run" as .*pg_current_xact_id\(\).* for no key update.*"replay_found"/); + expect(first.statements[2]).toMatch(/insert into heartbeat_run_events/); + expect(first.statements[3]).toMatch(/^commit/i); + expect(wakes).toEqual(["intent", "settled"]); + expect(first.result).toMatchObject({ disposition: "committed", highestContiguousSourceSeq: 1 }); + wakes.length = 0; + const replay = await sentStatements(db.$client, () => appendHeartbeatRunEvent(db, nativeEvent(runId, 1, "item.completed"))); + expect(wakes).toEqual([]); + expect(replay.statements).toHaveLength(4); + expect(replay.result).toEqual({ row: first.result.row, disposition: "duplicate", highestContiguousSourceSeq: 1 }); + expect(await readEvents(runId)).toEqual([first.result.row]); + } finally { unsubscribe(); } + }); + + it("gives the wake the transaction id of the transaction that took the run lock", async () => { + const runId = await makeRun(); + const [issue] = await db.insert(issues).values({ companyId, title: "Primed wake" }).returning(); + await db.update(heartbeatRuns).set({ nativeIssueId: issue!.id, runtimeMode: "native" }).where(eq(heartbeatRuns.id, runId)); + const topic = DELIVERY_QUEUES.chatNativeProgress; + const options = db.$client.options as { debug?: unknown }; + const sent: Array<{ text: string; parameters: unknown[] }> = []; + let ownXid: string | undefined; + options.debug = (_connection: number, text: string, parameters: unknown[]) => { + sent.push({ text: text.trim().replace(/\s+/g, " "), parameters }); + }; + try { + await expect(db.transaction(async tx => { + await appendHeartbeatRunEvent(tx as unknown as typeof db, nativeEvent(runId, 1, "item.completed")); + expect(sent.filter(statement => /^select pg_current_xact_id\(\)/.test(statement.text))).toEqual([]); + expect(sent.filter(statement => statement.text.includes("pg_current_xact_id()"))).toHaveLength(1); + const [own] = await tx.execute<{ xid: string; pid: number }>(sql`select pg_current_xact_id()::text as xid, pg_backend_pid() as pid`); + ownXid = own!.xid; + await otherDb.execute(sql`select pg_terminate_backend(${own!.pid})`); + await tx.execute(sql`select 1`); + })).rejects.toThrow(); + expect(databaseWorkPending(db, topic)).toBe(true); + sent.length = 0; + await reconcileDatabaseWork(db, topic); + } finally { options.debug = undefined; } + const statusRead = sent.find(statement => statement.text.includes("pg_xact_status")); + expect(ownXid).toMatch(/^\d+$/); + expect(statusRead?.parameters).toEqual([ownXid]); + expect(databaseWorkPending(db, topic)).toBe(false); + expect(await readEvents(runId)).toEqual([]); + }); + + it("primes nothing when the lock read fails, so the transaction is not left uncertain", async () => { + const runId = await makeRun(); + const [issue] = await db.insert(issues).values({ companyId, title: "Failed lock read" }).returning(); + await db.update(heartbeatRuns).set({ nativeIssueId: issue!.id, runtimeMode: "native" }).where(eq(heartbeatRuns.id, runId)); + let release!: () => void, locked!: () => void; + const released = new Promise(resolve => { release = resolve; }); + const lockedRun = new Promise(resolve => { locked = resolve; }); + const holder = otherDb.transaction(async tx => { + await tx.execute(sql`select id from heartbeat_runs where id = ${runId} for update`); + locked(); + await released; + }); + void holder.catch(() => undefined); + await lockedRun; + const wakes: string[] = []; + const unsubscribe = subscribeDatabaseWork(db, DELIVERY_QUEUES.chatNativeProgress, wake => { wakes.push(wake); }); + const options = db.$client.options as { debug?: unknown }; + const sent: string[] = []; + try { + await expect(db.transaction(async tx => { + await tx.execute(sql`set local lock_timeout = '100ms'`); + await appendHeartbeatRunEvent(tx as unknown as typeof db, nativeEvent(runId, 1, "item.completed")); + })).rejects.toThrow(); + expect(databaseWorkPending(db, DELIVERY_QUEUES.chatNativeProgress)).toBe(false); + await expect(db.transaction(async tx => { + await tx.execute(sql`set local lock_timeout = '100ms'`); + await expect(tx.transaction(savepoint => + appendHeartbeatRunEvent(savepoint as unknown as typeof db, nativeEvent(runId, 1, "item.completed")))).rejects.toThrow(); + options.debug = (_connection: number, text: string) => { sent.push(text.trim().replace(/\s+/g, " ")); }; + await signalDatabaseWork(tx, "lock-read-failed"); + options.debug = undefined; + throw new Error("roll back"); + })).rejects.toThrow("roll back"); + } finally { + options.debug = undefined; + unsubscribe(); + release(); + await holder; + } + expect(sent.filter(text => /^select pg_current_xact_id\(\)/.test(text))).toHaveLength(1); + await reconcileDatabaseWork(db, "lock-read-failed"); + expect(databaseWorkPending(db, "lock-read-failed")).toBe(false); + expect(wakes).toEqual([]); + expect(databaseWorkPending(db, DELIVERY_QUEUES.chatNativeProgress)).toBe(false); + expect(await readEvents(runId)).toEqual([]); + }); + + it("finds a replay that another writer committed while the lock read waited, and registers no wake for it", async () => { + const runId = await makeRun(); + const [issue] = await db.insert(issues).values({ companyId, title: "Progress race" }).returning(); + await db.update(heartbeatRuns).set({ nativeIssueId: issue!.id, runtimeMode: "native" }).where(eq(heartbeatRuns.id, runId)); + const event = nativeEvent(runId, 1, "item.completed"); + let release!: () => void, locked!: () => void; + const released = new Promise(resolve => { release = resolve; }); + const lockedRun = new Promise(resolve => { locked = resolve; }); + const holder = otherDb.transaction(async tx => { + await tx.execute(sql`select id from heartbeat_runs where id = ${runId} for update`); + await tx.insert(heartbeatRunEvents).values({ companyId, agentId, runId, seq: 1, eventType: event.eventType, stream: "system", + level: "info", payload: event.payload, sourceInstanceId: "native-writer", sourceEventId: event.nativeSource.sourceEventId, + sourceSeq: 1, sourcePayloadSha256: nativeSha256(event.nativeSource.canonicalPayload), protocolSchemaVersion: 1 }); + await tx.update(heartbeatRuns).set({ nextEventSeq: 2 }).where(eq(heartbeatRuns.id, runId)); + locked(); + await released; + }); + void holder.catch(() => undefined); + await lockedRun; + const wakes: string[] = []; + const unsubscribe = subscribeDatabaseWork(db, DELIVERY_QUEUES.chatNativeProgress, wake => { wakes.push(wake); }); + try { + const appended = sentStatements(db.$client, () => appendHeartbeatRunEvent(db, event)); + void appended.catch(() => undefined); + await vi.waitFor(async () => { + const [waiting] = await otherDb.execute<{ count: number }>(sql`select count(*)::int as count from pg_stat_activity + where datname = current_database() and wait_event_type = 'Lock' and query like '%"locked_run"%'`); + expect(waiting?.count).toBe(1); + }, { timeout: 10_000, interval: 20 }); + release(); + await holder; + const { result, statements } = await appended; + expect(result).toMatchObject({ disposition: "duplicate", row: { seq: 1, sourceSeq: 1 } }); + expect(statements.some(text => /^select \* from heartbeat_run_events/.test(text))).toBe(true); + expect(wakes).toEqual([]); + } finally { unsubscribe(); } + expect((await readEvents(runId)).map(row => row.seq)).toEqual([1]); + expect((await readRun(runId)).nextEventSeq).toBe(2); }); it("keeps one ordered row per source event when two servers append the same stream", async () => { @@ -601,11 +730,11 @@ describeEmbeddedPostgres("atomic ordinary run events", () => { expect(replay.highestContiguousSourceSeq).toBe(kind === "native" ? 1 : 0); expect((await readRun(runId)).nextEventSeq).toBe(2); expect(await readEvents(runId)).toEqual([committed.row]); - const lockReads = statements.filter(query => query.startsWith("select ") && - query.includes('from "heartbeat_runs"') && query.endsWith("for no key update")); + const lockReads = statements.filter(query => /^(select|with) /.test(query) && + query.includes('from "heartbeat_runs"') && query.includes("for no key update")); expect(lockReads).toHaveLength(2); for (const query of lockReads) { - expect(query).toMatch(/^select "company_id", "agent_id"(, \("native_source_cursors" ->> \$1\)::bigint as "source_cursor")? from "heartbeat_runs" /); + expect(query).toMatch(/select "company_id", "agent_id"(, \("native_source_cursors" ->> \$\d+\)::bigint as "source_cursor")?(, pg_current_xact_id\(\)::text as "xid")?(, "next_event_seq")? from "heartbeat_runs" /); expect(query).not.toMatch(/context_snapshot|runner_profile_json|result_json|stdout_excerpt/); } }); @@ -631,9 +760,11 @@ describeEmbeddedPostgres("atomic ordinary run events", () => { holderLocked(); await held; }); + void holder.catch(() => undefined); await locked; let secondSettled = false; const second = appendHeartbeatRunEvent(db, native(2)).finally(() => { secondSettled = true; }); + void second.catch(() => undefined); try { // A controller checkpoint row references the run row by foreign key. // Its insert must not wait for the open append. diff --git a/server/src/services/heartbeat-run-events.ts b/server/src/services/heartbeat-run-events.ts index 4aa41b431ff..30f41f5c78a 100644 --- a/server/src/services/heartbeat-run-events.ts +++ b/server/src/services/heartbeat-run-events.ts @@ -9,7 +9,7 @@ import { boundRunIssueIdSql } from "./run-task-binding.js"; import { and, asc, eq, getTableColumns, is, sql } from "drizzle-orm"; import { PgTransaction } from "drizzle-orm/pg-core"; import type { Db } from "@paperclipai/db"; -import { heartbeatRunEvents, heartbeatRuns } from "@paperclipai/db"; +import { heartbeatRunEvents, heartbeatRuns, primeDatabaseWorkTransactionId } from "@paperclipai/db"; import { nativeSha256 } from "./native-runtime/canonical.js"; import { isUniqueViolation } from "../db-errors.js"; @@ -345,9 +345,9 @@ function decodeRunEventRow(raw: Record): typeof heartbeatRunEve /** * A native source event: one lock read, then one statement that finds a * replayed event or reserves the sequence and inserts, and reads how much of - * the source is retained. The lock must stay a statement of its own. A - * statement that waits for the run lock keeps the snapshot it took before - * waiting, so it would not see an event the lock holder had just committed. + * the source is retained. A statement that waits for the run lock keeps the + * snapshot it took before waiting, so it would not see an event the lock + * holder had just committed. */ async function appendNativeSourceEvent( tx: Db, @@ -361,15 +361,26 @@ async function appendNativeSourceEvent( // checkpoints, that are written for the same run while events stream. // Only progress needs the task binding. Other native/recovery events // must not read a potentially large context snapshot to append a log. + const replayed = sql`run_id = ${input.runId} and (source_event_id = ${source.sourceEventId} + or (source_instance_id = ${source.sourceInstanceId} and source_seq = ${source.sourceSeq}))`; + const probes = wakes.reassessment || wakes.progress || wakes.response || wakes.retention; + const columns = sql`"company_id", "agent_id", ("native_source_cursors" ->> ${source.sourceInstanceId})::bigint as "source_cursor"${ + wakes.progress || wakes.response || wakes.reassessment ? sql`, "runtime_mode"` : sql.empty()}${ + wakes.progress ? sql`, ${boundRunIssueIdSql()} as "issue_id"` : sql.empty()}${ + Object.values(wakes).some(Boolean) ? sql`, pg_current_xact_id()::text as "xid"` : sql.empty()}`; const [run] = await tx.execute<{ company_id: string; agent_id: string; source_cursor: string | number | null; - runtime_mode?: string | null; issue_id?: string | null }>( - sql`select "company_id", "agent_id", ("native_source_cursors" ->> ${source.sourceInstanceId})::bigint as "source_cursor"${ - wakes.progress || wakes.response || wakes.reassessment ? sql`, "runtime_mode"` : sql.empty()}${ - wakes.progress ? sql`, ${boundRunIssueIdSql()} as "issue_id"` : sql.empty() - } from "heartbeat_runs" where "id" = ${input.runId} for no key update`); + runtime_mode?: string | null; issue_id?: string | null; xid?: string; probe_current?: boolean; replay_found?: boolean }>(probes + ? sql`with "snapshot_run" as (select "next_event_seq" from "heartbeat_runs" where "id" = ${input.runId}), + "locked_run" as (select ${columns}, "next_event_seq" from "heartbeat_runs" where "id" = ${input.runId} for no key update) + select "locked_run".*, "locked_run"."next_event_seq" = (select "next_event_seq" from "snapshot_run") as "probe_current", + exists (select 1 from heartbeat_run_events where ${replayed}) as "replay_found" + from "locked_run"` + : sql`select ${columns} from "heartbeat_runs" where "id" = ${input.runId} for no key update`); if (!run || run.company_id !== input.companyId || run.agent_id !== input.agentId) { throw new Error("heartbeat_run_event_binding_mismatch"); } + if (run.xid) primeDatabaseWorkTransactionId(tx, run.xid); + const knownNew = run.probe_current === true && run.replay_found === false; // The stored cursor says the run holds every event of this source up to it. // An event at or just past it moves the cursor without counting the run's // events; any other event, or a run without a cursor, recounts them. @@ -380,15 +391,14 @@ async function appendNativeSourceEvent( if (wakes.recovery) await notifyLiveRunRecovery(tx, input); const sourceHash = nativeSha256(source.canonicalPayload); - const replayed = sql`run_id = ${input.runId} and (source_event_id = ${source.sourceEventId} - or (source_instance_id = ${source.sourceInstanceId} and source_seq = ${source.sourceSeq}))`; const native = run.runtime_mode === "native"; const reassessment = wakes.reassessment && native, progress = wakes.progress && native && run.issue_id; const response = wakes.response && native; if (reassessment || progress || response || wakes.retention) { // A wake is registered before the write it announces, and a replayed // event registers none. So these events look for their replay first. - const [existing] = await tx.execute>(sql`select * from heartbeat_run_events where ${replayed} limit 1`); + const [existing] = knownNew ? [] + : await tx.execute>(sql`select * from heartbeat_run_events where ${replayed} limit 1`); if (existing) { const row = decodeRunEventRow(existing); if (row.sourcePayloadSha256 !== sourceHash) throw new HeartbeatRunEventConflictError(); diff --git a/server/src/services/native-runtime/native-runner-checkpoint-save.test.ts b/server/src/services/native-runtime/native-runner-checkpoint-save.test.ts new file mode 100644 index 00000000000..bd179a05724 --- /dev/null +++ b/server/src/services/native-runtime/native-runner-checkpoint-save.test.ts @@ -0,0 +1,82 @@ +import { randomBytes, randomUUID } from "node:crypto"; +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { eq } from "drizzle-orm"; +import { agents, checkpointDb, closeCheckpointDb, companies, createDb, heartbeatRuns, issues, nativeRunFinalizations, nativeRunnerCheckpoints, + subscribeDatabaseWork } from "@paperclipai/db"; +import { DELIVERY_QUEUES } from "../delivery-work-notifications.js"; +import { nativeRunnerCheckpointWriter } from "./native-runner-checkpoint.js"; +import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "../../__tests__/helpers/embedded-postgres.js"; + +const support = await getEmbeddedPostgresTestSupport(); + +(support.supported ? describe : describe.skip)("native runner checkpoint save", () => { + let temporary: Awaited>; + let db: ReturnType; + beforeAll(async () => { + process.env.PAPERCLIP_SECRETS_MASTER_KEY ??= randomBytes(32).toString("base64"); + temporary = await startEmbeddedPostgresTestDatabase("paperclip-checkpoint-save-"); + db = createDb(temporary.connectionString); + }, 60_000); + afterAll(async () => { + if (db) await closeCheckpointDb(db, 1); + await db?.$client.end({ timeout: 1 }); + await temporary?.cleanup(); + }); + + const fixture = async (leaseExpiresAt: Date) => { + const [company] = await db.insert(companies).values({ name: "Checkpoint save", issuePrefix: `CS${randomUUID().slice(0, 4)}` }).returning(); + const [agent] = await db.insert(agents).values({ companyId: company!.id, name: "Checkpoint agent", adapterType: "process" }).returning(); + const [issue] = await db.insert(issues).values({ companyId: company!.id, title: "Checkpoint task" }).returning(); + const runId = randomUUID(), leaseOwner = randomUUID(), bootId = randomUUID(); + await db.insert(heartbeatRuns).values({ id: runId, companyId: company!.id, agentId: agent!.id, status: "running", + runtimeMode: "native", nativeIssueId: issue!.id, issueId: issue!.id }); + await db.insert(nativeRunFinalizations).values({ runId, companyId: company!.id, issueId: issue!.id, phase: "observed", attempt: 0, + controllerBootId: bootId, controllerPid: process.pid, controllerProcessStartedAt: new Date(), controllerGeneration: 1, + leaseOwner, leaseExpiresAt }); + const write = nativeRunnerCheckpointWriter(db, { companyId: company!.id, issueId: issue!.id, runId, authorityRunId: runId, + bootId, generation: 1, leaseOwner }); + const checkpoint = (changed: Map, reset: boolean) => ({ + runId, records: new Map(changed), changedRecords: changed, removedRecords: [], reset, workCommandIds: [], predecessor: null, + runnerVersion: "test", runnerDigest: `sha256:${"a".repeat(64)}`, + recoveryEvidence: { runnerInstanceId: "test", normalizedSessionId: "test", authenticatedConnectionCount: 1, highestRecordedSourceSeq: 0 }, + restoreSessionDirectory() {}, + }) as unknown as Parameters[0]; + return { runId, write, checkpoint }; + }; + + const sentStatements = async (action: () => Promise) => { + const statements: string[] = []; + const options = checkpointDb(db).$client.options as { debug?: unknown }; + options.debug = (_connection: number, text: string) => { statements.push(text.trim().replace(/\s+/g, " ")); }; + try { await action(); return statements; } finally { options.debug = undefined; } + }; + + it("saves a checkpoint with the owner lock, the write and the commit, and registers its wake before the write", async () => { + const f = await fixture(new Date(Date.now() + 3_600_000)); + await f.write(f.checkpoint(new Map([["header", "{}"], ["cursor", "{\"ackedSourceSeq\":0}"]]), true)); + const events: string[] = []; + const unsubscribe = subscribeDatabaseWork(db, DELIVERY_QUEUES.nativeProcessSessions, event => { events.push(event); }); + let statements: string[]; + try { + statements = await sentStatements(() => f.write(f.checkpoint(new Map([["event:1", "{}"], ["cursor", "{\"ackedSourceSeq\":1}"]]), false))); + } finally { unsubscribe(); } + expect(statements).toHaveLength(4); + expect(statements[0]).toMatch(/^begin/i); + expect(statements[1]).toMatch(/pg_current_xact_id\(\)/); + expect(statements[2]).toMatch(/^insert into "native_runner_checkpoints"/); + expect(statements[3]).toMatch(/^commit/i); + expect(events).toEqual(["intent", "settled"]); + expect((await db.select({ part: nativeRunnerCheckpoints.part }).from(nativeRunnerCheckpoints) + .where(eq(nativeRunnerCheckpoints.runId, f.runId))).map(row => row.part).sort()).toEqual(["cursor", "event:1", "header"]); + }); + + it("refuses a save after the owner's lease expired and writes nothing", async () => { + const f = await fixture(new Date(Date.now() + 3_600_000)); + await f.write(f.checkpoint(new Map([["header", "{}"]]), true)); + await db.update(nativeRunFinalizations).set({ leaseExpiresAt: new Date(Date.now() - 1_000) }) + .where(eq(nativeRunFinalizations.runId, f.runId)); + await expect(f.write(f.checkpoint(new Map([["event:1", "{}"]]), false))).rejects.toThrow("native_runner_checkpoint_owner_lost"); + expect((await db.select({ part: nativeRunnerCheckpoints.part }).from(nativeRunnerCheckpoints) + .where(eq(nativeRunnerCheckpoints.runId, f.runId))).map(row => row.part)).toEqual(["header"]); + }); +}); diff --git a/server/src/services/native-runtime/native-runner-checkpoint.ts b/server/src/services/native-runtime/native-runner-checkpoint.ts index d4a9b2be91e..8582d339ed7 100644 --- a/server/src/services/native-runtime/native-runner-checkpoint.ts +++ b/server/src/services/native-runtime/native-runner-checkpoint.ts @@ -3,7 +3,7 @@ import { NATIVE_PREPARATION_FENCING_VERSION } from "./native-active-runs.js"; import { notifyLiveRunRecovery } from "../live-run-recovery-notifications.js"; import { createHash } from "node:crypto"; import { and, eq, exists, gt, isNull, lte, inArray, notExists, or, sql, type SQL } from "drizzle-orm"; -import { checkpointDb, heartbeatRuns, issues, nativeRunFinalizations, nativeRunnerCheckpoints, type Db } from "@paperclipai/db"; +import { checkpointDb, heartbeatRuns, issues, nativeRunFinalizations, nativeRunnerCheckpoints, primeDatabaseWorkTransactionId, type Db } from "@paperclipai/db"; import { inspectDurablePrpCheckpointHeader, inspectUnauthenticatedBootstrapCheckpoint, inspectUnclaimedBootstrapCheckpoint, decodeDurablePrpCheckpoint, type DurablePrpCheckpoint, type DurablePrpControlPlane, type DurableRecoveryIdentity } from "../../vendor/paperclip-runner/index.js"; import { createLocalEncryptedCipher, deriveLocalEncryptedInstanceSecret } from "../../secrets/local-encrypted-provider.js"; import { legacyControllerBootId } from "../legacy-controller-lease.js"; @@ -76,9 +76,12 @@ export async function lockNativeRunnerCheckpointOwner( // wall time. Sampling in the locking SELECT can precede its lock wait. // Keep both steps in one statement without another database round trip. const locked = nativeRunnerCheckpointOwnerLock(tx, owner); - const [owned] = await tx.with(locked).select({ leaseOwner: locked.leaseOwner, - leaseExpiresAt: locked.leaseExpiresAt, resultId: locked.resultId, phase: locked.phase, now: sql`clock_timestamp()` }).from(locked); - if (!owned) throw new Error("native_runner_checkpoint_owner_lost"); + const [row] = await tx.with(locked).select({ leaseOwner: locked.leaseOwner, + leaseExpiresAt: locked.leaseExpiresAt, resultId: locked.resultId, phase: locked.phase, now: sql`clock_timestamp()`, + xid: sql`pg_current_xact_id()::text` }).from(locked); + if (!row) throw new Error("native_runner_checkpoint_owner_lost"); + const { xid, ...owned } = row; + primeDatabaseWorkTransactionId(tx, xid); return owned; }