From 1949e726f8515127a04314e3793c427ccfc33c1e Mon Sep 17 00:00:00 2001 From: Devin Foley Date: Sun, 25 Oct 2026 11:46:29 -0700 Subject: [PATCH] perf(recovery): read only legacy terminal control fields under lock (#17646) Select only runtime mode, status, complete result evidence and the delivery token from the locked legacy terminal-state read. Keep filters, row lock, guards, writes, full returned rows, source retention and recovery holds. The synthetic regression reduces a 699,454-byte read to under 4 KB while preserving full stored and returned data. All CI checks, 236 combined tests, workspace typecheck and server build pass. Apex 5/5 after correcting a documentation contradiction, with no unresolved comments. Co-Authored-By: Paperclip --- doc/DATABASE.md | 8 ++++- .../legacy-workspace-restore-recovery.test.ts | 29 ++++++++++++++++++- .../src/services/legacy-execution-recovery.ts | 4 ++- 3 files changed, 38 insertions(+), 3 deletions(-) diff --git a/doc/DATABASE.md b/doc/DATABASE.md index 961d532aef9..c7b525877fc 100644 --- a/doc/DATABASE.md +++ b/doc/DATABASE.md @@ -70,7 +70,8 @@ The two preliminary reconciliation reads omit named launch renderings and preparation payloads. Native profiles omit unused native execution input. They retain all other context, full results, legacy profiles, task identity, retry accounting, and unknown controls. Native runs still reach their native -recovery guard. Terminalization keeps its full locked source and stored input. +recovery guard. Terminalization locks its current control state and returns +full rows from its writes. Stored input remains intact. Execution-blocker reads select process, lifecycle, and task-binding fields rather than full execution rows. Context retains typed task and pre-dispatch evidence; @@ -96,6 +97,11 @@ payloads. It retains current coordinator checks, provider evidence, local-host identity checks, and every lease's termination receipt. The stop receipt and process-field update remain in the same transaction. +Legacy terminalization locks runtime mode, status, full result evidence, and the +status-delivery token without loading unused input, profile, or log bodies. +It keeps the existing lock, writes, full returned run, workspace-retention +receipts, and source-bound recovery holds. Stored payloads remain unchanged. + Conversation ownership checks select process identity and full recovery result evidence without execution prompts or profiles. Committed native reply repair selects its persisted result and identity bindings plus typed presentation flags. diff --git a/server/src/__tests__/legacy-workspace-restore-recovery.test.ts b/server/src/__tests__/legacy-workspace-restore-recovery.test.ts index 4629092e10e..84da47b84b7 100644 --- a/server/src/__tests__/legacy-workspace-restore-recovery.test.ts +++ b/server/src/__tests__/legacy-workspace-restore-recovery.test.ts @@ -1,4 +1,5 @@ -import { randomUUID } from "node:crypto"; +import { randomBytes, randomUUID } from "node:crypto"; +import { createDatabaseResponseCapture, installDatabaseResponseCapture } from "@paperclipai/db/response-capture"; import { and, eq, sql } from "drizzle-orm"; import { afterAll, beforeAll, describe, expect, it, vi } from "vitest"; import { agents, companies, createDb, environmentLeases, environments, executionWorkspaces, heartbeatRuns, issueRecoveryActions, issues, projects } from "@paperclipai/db"; @@ -46,6 +47,32 @@ const support = externalTestDatabaseUrl ? { supported: true } : await getEmbedde } }; const readLease = async (id: string) => (await db.select().from(environmentLeases).where(eq(environmentLeases.id, id)))[0]; + it("locks only terminal control state while returning and retaining the complete run", async () => { + const f = await seed(); + const bulk = randomBytes(128 * 1024).toString("base64"); + const [saved] = await db.update(heartbeatRuns).set({ contextSnapshot: { ...f.run.contextSnapshot, savedInput: bulk }, + runnerProfileJson: { savedProfile: bulk }, stdoutExcerpt: bulk, stderrExcerpt: bulk, + }).where(eq(heartbeatRuns.id, f.runId)).returning(); + const events: Array> = [], now = Date.now(); + const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event)); + installDatabaseResponseCapture(db, { heartbeatRuns }, capture); + let returned: Awaited>; + try { returned = await terminalizeLegacyExecution({ db, run: saved!, status: "failed", patch }); } + finally { capture.flush(); } + const groups = events.filter(event => event.event === "paperclip.db_response_capture.entry.v1"); + const reads = groups.filter(group => group.table === "heartbeat_runs" && group.operation === "select"); + expect(reads.reduce((sum, group) => sum + Number(group.calls), 0)).toBe(1); + expect(reads.reduce((sum, group) => sum + Number(group.responseValueBytes), 0)).toBeLessThan(4096); + expect(returned).toMatchObject({ id: f.runId, status: "failed", contextSnapshot: saved!.contextSnapshot, + runnerProfileJson: saved!.runnerProfileJson, stdoutExcerpt: bulk, stderrExcerpt: bulk }); + expect(hasRequiredWorkspaceRecovery(returned!.resultJson)).toBe(true); + expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.runId)))[0]).toEqual(returned); + expect(await readLease(f.lease.id)).toMatchObject({ status: "pending_cleanup", leasePolicy: "retain_on_failure" }); + expect(await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.sourceIssueId, f.issueId))) + .toEqual([expect.objectContaining({ evidence: expect.objectContaining({ runId: f.runId }), ownerType: "board" })]); + expect(JSON.stringify(events)).not.toContain(bulk.slice(0, 64)); + }); + it("preserves pending and acknowledged status delivery on unpatched legacy recovery retries", async () => { await deliverExecutionStatuses(db, { publish: vi.fn() }); const f = await seed(); diff --git a/server/src/services/legacy-execution-recovery.ts b/server/src/services/legacy-execution-recovery.ts index 3561bff8768..5d5fe783b3f 100644 --- a/server/src/services/legacy-execution-recovery.ts +++ b/server/src/services/legacy-execution-recovery.ts @@ -118,7 +118,9 @@ export async function terminalizeLegacyExecution(input: { ) .for("update") : []; - const [current] = await tx.select().from(heartbeatRuns).where(and( + const [current] = await tx.select({ runtimeMode: heartbeatRuns.runtimeMode, status: heartbeatRuns.status, + resultJson: heartbeatRuns.resultJson, executionStatusDeliveryId: heartbeatRuns.executionStatusDeliveryId, + }).from(heartbeatRuns).where(and( eq(heartbeatRuns.id, run.id), eq(heartbeatRuns.companyId, run.companyId), )).for("update"); if (!current || (current.runtimeMode !== "legacy" && !(input.recordRestoreFailureOnly && current.runtimeMode === "native")) ||