Drive remaining live run and resource recovery from durable work (#18419)

Reviewed and verified at 6a4c193906.

Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
DottaandPaperclip authored and GitHub committed 2026-10-31 16:00:35 -05:00
1 parent eb12cfc96b
commit 9ca9af544a
62 files changed
+1822 -237

No files matched your search

@@ -41,6 +41,32 @@ export function nativeRetryCancellationEligible(input: {
&& !!native.acknowledgementAuditId && native.acknowledgementAuditId !== native.intentAuditId));
}
export function pendingFailedNativeRetryCancellationCondition() {
const native = sql`${heartbeatRuns.resultJson}->'nativeCancellation'`;
const continuingStop = sql`${nativeRunFinalizations.phase} = 'terminal_failure'
and ${nativeRunFinalizations.failureCode} = 'native_retry_cancelled'
and ${native}->>'schema' = 'paperclip.native-cancellation.v1'
and jsonb_typeof(${native}->'intentId') = 'string' and length(${native}->>'intentId') > 0
and ${native}->>'runId' = ${heartbeatRuns.id}::text
and ${native}->>'companyId' = ${heartbeatRuns.companyId}::text
and ${native}->>'issueId' = ${heartbeatRuns.nativeIssueId}::text
and ${native}->>'scope' = 'run'
and ${native}->>'dispatchState' in ('pending', 'acknowledged')
and jsonb_typeof(${native}->'dispatched') = 'boolean'
and jsonb_typeof(${native}->'intentAuditId') = 'string' and length(${native}->>'intentAuditId') > 0
and (${native}->>'dispatchState' = 'pending' or (
jsonb_typeof(${native}->'acknowledgementAuditId') = 'string'
and length(${native}->>'acknowledgementAuditId') > 0
and ${native}->>'acknowledgementAuditId' <> ${native}->>'intentAuditId'))`;
return sql`${heartbeatRuns.runtimeMode} = 'native' and ${heartbeatRuns.status} = 'failed'
and exists (select 1 from ${nativeRunFinalizations}
where ${nativeRunFinalizations.runId} = ${heartbeatRuns.id}
and ${nativeRunFinalizations.companyId} = ${heartbeatRuns.companyId}
and ${nativeRunFinalizations.issueId} = ${heartbeatRuns.nativeIssueId}
and (${nativeRunFinalizations.phase} = 'retryable_failure' or (${continuingStop}))
and ${nativeRunFinalizations.resultId} is null)`;
}
/** Final failed -> cancelled CAS: a later board/recovery outcome must win.
* The row-locking subquery prevents a coordinator-only writer from changing
* the retry disposition between this check and the status update. */
@@ -1,3 +1,4 @@
import { notifyLiveRunRecovery } from "../live-run-recovery-notifications.js";
import type { Db } from "@paperclipai/db";
import { heartbeatRuns, livenessDb, nativeRunFinalizations } from "@paperclipai/db";
import { and, eq, gt, sql } from "drizzle-orm";
@@ -74,6 +75,7 @@ async function renewNativeSessionExecutionLeaseDeadline(
// but conservatively include UPDATE/commit response time without assuming
// that this server's wall clock agrees with the database's clock.
confirmedDeadline = performance.now() + leaseTtlMs;
await notifyLiveRunRecovery(tx, { companyId: input.companyId, runId: input.runId });
return tx.update(nativeRunFinalizations)
.set({
leaseExpiresAt: sql`clock_timestamp() + (${leaseTtlMs} * interval '1 millisecond')`,
@@ -99,7 +99,7 @@ export async function preserveNativeFailedWorkspace(db: Db, selected: Pick<typeo
}
export async function preserveOrphanedNativeWorkspace(db: Db, lease: Pick<typeof environmentLeases.$inferSelect,
"companyId" | "heartbeatRunId">): Promise<boolean> {
"companyId" | "heartbeatRunId">, options: { onDeferredDeadline?: (at: number) => void } = {}): Promise<boolean> {
if (!lease.heartbeatRunId) return false;
const [run] = await db.select().from(heartbeatRuns).where(and(
eq(heartbeatRuns.companyId, lease.companyId), eq(heartbeatRuns.id, lease.heartbeatRunId),
@@ -122,6 +122,7 @@ export async function preserveOrphanedNativeWorkspace(db: Db, lease: Pick<typeof
if (error instanceof NativeWorkspaceFinalizationBusyError) return false;
const code = retentionContentionCode(error);
if (!code) throw error;
options.onDeferredDeadline?.(Date.now() + 30_000);
logger.warn({ runId: run.id, code }, "Native workspace retention is waiting for its current owner");
return false;
}
@@ -1,3 +1,4 @@
import { notifyLiveRunRecovery } from "../live-run-recovery-notifications.js";
import { notifyStatusCardIssueChange } from "../status-card-notifications.js";
import { notifyExternalAgentSettlement } from "../../modules/external-agents/index.js";
import { pendingNativeRunProjectionCondition } from "./native-run-projection.js";
@@ -80,6 +81,8 @@ export async function dispatchNativeSessionResumptions(input: {
now?: Date;
limit?: number;
dispatch: (claim: NativeSessionResumeClaim) => void;
deferIneligible?: boolean;
onError?: (error: unknown) => void;
}) {
const claims = await claimNativeSessionResumptions(input);
for (const claim of claims) input.dispatch(claim);
@@ -98,6 +101,8 @@ export async function claimNativeSessionResumptions(input: {
runIds?: string[];
now?: Date;
limit?: number;
deferIneligible?: boolean;
onError?: (error: unknown) => void;
}): Promise<NativeSessionResumeClaim[]> {
const now = input.now ?? new Date();
const candidates = await input.db
@@ -136,6 +141,7 @@ export async function claimNativeSessionResumptions(input: {
for (const candidate of candidates) {
const leaseOwner = `${input.runnerInstanceId}:resume:${randomUUID()}`;
let terminalRunToEmit: typeof heartbeatRuns.$inferSelect | null = null;
let claimFailed = false;
const claimed = await input.db.transaction(async (tx) => {
await tx.execute(sql`select set_config('statement_timeout', '10000', true), set_config('lock_timeout', '2000', true)`);
const [issue] = await tx.select().from(issues).where(and(
@@ -245,6 +251,7 @@ export async function claimNativeSessionResumptions(input: {
) {
const failureCode = originalFailureCode;
await notifyDeliveryWork(tx, DELIVERY_QUEUES.accounting, row.run.companyId);
await notifyLiveRunRecovery(tx, { companyId: row.run.companyId, runId: row.run.id, agentId: row.run.agentId });
await tx.update(nativeRunFinalizations).set({
phase: "terminal_failure",
leaseOwner: null,
@@ -313,6 +320,7 @@ export async function claimNativeSessionResumptions(input: {
}
const leaseExpiresAt = new Date(now.getTime() + 20 * 60_000);
await notifyLiveRunRecovery(tx, { companyId: row.run.companyId, runId: row.run.id, agentId: row.run.agentId });
const updated = await tx.update(nativeRunFinalizations).set({
phase: "observed",
leaseOwner,
@@ -350,6 +358,8 @@ export async function claimNativeSessionResumptions(input: {
});
return true;
}).catch((error: unknown) => {
claimFailed = true;
input.onError?.(error);
terminalRunToEmit = null;
logger.warn({ err: error, companyId: candidate.companyId, runId: candidate.runId },
"failed to claim native session recovery candidate");
@@ -362,8 +372,9 @@ export async function claimNativeSessionResumptions(input: {
void reportRunFailure(input.db, terminalRunToEmit);
}
if (claimed) claims.push({ runId: candidate.runId, leaseOwner });
else await input.db.transaction(async tx => {
else if (claimFailed || input.deferIneligible !== false) await input.db.transaction(async tx => {
await tx.execute(sql`select set_config('statement_timeout', '10000', true), set_config('lock_timeout', '2000', true)`);
await notifyLiveRunRecovery(tx, { companyId: candidate.companyId, runId: candidate.runId });
await tx.update(nativeRunFinalizations).set({
nextAttemptAt: new Date(now.getTime() + 30_000),
updatedAt: now,
@@ -377,6 +388,7 @@ export async function claimNativeSessionResumptions(input: {
or(isNull(nativeRunFinalizations.leaseOwner), isNull(nativeRunFinalizations.leaseExpiresAt), lte(nativeRunFinalizations.leaseExpiresAt, now)),
));
}).catch((error: unknown) => {
input.onError?.(error);
logger.warn({ err: error, companyId: candidate.companyId, runId: candidate.runId },
"failed to defer native session recovery candidate");
});
@@ -1,3 +1,4 @@
import { notifyLiveRunRecovery } from "../live-run-recovery-notifications.js";
import { and, eq, inArray, sql } from "drizzle-orm";
import type { Db } from "@paperclipai/db";
@@ -348,6 +349,7 @@ export async function requestNativeQuestionRunCancellation(
...cause,
requestedAt: new Date().toISOString(),
});
await notifyLiveRunRecovery(db, { companyId: interaction.companyId, runId: run.id });
return db.update(heartbeatRuns).set({
contextSnapshot: sql`jsonb_set(
case
@@ -1,3 +1,4 @@
import { notifyLiveRunRecovery } from "../live-run-recovery-notifications.js";
import { NATIVE_RETAINED_OWNER_WAIT } from "./native-controller-recovery.js";
import { notifyNativeEnvironmentSettlement } from "../native-environment-settlement-notifications.js";
import { remoteWorkspaceRunReferenceSchema } from "@paperclipai/shared";
@@ -664,6 +665,7 @@ export async function claimNativeRestartRecoveries(input: {
hasProviderEvidence: false,
stderrTail: row.run.stderrExcerpt,
});
await notifyLiveRunRecovery(tx, { companyId: row.run.companyId, runId: row.run.id, agentId: row.run.agentId });
await tx
.update(nativeRunFinalizations)
.set({
@@ -1048,6 +1050,7 @@ export async function claimNativeRestartRecoveries(input: {
recycledProviderPids: providerProcesses.recycledPids,
stderrTail: row.run.stderrExcerpt,
});
await notifyLiveRunRecovery(tx, { companyId: row.run.companyId, runId: row.run.id, agentId: row.run.agentId });
const claimed = await tx
.update(nativeRunFinalizations)
.set({
@@ -1,3 +1,4 @@
import { notifyLiveRunRecovery } from "../live-run-recovery-notifications.js";
import { and, eq, exists, gt, isNull, lte, inArray, notExists, or, sql } from "drizzle-orm";
import { checkpointDb, heartbeatRuns, issues, nativeRunFinalizations, nativeRunnerCheckpoints, type Db } from "@paperclipai/db";
import { inspectDurablePrpCheckpointHeader, decodeDurablePrpCheckpoint, type DurablePrpCheckpoint, type DurablePrpControlPlane } from "../../vendor/paperclip-runner/index.js";
@@ -105,6 +106,7 @@ export function nativeRunnerCheckpointWriter(db: Db, owner: NativeRunnerCheckpoi
eq(heartbeatRuns.nativeSessionId, checkpoint.predecessor.normalizedSessionId)));
if (!prior) throw new Error("native_runner_checkpoint_binding_mismatch");
}
await notifyLiveRunRecovery(tx, { companyId: owner.companyId, runId: owner.runId });
if (checkpoint.reset) await tx.delete(nativeRunnerCheckpoints).where(and(
eq(nativeRunnerCheckpoints.companyId, owner.companyId), eq(nativeRunnerCheckpoints.runId, owner.runId)));
else if (checkpoint.removedRecords.length) await tx.delete(nativeRunnerCheckpoints).where(and(