mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-11-16 03:31:44 +01:00
Drive remaining live run and resource recovery from durable work (#18419)
Reviewed and verified at 6a4c193906.
Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
1 parent
eb12cfc96b
commit
9ca9af544a
62 files changed
+1822
-237
No files matched your search
@@ -41,6 +41,32 @@ export function nativeRetryCancellationEligible(input: {
|
||||
&& !!native.acknowledgementAuditId && native.acknowledgementAuditId !== native.intentAuditId));
|
||||
}
|
||||
|
||||
export function pendingFailedNativeRetryCancellationCondition() {
|
||||
const native = sql`${heartbeatRuns.resultJson}->'nativeCancellation'`;
|
||||
const continuingStop = sql`${nativeRunFinalizations.phase} = 'terminal_failure'
|
||||
and ${nativeRunFinalizations.failureCode} = 'native_retry_cancelled'
|
||||
and ${native}->>'schema' = 'paperclip.native-cancellation.v1'
|
||||
and jsonb_typeof(${native}->'intentId') = 'string' and length(${native}->>'intentId') > 0
|
||||
and ${native}->>'runId' = ${heartbeatRuns.id}::text
|
||||
and ${native}->>'companyId' = ${heartbeatRuns.companyId}::text
|
||||
and ${native}->>'issueId' = ${heartbeatRuns.nativeIssueId}::text
|
||||
and ${native}->>'scope' = 'run'
|
||||
and ${native}->>'dispatchState' in ('pending', 'acknowledged')
|
||||
and jsonb_typeof(${native}->'dispatched') = 'boolean'
|
||||
and jsonb_typeof(${native}->'intentAuditId') = 'string' and length(${native}->>'intentAuditId') > 0
|
||||
and (${native}->>'dispatchState' = 'pending' or (
|
||||
jsonb_typeof(${native}->'acknowledgementAuditId') = 'string'
|
||||
and length(${native}->>'acknowledgementAuditId') > 0
|
||||
and ${native}->>'acknowledgementAuditId' <> ${native}->>'intentAuditId'))`;
|
||||
return sql`${heartbeatRuns.runtimeMode} = 'native' and ${heartbeatRuns.status} = 'failed'
|
||||
and exists (select 1 from ${nativeRunFinalizations}
|
||||
where ${nativeRunFinalizations.runId} = ${heartbeatRuns.id}
|
||||
and ${nativeRunFinalizations.companyId} = ${heartbeatRuns.companyId}
|
||||
and ${nativeRunFinalizations.issueId} = ${heartbeatRuns.nativeIssueId}
|
||||
and (${nativeRunFinalizations.phase} = 'retryable_failure' or (${continuingStop}))
|
||||
and ${nativeRunFinalizations.resultId} is null)`;
|
||||
}
|
||||
|
||||
/** Final failed -> cancelled CAS: a later board/recovery outcome must win.
|
||||
* The row-locking subquery prevents a coordinator-only writer from changing
|
||||
* the retry disposition between this check and the status update. */
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { notifyLiveRunRecovery } from "../live-run-recovery-notifications.js";
|
||||
import type { Db } from "@paperclipai/db";
|
||||
import { heartbeatRuns, livenessDb, nativeRunFinalizations } from "@paperclipai/db";
|
||||
import { and, eq, gt, sql } from "drizzle-orm";
|
||||
@@ -74,6 +75,7 @@ async function renewNativeSessionExecutionLeaseDeadline(
|
||||
// but conservatively include UPDATE/commit response time without assuming
|
||||
// that this server's wall clock agrees with the database's clock.
|
||||
confirmedDeadline = performance.now() + leaseTtlMs;
|
||||
await notifyLiveRunRecovery(tx, { companyId: input.companyId, runId: input.runId });
|
||||
return tx.update(nativeRunFinalizations)
|
||||
.set({
|
||||
leaseExpiresAt: sql`clock_timestamp() + (${leaseTtlMs} * interval '1 millisecond')`,
|
||||
|
||||
@@ -99,7 +99,7 @@ export async function preserveNativeFailedWorkspace(db: Db, selected: Pick<typeo
|
||||
}
|
||||
|
||||
export async function preserveOrphanedNativeWorkspace(db: Db, lease: Pick<typeof environmentLeases.$inferSelect,
|
||||
"companyId" | "heartbeatRunId">): Promise<boolean> {
|
||||
"companyId" | "heartbeatRunId">, options: { onDeferredDeadline?: (at: number) => void } = {}): Promise<boolean> {
|
||||
if (!lease.heartbeatRunId) return false;
|
||||
const [run] = await db.select().from(heartbeatRuns).where(and(
|
||||
eq(heartbeatRuns.companyId, lease.companyId), eq(heartbeatRuns.id, lease.heartbeatRunId),
|
||||
@@ -122,6 +122,7 @@ export async function preserveOrphanedNativeWorkspace(db: Db, lease: Pick<typeof
|
||||
if (error instanceof NativeWorkspaceFinalizationBusyError) return false;
|
||||
const code = retentionContentionCode(error);
|
||||
if (!code) throw error;
|
||||
options.onDeferredDeadline?.(Date.now() + 30_000);
|
||||
logger.warn({ runId: run.id, code }, "Native workspace retention is waiting for its current owner");
|
||||
return false;
|
||||
}
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { notifyLiveRunRecovery } from "../live-run-recovery-notifications.js";
|
||||
import { notifyStatusCardIssueChange } from "../status-card-notifications.js";
|
||||
import { notifyExternalAgentSettlement } from "../../modules/external-agents/index.js";
|
||||
import { pendingNativeRunProjectionCondition } from "./native-run-projection.js";
|
||||
@@ -80,6 +81,8 @@ export async function dispatchNativeSessionResumptions(input: {
|
||||
now?: Date;
|
||||
limit?: number;
|
||||
dispatch: (claim: NativeSessionResumeClaim) => void;
|
||||
deferIneligible?: boolean;
|
||||
onError?: (error: unknown) => void;
|
||||
}) {
|
||||
const claims = await claimNativeSessionResumptions(input);
|
||||
for (const claim of claims) input.dispatch(claim);
|
||||
@@ -98,6 +101,8 @@ export async function claimNativeSessionResumptions(input: {
|
||||
runIds?: string[];
|
||||
now?: Date;
|
||||
limit?: number;
|
||||
deferIneligible?: boolean;
|
||||
onError?: (error: unknown) => void;
|
||||
}): Promise<NativeSessionResumeClaim[]> {
|
||||
const now = input.now ?? new Date();
|
||||
const candidates = await input.db
|
||||
@@ -136,6 +141,7 @@ export async function claimNativeSessionResumptions(input: {
|
||||
for (const candidate of candidates) {
|
||||
const leaseOwner = `${input.runnerInstanceId}:resume:${randomUUID()}`;
|
||||
let terminalRunToEmit: typeof heartbeatRuns.$inferSelect | null = null;
|
||||
let claimFailed = false;
|
||||
const claimed = await input.db.transaction(async (tx) => {
|
||||
await tx.execute(sql`select set_config('statement_timeout', '10000', true), set_config('lock_timeout', '2000', true)`);
|
||||
const [issue] = await tx.select().from(issues).where(and(
|
||||
@@ -245,6 +251,7 @@ export async function claimNativeSessionResumptions(input: {
|
||||
) {
|
||||
const failureCode = originalFailureCode;
|
||||
await notifyDeliveryWork(tx, DELIVERY_QUEUES.accounting, row.run.companyId);
|
||||
await notifyLiveRunRecovery(tx, { companyId: row.run.companyId, runId: row.run.id, agentId: row.run.agentId });
|
||||
await tx.update(nativeRunFinalizations).set({
|
||||
phase: "terminal_failure",
|
||||
leaseOwner: null,
|
||||
@@ -313,6 +320,7 @@ export async function claimNativeSessionResumptions(input: {
|
||||
}
|
||||
|
||||
const leaseExpiresAt = new Date(now.getTime() + 20 * 60_000);
|
||||
await notifyLiveRunRecovery(tx, { companyId: row.run.companyId, runId: row.run.id, agentId: row.run.agentId });
|
||||
const updated = await tx.update(nativeRunFinalizations).set({
|
||||
phase: "observed",
|
||||
leaseOwner,
|
||||
@@ -350,6 +358,8 @@ export async function claimNativeSessionResumptions(input: {
|
||||
});
|
||||
return true;
|
||||
}).catch((error: unknown) => {
|
||||
claimFailed = true;
|
||||
input.onError?.(error);
|
||||
terminalRunToEmit = null;
|
||||
logger.warn({ err: error, companyId: candidate.companyId, runId: candidate.runId },
|
||||
"failed to claim native session recovery candidate");
|
||||
@@ -362,8 +372,9 @@ export async function claimNativeSessionResumptions(input: {
|
||||
void reportRunFailure(input.db, terminalRunToEmit);
|
||||
}
|
||||
if (claimed) claims.push({ runId: candidate.runId, leaseOwner });
|
||||
else await input.db.transaction(async tx => {
|
||||
else if (claimFailed || input.deferIneligible !== false) await input.db.transaction(async tx => {
|
||||
await tx.execute(sql`select set_config('statement_timeout', '10000', true), set_config('lock_timeout', '2000', true)`);
|
||||
await notifyLiveRunRecovery(tx, { companyId: candidate.companyId, runId: candidate.runId });
|
||||
await tx.update(nativeRunFinalizations).set({
|
||||
nextAttemptAt: new Date(now.getTime() + 30_000),
|
||||
updatedAt: now,
|
||||
@@ -377,6 +388,7 @@ export async function claimNativeSessionResumptions(input: {
|
||||
or(isNull(nativeRunFinalizations.leaseOwner), isNull(nativeRunFinalizations.leaseExpiresAt), lte(nativeRunFinalizations.leaseExpiresAt, now)),
|
||||
));
|
||||
}).catch((error: unknown) => {
|
||||
input.onError?.(error);
|
||||
logger.warn({ err: error, companyId: candidate.companyId, runId: candidate.runId },
|
||||
"failed to defer native session recovery candidate");
|
||||
});
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { notifyLiveRunRecovery } from "../live-run-recovery-notifications.js";
|
||||
import { and, eq, inArray, sql } from "drizzle-orm";
|
||||
|
||||
import type { Db } from "@paperclipai/db";
|
||||
@@ -348,6 +349,7 @@ export async function requestNativeQuestionRunCancellation(
|
||||
...cause,
|
||||
requestedAt: new Date().toISOString(),
|
||||
});
|
||||
await notifyLiveRunRecovery(db, { companyId: interaction.companyId, runId: run.id });
|
||||
return db.update(heartbeatRuns).set({
|
||||
contextSnapshot: sql`jsonb_set(
|
||||
case
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { notifyLiveRunRecovery } from "../live-run-recovery-notifications.js";
|
||||
import { NATIVE_RETAINED_OWNER_WAIT } from "./native-controller-recovery.js";
|
||||
import { notifyNativeEnvironmentSettlement } from "../native-environment-settlement-notifications.js";
|
||||
import { remoteWorkspaceRunReferenceSchema } from "@paperclipai/shared";
|
||||
@@ -664,6 +665,7 @@ export async function claimNativeRestartRecoveries(input: {
|
||||
hasProviderEvidence: false,
|
||||
stderrTail: row.run.stderrExcerpt,
|
||||
});
|
||||
await notifyLiveRunRecovery(tx, { companyId: row.run.companyId, runId: row.run.id, agentId: row.run.agentId });
|
||||
await tx
|
||||
.update(nativeRunFinalizations)
|
||||
.set({
|
||||
@@ -1048,6 +1050,7 @@ export async function claimNativeRestartRecoveries(input: {
|
||||
recycledProviderPids: providerProcesses.recycledPids,
|
||||
stderrTail: row.run.stderrExcerpt,
|
||||
});
|
||||
await notifyLiveRunRecovery(tx, { companyId: row.run.companyId, runId: row.run.id, agentId: row.run.agentId });
|
||||
const claimed = await tx
|
||||
.update(nativeRunFinalizations)
|
||||
.set({
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { notifyLiveRunRecovery } from "../live-run-recovery-notifications.js";
|
||||
import { and, eq, exists, gt, isNull, lte, inArray, notExists, or, sql } from "drizzle-orm";
|
||||
import { checkpointDb, heartbeatRuns, issues, nativeRunFinalizations, nativeRunnerCheckpoints, type Db } from "@paperclipai/db";
|
||||
import { inspectDurablePrpCheckpointHeader, decodeDurablePrpCheckpoint, type DurablePrpCheckpoint, type DurablePrpControlPlane } from "../../vendor/paperclip-runner/index.js";
|
||||
@@ -105,6 +106,7 @@ export function nativeRunnerCheckpointWriter(db: Db, owner: NativeRunnerCheckpoi
|
||||
eq(heartbeatRuns.nativeSessionId, checkpoint.predecessor.normalizedSessionId)));
|
||||
if (!prior) throw new Error("native_runner_checkpoint_binding_mismatch");
|
||||
}
|
||||
await notifyLiveRunRecovery(tx, { companyId: owner.companyId, runId: owner.runId });
|
||||
if (checkpoint.reset) await tx.delete(nativeRunnerCheckpoints).where(and(
|
||||
eq(nativeRunnerCheckpoints.companyId, owner.companyId), eq(nativeRunnerCheckpoints.runId, owner.runId)));
|
||||
else if (checkpoint.removedRecords.length) await tx.delete(nativeRunnerCheckpoints).where(and(
|
||||
|
||||
Reference in new issue
Block a user