Jannik/PaperClipAI
1
0 Fork 0
mirror of https://github.com/paperclipai/paperclip.git synced 2026-11-01 23:43:16 +01:00
Code Issues Packages Projects Releases Wiki Activity

perf(heartbeat): drive retry promotion from durable deadlines (#18387)

Browse Source 
Reviewed and verified at f1c48c924d.

Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
DottaandPaperclip authored and GitHub committed 2026-10-31 09:07:52 -05:00
1 parent f46c8071fb
commit baf651c8d0
53 files changed
+970 -70
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

server/src/services/native-runtime/native-failed-workspace-recovery.ts
+1 -1
View File
@@ -65,7 +65,7 @@ export async function preserveNativeFailedWorkspace(db: Db, selected: Pick<typeo
let retained = !retainFiles || alreadyRetained ? run : await recordRequiredWorkspaceRestoreFailure(tx as unknown as Db, run, { workspaceRestoreFailure: "restore_failed" }, source);
if (!retained || retainFiles && !hasRequiredWorkspaceRecovery(retained.resultJson)) return;
if (options.allowPreterminal && coordinator.phase === "observed") {
await notifyExecutionRunWork(tx);
await notifyExecutionRunWork(tx, { companyId: run.companyId });
const [settled] = await tx.update(nativeRunFinalizations).set({ phase: "terminal_failure",
leaseOwner: null, leaseExpiresAt: null, controlDeadlineAt: null, nextAttemptAt: null,
recoveryState: "blocked", failureCode: coordinator.failureCode ?? "native_continuation_requires_reconciliation", updatedAt: new Date(),
server/src/services/native-runtime/native-provider-capacity-retry.ts
+2
View File
@@ -1,3 +1,4 @@
import { notifyScheduledRetryDispatch } from "../run-dispatch-notifications.js";
import { randomUUID } from "node:crypto";
import { and, eq } from "drizzle-orm";
import { agentWakeupRequests, heartbeatRuns, type Db } from "@paperclipai/db";
@@ -49,6 +50,7 @@ export async function materializeNativeProviderCapacityRetry(input: {
scheduledRetryAt: dueAt.toISOString(),
executionRetryAccounting: accountingForScheduledRetry(run, NATIVE_PROVIDER_OVERLOADED_CODE, attempt),
};
await notifyScheduledRetryDispatch(input.tx, { companyId: run.companyId });
const [wake] = await input.tx.insert(agentWakeupRequests).values({
companyId: run.companyId, agentId: run.agentId, source: "automation", triggerDetail: "system",
reason: NATIVE_PROVIDER_OVERLOADED_CODE, status: "queued", runId: retryId,
server/src/services/native-runtime/native-restart-recovery.ts
+1 -1
View File
@@ -924,7 +924,7 @@ export async function claimNativeRestartRecoveries(input: {
recycledProviderPids: providerProcesses.recycledPids,
stderrTail: row.run.stderrExcerpt,
});
await notifyExecutionRunWork(tx);
await notifyExecutionRunWork(tx, { companyId: row.run.companyId });
await tx
.update(nativeRunFinalizations)
.set({
server/src/services/native-runtime/native-run-finalizer.ts
+2 -2
View File
@@ -401,7 +401,7 @@ async function recordRetryableFailure(input: {
if (phase === "terminal_failure" && failureCode === "native_workspace_sync_out_unsafe_archive") {
await notifyUnsafeWorkspaceExportRecovery(tx, { companyId: input.run.companyId, issueId: input.coordinator.issueId });
}
await notifyExecutionRunWork(tx);
await notifyExecutionRunWork(tx, { companyId: input.run.companyId });
await notifyChatRunMilestones(tx, { companyId: input.run.companyId, issueId: input.coordinator.issueId });
await tx
.update(nativeRunFinalizations)
@@ -1659,7 +1659,7 @@ export async function finalizeNativeRun(input: {
if (input.projectRunStatus) await notifyCancelledWorkspacePreservation(tx, { companyId: run.companyId, runId: run.id });
if (input.projectRunStatus) await notifyUnsafeWorkspaceExportRunChange(tx, run);
if (input.projectRunStatus) await notifyDeliveryWork(tx, DELIVERY_QUEUES.accounting, run.companyId);
await notifyExecutionRunWork(tx);
await notifyExecutionRunWork(tx, { companyId: run.companyId });
await notifyChatRunMilestones(tx, { companyId: run.companyId, issueId: coordinator.issueId });
const [after] = await projectRun(tx);
return after ? { before, after } : null;
server/src/services/native-runtime/native-run-projection.ts
+1 -1
View File
@@ -138,7 +138,7 @@ export async function projectCommittedRun(input: {
await notifyCancelledWorkspacePreservation(tx, { companyId: input.run.companyId, runId: input.run.id });
await notifyUnsafeWorkspaceExportRunChange(tx, input.run);
await notifyDeliveryWork(tx, DELIVERY_QUEUES.accounting, input.run.companyId);
await notifyExecutionRunWork(tx);
await notifyExecutionRunWork(tx, { companyId: input.run.companyId });
await notifyChatRunMilestones(tx, { companyId: input.run.companyId, issueId: input.coordinator.issueId });
const [after] = await tx
.update(heartbeatRuns)
server/src/services/native-runtime/native-safe-replacement.test.ts
+10
View File
@@ -1,3 +1,4 @@
import { createDeliveryWorkCoordinator } from "../delivery-work-coordinator.js";
import { getExecutionBlocker } from "../execution-blocker.js";
import { createRunDispatch, deriveCommentId } from "../../modules/run-dispatch/index.js";
import { buildExecutionContinuation } from "../execution-continuation.js";
@@ -577,7 +578,12 @@ const support = externalDatabaseUrl
});
it("persists exactly one linked successor under competing sweepers and restarts", async () => {
const source = await seed(2);
const owner = heartbeatService(db);
const coordinator = createDeliveryWorkCoordinator({ owner: db, canRun: () => true, onError: error => { throw error; } });
const worker = owner.registerRunDispatchWork(coordinator, () => true);
await worker.ready;
const now = new Date();
try {
await Promise.all([
reconcileSafeNativeReplacements(db, now),
reconcileSafeNativeReplacements(db, now),
@@ -608,6 +614,10 @@ const support = externalDatabaseUrl
.where(eq(nativeRunFinalizations.runId, source.runId))
)[0]?.failureDetail?.successorRunId,
).toBe(children[0]!.id);
await vi.waitFor(() => expect(coordinator.nextWakeAt()).toBe(children[0]!.scheduledRetryAt!.getTime()), { timeout: 5_000 });
} finally {
await worker.stop(); await coordinator.stop(); await owner.drainActiveRunExecutions();
}
});
it("retries status publication after a crash without dispatching provider work", async () => {
await deliverExecutionStatuses(db);
server/src/services/native-runtime/native-safe-replacement.ts
+4 -2
View File
@@ -1,3 +1,4 @@
import { notifyScheduledRetryDispatch } from "../run-dispatch-notifications.js";
import { notifyNativeStatusReassessment } from "../native-status-reassessment-notifications.js";
import { notifyUnsafeWorkspaceExportRecovery } from "../unsafe-workspace-export-notifications.js";
import { notifyExecutionRunWork } from "../execution-control-notifications.js";
@@ -267,7 +268,7 @@ export async function reconcileSafeNativeReplacements(
current.failureDetail?.replacementDenied
)
return;
await notifyExecutionRunWork(tx);
await notifyExecutionRunWork(tx, { companyId: run.companyId });
await tx
.update(nativeRunFinalizations)
.set({
@@ -409,7 +410,8 @@ export async function reconcileSafeNativeReplacements(
idempotencyKey: `native-safe-replacement:${run.id}`,
})
.returning();
await notifyExecutionRunWork(tx);
await notifyExecutionRunWork(tx, { companyId: run.companyId });
await notifyScheduledRetryDispatch(tx, { companyId: run.companyId });
await tx.insert(heartbeatRuns).values({
id: successorRunId,
companyId: run.companyId,
server/src/services/native-runtime/native-session-executor.ts
+3 -3
View File
@@ -9902,7 +9902,7 @@ async function executePaperclipNativeSessionWithinScope(
eq(nativeRunFinalizations.runId, input.execution.binding.runId),
eq(nativeRunFinalizations.leaseOwner, leaseOwner),
),
));
), { companyId: input.execution.binding.companyId });
const failedAtMs = Date.now();
const executionFailureMessage = redactIdentityText(
@@ -9954,7 +9954,7 @@ async function executePaperclipNativeSessionWithinScope(
eq(nativeRunFinalizations.leaseOwner, leaseOwner),
eq(nativeRunFinalizations.attempt, attempt),
isNull(nativeRunFinalizations.resultId),
)).returning({ runId: nativeRunFinalizations.runId }));
)).returning({ runId: nativeRunFinalizations.runId }), { companyId: input.execution.binding.companyId });
if (settled) {
await stoppedLeaseRenewal;
if (warmSessionId !== null && lifecyclePolicy.mode === "warm") {
@@ -10041,7 +10041,7 @@ async function executePaperclipNativeSessionWithinScope(
),
)
.for("update");
await notifyExecutionRunWork(tx);
await notifyExecutionRunWork(tx, { companyId: input.execution.binding.companyId });
await notifyNativeFinalizationWork(tx, input.execution.binding);
const updated = await tx
.update(nativeRunFinalizations)
server/src/services/native-runtime/native-superseded-retry.ts
+1 -1
View File
@@ -39,7 +39,7 @@ export async function retireSupersededNativeRetry(db: Db, input: {
)).limit(1);
if (!source) return false;
await notifyExecutionRunWork(db);
await notifyExecutionRunWork(db, { companyId: run.companyId });
await db.update(nativeRunFinalizations).set({ phase: "terminal_failure", nextAttemptAt: null,
leaseExpiresAt: null, controlDeadlineAt: null, recoveryState: "blocked",
failureCode: "native_retry_superseded", failureDetail: { ...coordinator.failureDetail,