diff --git a/server/src/__tests__/native-finalization-retry-budget.test.ts b/server/src/__tests__/native-finalization-retry-budget.test.ts new file mode 100644 index 00000000000..2291f0bf82e --- /dev/null +++ b/server/src/__tests__/native-finalization-retry-budget.test.ts @@ -0,0 +1,132 @@ +import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js"; +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { eq } from "drizzle-orm"; +import { + agents, + companies, + completionContracts, + createDb, + heartbeatRuns, + issues, + nativeRunFinalizations, + nativeRunResults, + workspaceOperations, +} from "@paperclipai/db"; +import { + CONTROL_PLANE_CONFORMANCE_OPEN, + CONTROL_PLANE_CONFORMANCE_RESULT, + CONTROL_PLANE_CONFORMANCE_TERMINAL, +} from "../vendor/paperclip-runner/testing.js"; +import { startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; +import { reconcileNativeFinalizations } from "../services/native-runtime/native-finalization-reconciler.js"; +import { PaperclipControlPlanePort } from "../services/native-runtime/paperclip-control-plane-port.js"; + +const external = resolveExternalTestDatabaseUrl("PAPERCLIP_NATIVE_FINALIZATION_TEST_DATABASE_URL"); + +// Restart chaos w-20261110T044338-s20385/t03: three restarts resumed the run +// (provider attempt 3), the resumed turn's result was accepted, and the first +// transient finalization failure was judged against the provider attempt +// count. The coordinator went terminal at once and the issue rested in review +// with no Retry. +describe("native finalization retry budget after restart resumes", () => { + let temporary: Awaited>; + let db: ReturnType; + const companyId = "72100000-0000-4000-8000-000000000001"; + const agentId = "72100000-0000-4000-8000-000000000002"; + const issueId = "72100000-0000-4000-8000-000000000003"; + const contractId = "72100000-0000-4000-8000-000000000004"; + const runId = "72100000-0000-4000-8000-000000000005"; + + beforeAll(async () => { + if (!external) temporary = await startEmbeddedPostgresTestDatabase("paperclip-native-retry-budget-"); + db = createDb(external ?? temporary.connectionString); + await db.insert(companies).values({ id: companyId, name: "Native retry budget", issuePrefix: "NRB" }); + await db.insert(agents).values({ id: agentId, companyId, name: "Budget agent", adapterType: "codex_local", status: "running" }); + await db.insert(issues).values({ + id: issueId, companyId, title: "Finalize after three resumes", status: "in_progress", + assigneeAgentId: agentId, workMode: "standard", + }); + await db.insert(completionContracts).values({ + id: contractId, companyId, issueId, revision: 1, + schemaVersion: "paperclip.completion-contract.v1", policyVersion: "phase6-v1", risk: "standard", + completionAuthority: "server_arbiter", incompleteCriteriaPolicy: "preserve_non_terminal", + contractJson: { + revision: "phase6-v1", objective: "Finalize after three resumes", + criteria: [{ id: "objective", requirement: "Finalization keeps its own budget" }], + }, + canonicalSha256: "native-retry-budget-contract", createdByActorType: "system", createdByActorId: "test", + }); + await db.insert(heartbeatRuns).values({ + id: runId, companyId, agentId, status: "running", runtimeMode: "native", + runtimeModeReason: "persisted_before_kill_switch", nativeIssueId: issueId, nativeSessionId: runId, + runnerInstanceId: contractId, completionContractId: contractId, + completionContractSha256: "native-retry-budget-contract", contextSnapshot: { issueId }, + }); + const port = new PaperclipControlPlanePort(db, { + companyId, issueId, runId, agentId, sessionId: runId, completionContractId: contractId, + completionContractSha256: "native-retry-budget-contract", sourceInstanceId: contractId, + controlPlaneSourceInstanceId: "budget-control", + }); + await port.openRun({ + ...CONTROL_PLANE_CONFORMANCE_OPEN, + identity: { companyId, issueId, runId, agentId, sessionId: runId }, + sourceInstanceId: contractId, + }); + await port.completeRun({ + result: CONTROL_PLANE_CONFORMANCE_RESULT, + terminal: CONTROL_PLANE_CONFORMANCE_TERMINAL, + callerResultId: "budget-result", + }); + // Each finalization attempt fails, as a status race under contention did. + const stored = await db.select().from(nativeRunResults).where(eq(nativeRunResults.runId, runId)) + .limit(1).then((rows) => rows[0]!); + await db.update(nativeRunResults).set({ + resultJson: { + ...(stored.resultJson as Record), + terminal: { ...CONTROL_PLANE_CONFORMANCE_TERMINAL, runTerminalState: "unknown" }, + }, + }).where(eq(nativeRunResults.id, stored.id)); + await db.insert(workspaceOperations).values({ + companyId, heartbeatRunId: runId, issueId, phase: "workspace_finalize", status: "succeeded", + }); + // Saved as the previous release left it after three restart resumes: the + // provider attempt count is spent and no finalization count exists yet. + await db.update(nativeRunFinalizations).set({ attempt: 3 }).where(eq(nativeRunFinalizations.runId, runId)); + }, 30_000); + + afterAll(async () => { + await db?.$client.end({ timeout: 1 }); + await temporary?.cleanup(); + }); + + it("keeps a resumed run's first finalization failure retryable and still bounds finalization", async () => { + await expect(reconcileNativeFinalizations(db, [runId])).resolves.toEqual([ + expect.objectContaining({ phase: "retryable_failure", failureCode: "native_finalization_invalid" }), + ]); + await expect(db.select().from(issues).where(eq(issues.id, issueId))).resolves.toEqual([ + expect.objectContaining({ status: "in_progress" }), + ]); + await expect(db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, runId))).resolves.toEqual([ + expect.objectContaining({ + nativePhase: "retryable_failure", + resultJson: expect.objectContaining({ finalizationFailureCount: 1 }), + }), + ]); + + await db.update(nativeRunFinalizations).set({ nextAttemptAt: new Date(0) }) + .where(eq(nativeRunFinalizations.runId, runId)); + await expect(reconcileNativeFinalizations(db, [runId])).resolves.toEqual([ + expect.objectContaining({ phase: "retryable_failure" }), + ]); + + await db.update(nativeRunFinalizations).set({ nextAttemptAt: new Date(0) }) + .where(eq(nativeRunFinalizations.runId, runId)); + await reconcileNativeFinalizations(db, [runId]); + await expect(db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, runId))).resolves.toEqual([ + expect.objectContaining({ phase: "terminal_failure", failureCode: "native_finalization_retry_exhausted" }), + ]); + await expect(db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, runId))).resolves.toEqual([ + expect.objectContaining({ resultJson: expect.objectContaining({ finalizationFailureCount: 3 }) }), + ]); + }); +}); diff --git a/server/src/services/native-runtime/native-run-finalizer-telemetry.test.ts b/server/src/services/native-runtime/native-run-finalizer-telemetry.test.ts index df7aa4b6c04..7ec2ea198b1 100644 --- a/server/src/services/native-runtime/native-run-finalizer-telemetry.test.ts +++ b/server/src/services/native-runtime/native-run-finalizer-telemetry.test.ts @@ -858,6 +858,9 @@ describeEmbeddedPostgres("native run finalizer / status decision committer — a it("records the finalization error code when provider-scope retries fail an unsucceeded run", async () => { const fixture = await seedNativeRun(); + // Two earlier finalizations of this result failed; this is the third. + await db.update(heartbeatRuns).set({ resultJson: { finalizationFailureCount: 2 } }) + .where(eq(heartbeatRuns.id, fixture.runId)); await db.insert(nativeRunFinalizations).values({ runId: fixture.runId, companyId, issueId: fixture.issueId, phase: "observed", attempt: 3, }); @@ -869,6 +872,20 @@ describeEmbeddedPostgres("native run finalizer / status decision committer — a expect(run?.error).toBeTruthy(); }); + it("keeps a provider-succeeded run's first finalization failure retryable after three provider attempts", async () => { + const fixture = await seedNativeRun(); + await db.update(heartbeatRuns).set({ resultJson: { prpRunTerminalState: "succeeded" } }) + .where(eq(heartbeatRuns.id, fixture.runId)); + await db.insert(nativeRunFinalizations).values({ + runId: fixture.runId, companyId, issueId: fixture.issueId, phase: "observed", attempt: 3, + }); + const failure = await recordNativeFinalizationFailure({ + db, runId: fixture.runId, error: new Error("native_finalization_invalid"), projectRunStatus: true, + }); + expect(failure).toMatchObject({ phase: "retryable_failure", failureCode: "native_finalization_invalid" }); + await expect(readRunError(fixture.runId)).resolves.toEqual({ status: "running", errorCode: null, error: null }); + }); + it("fails a provider-succeeded run with an error code when workspace retries are exhausted", async () => { // The reported case: the provider turn succeeded, then the workspace // copy-back failed three times. Workspace scope fails the run regardless @@ -887,7 +904,7 @@ describeEmbeddedPostgres("native run finalizer / status decision committer — a it("does not attach a finalization error to a provider-succeeded run when provider retries are exhausted", async () => { const fixture = await seedNativeRun(); - await db.update(heartbeatRuns).set({ resultJson: { prpRunTerminalState: "succeeded" } }) + await db.update(heartbeatRuns).set({ resultJson: { prpRunTerminalState: "succeeded", finalizationFailureCount: 2 } }) .where(eq(heartbeatRuns.id, fixture.runId)); await db.insert(nativeRunFinalizations).values({ runId: fixture.runId, companyId, issueId: fixture.issueId, phase: "observed", attempt: 3, diff --git a/server/src/services/native-runtime/native-run-finalizer.ts b/server/src/services/native-runtime/native-run-finalizer.ts index 80768cfc70d..3b62586d7e7 100644 --- a/server/src/services/native-runtime/native-run-finalizer.ts +++ b/server/src/services/native-runtime/native-run-finalizer.ts @@ -326,6 +326,14 @@ function terminalFinalizationFailureMessage(failureCode: string, failureScope: " : `Native run finalization failed (${failureCode}).`; } +const NATIVE_FINALIZATION_RETRY_LIMIT = 3; + +/** Failed finalizations of this run's saved result. Rows saved before the count existed start at zero. */ +function readFinalizationFailureCount(resultJson: unknown) { + const count = record(resultJson).finalizationFailureCount; + return typeof count === "number" && Number.isInteger(count) && count >= 0 ? count : 0; +} + async function recordRetryableFailure(input: { db: Db; run: typeof heartbeatRuns.$inferSelect; @@ -449,11 +457,19 @@ async function recordRetryableFailure(input: { ? priorFailureDetail.workspaceFinalizeAttempt : 0) + 1 : null; + // Finalization has its own budget. `coordinator.attempt` counts provider + // attempts, so a run resumed three times after restarts would otherwise + // turn its first transient finalization failure into a terminal one and + // strand an accepted result. The count lives on the run's result because + // every claim clears the coordinator's failure detail. + const finalizationFailureCount = input.failureScope === "workspace" + ? null + : readFinalizationFailureCount(input.run.resultJson) + 1; const exhausted = input.permanent === true || (workspaceFinalizeAttempt !== null ? workspaceFinalizeAttempt >= 3 - : input.coordinator.attempt >= 3); + : finalizationFailureCount! >= NATIVE_FINALIZATION_RETRY_LIMIT); const phase = supersededByNewerRun || exhausted ? ("terminal_failure" as const) @@ -541,6 +557,7 @@ async function recordRetryableFailure(input: { originalFailureCode: input.failureCode, // The latest attempt's redacted cause; cleared when it named none. failureCause, + ...(finalizationFailureCount === null ? {} : { finalizationFailureCount }), nextAttemptAt: supersededByNewerRun || exhausted ? null : nextAttemptAt.toISOString(), })}::jsonb`, updatedAt: now,