From 379a70315e47de1ac2c8e4eb452eb3121401c5c7 Mon Sep 17 00:00:00 2001 From: Dotta Date: Tue, 22 Sep 2026 20:04:07 -0700 Subject: [PATCH] Capture delayed repair promotion failure and fail live cells promptly on cancellation --- .../legacy-continuation-authority.test.ts | 16 ++++++++++++++++ tests/lifecycle-baseline/inventory.mjs | 2 +- tests/runner-e2e/accounting-flow.ts | 4 ++-- tests/runner-e2e/catalog.ts | 2 +- 4 files changed, 20 insertions(+), 4 deletions(-) diff --git a/server/src/__tests__/legacy-continuation-authority.test.ts b/server/src/__tests__/legacy-continuation-authority.test.ts index 1cf4761baf8..9906160ce88 100644 --- a/server/src/__tests__/legacy-continuation-authority.test.ts +++ b/server/src/__tests__/legacy-continuation-authority.test.ts @@ -2,6 +2,7 @@ import { randomUUID } from "node:crypto"; import { eq } from "drizzle-orm"; import { afterAll, beforeAll, describe, expect, it } from "vitest"; import { agents, agentWakeupRequests, companies, createDb, heartbeatRuns, issueComments, issueRecoveryActions, issueThreadInteractions, issues } from "@paperclipai/db"; +import { heartbeatService } from "../services/heartbeat.js"; import { recoveryService } from "../services/recovery/service.js"; import { startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; @@ -105,6 +106,21 @@ describe("legacy continuation persisted authority", () => { expect((await f.actions())[0].attemptCount).toBe(2); }); + it("ACCT-04 a delayed second repair remains promotable after controller restart", async () => { + const f = await fixture(); + await f.createRecovery().reconcileLegacyContinuation(f.runId); + const first = (await f.runs()).find(r => r.id !== f.runId)!; + await f.finish(first); + await f.createRecovery().reconcileLegacyContinuation(first.id); + const second = (await f.runs()).find(r => r.status === "scheduled_retry")!; + expect(await f.createRecovery().legacyRepairDispatchBlock(second.id)).toBeNull(); + const restarted = heartbeatService(db); + const promoted = await restarted.promoteDueScheduledRetries(new Date(second.scheduledRetryAt!.getTime() + 1)); + expect.soft(promoted.runIds).toContain(second.id); + expect((await f.runs()).find(r => r.id === second.id)).toMatchObject({ status: "queued", errorCode: null }); + expect((await f.actions())[0].attemptCount).toBe(2); + }); + it("persists the source identity while the second repair waits to dispatch", async () => { const f = await fixture(); await f.createRecovery().reconcileLegacyContinuation(f.runId); diff --git a/tests/lifecycle-baseline/inventory.mjs b/tests/lifecycle-baseline/inventory.mjs index 8d2c6333434..e8ed86280c3 100644 --- a/tests/lifecycle-baseline/inventory.mjs +++ b/tests/lifecycle-baseline/inventory.mjs @@ -301,7 +301,7 @@ export const scenarios = [ ref("unit", "tests/lifecycle-baseline/accounting.test.ts", id), ...(id === "ACCT-01" || id === "ACCT-02" ? [integ("heartbeat-retry-scheduling", id)] : []), ...(id === "ACCT-02" || id === "ACCT-03" ? [integ("legacy-continuation-authority", id)] : []), - ...(id === "ACCT-04" ? [runner("ACCT-04"), integ("legacy-continuation-authority", "replay|restart|fast repair"), integ("heartbeat-retry-scheduling", "concurrent|coalesces")] : []), + ...(id === "ACCT-04" ? [runner("ACCT-04"), integ("legacy-continuation-authority", "ACCT-04|replay|restart|fast repair"), integ("heartbeat-retry-scheduling", "concurrent|coalesces")] : []), ...(id === "ACCT-03" ? [ref("integration", "server/test-baselines/lifecycle-heartbeat.test.ts", "LCA-04 LCA-10"), integ("heartbeat-stale-queue-invalidation", "gate|cap|ownership")] : []), ref("grading", "tests/runner-e2e/accounting.test.ts"), ], live: ["continuation-accounting:accounting-productive-neutral", "continuation-accounting:accounting-productive-noisy", "continuation-accounting:accounting-exhaustion-neutral", "continuation-accounting:accounting-exhaustion-noisy", "continuation-accounting:accounting-repair-stop", "continuation-accounting:accounting-repair-approval"] })), diff --git a/tests/runner-e2e/accounting-flow.ts b/tests/runner-e2e/accounting-flow.ts index 7a3ef3eada6..67b13ee94bc 100644 --- a/tests/runner-e2e/accounting-flow.ts +++ b/tests/runner-e2e/accounting-flow.ts @@ -47,7 +47,7 @@ export async function runAccountingFlow(input: { const key = accept(s) ? JSON.stringify([s.issue.status, s.runs.map(r => [r.id, r.status]), s.interactions.map(i => [i.id, i.status]), s.comments.length]) : ""; const ready = !!key && key === stable; stable = key; return ready; }, - reject: s => s.runs.length > execution.task.expectedRunCount ? "Unexpected additional run exceeds the declared accounting allowance" : s.runs.some(r => ["failed", "timed_out"].includes(r.status)) ? "Provider run failed; inspect retained evidence" : undefined, + reject: s => s.runs.length > execution.task.expectedRunCount ? "Unexpected additional run exceeds the declared accounting allowance" : s.runs.some(r => ["failed", "timed_out", "cancelled"].includes(r.status)) ? `Unexpected terminated run: ${s.runs.filter(r => ["failed", "timed_out", "cancelled"].includes(r.status)).map(r => `${r.status}:${r.errorCode ?? "no-code"}`).join(", ")}` : undefined, }); } async function checkpoint(phase: string, screenshot = false) { @@ -117,7 +117,7 @@ export async function runAccountingFlow(input: { } finally { checks = gradeAccounting({ probe, nonce, agentId: fixtures.agent.id, runtime: execution.profile.expectedRuntimeMode, checkpoints }); if (issue) input.observe(issue, state?.runs ?? [], checks); - await input.evidence("continuation.json", { probe, checkpoints, checks }); + await input.evidence("continuation.json", { probe, checkpoints, checks, lastObserved: state }); } const failed = checks.filter(c => !c.passed); if (failed.length) throw new Error(`Accounting assertions failed: ${failed.map(c => c.id).join(", ")}`); diff --git a/tests/runner-e2e/catalog.ts b/tests/runner-e2e/catalog.ts index 86087d84f51..42e58b7112b 100644 --- a/tests/runner-e2e/catalog.ts +++ b/tests/runner-e2e/catalog.ts @@ -915,7 +915,7 @@ export const runnerSuites: readonly RunnerSuiteFixture[] = [ groups: ["local"], environments: [localEnvironment], profiles: codexContinuityProfiles.map(productionStoryProfile), tasks: accountingTasks, expectedMatrixSize: 8, excludedExecutionIds: accountingTasks.filter(t => !t.id.includes("productive")).map(t => `continuation-accounting.runner-codex.local.${t.id}`), - definitionMetadata: { version: 3, grading: "accounting-v3-plain-tokens", scheduling: "explicit-only", providerTurns: "five productive, three repair, two executed plus one cancelled for Stop" }, + definitionMetadata: { version: 4, grading: "accounting-v4-cancellation-evidence", scheduling: "explicit-only", providerTurns: "five productive, three repair, two executed plus one cancelled for Stop" }, }, { id: "lifecycle-baseline", label: "Lifecycle authority baseline", manualOnly: true,