Capture delayed repair promotion failure and fail live cells promptly on cancellation

server/src/__tests__/legacy-continuation-authority.test.ts

@@ -2,6 +2,7 @@ import { randomUUID } from "node:crypto";
 import { eq } from "drizzle-orm";
 import { afterAll, beforeAll, describe, expect, it } from "vitest";
 import { agents, agentWakeupRequests, companies, createDb, heartbeatRuns, issueComments, issueRecoveryActions, issueThreadInteractions, issues } from "@paperclipai/db";
+import { heartbeatService } from "../services/heartbeat.js";
 import { recoveryService } from "../services/recovery/service.js";
 import { startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js";
 
@@ -105,6 +106,21 @@ describe("legacy continuation persisted authority", () => {
     expect((await f.actions())[0].attemptCount).toBe(2);
   });
 
+  it("ACCT-04 a delayed second repair remains promotable after controller restart", async () => {
+    const f = await fixture();
+    await f.createRecovery().reconcileLegacyContinuation(f.runId);
+    const first = (await f.runs()).find(r => r.id !== f.runId)!;
+    await f.finish(first);
+    await f.createRecovery().reconcileLegacyContinuation(first.id);
+    const second = (await f.runs()).find(r => r.status === "scheduled_retry")!;
+    expect(await f.createRecovery().legacyRepairDispatchBlock(second.id)).toBeNull();
+    const restarted = heartbeatService(db);
+    const promoted = await restarted.promoteDueScheduledRetries(new Date(second.scheduledRetryAt!.getTime() + 1));
+    expect.soft(promoted.runIds).toContain(second.id);
+    expect((await f.runs()).find(r => r.id === second.id)).toMatchObject({ status: "queued", errorCode: null });
+    expect((await f.actions())[0].attemptCount).toBe(2);
+  });
+
   it("persists the source identity while the second repair waits to dispatch", async () => {
     const f = await fixture();
     await f.createRecovery().reconcileLegacyContinuation(f.runId);

tests/lifecycle-baseline/inventory.mjs

@@ -301,7 +301,7 @@ export const scenarios = [
     ref("unit", "tests/lifecycle-baseline/accounting.test.ts", id),
     ...(id === "ACCT-01" || id === "ACCT-02" ? [integ("heartbeat-retry-scheduling", id)] : []),
     ...(id === "ACCT-02" || id === "ACCT-03" ? [integ("legacy-continuation-authority", id)] : []),
-    ...(id === "ACCT-04" ? [runner("ACCT-04"), integ("legacy-continuation-authority", "replay|restart|fast repair"), integ("heartbeat-retry-scheduling", "concurrent|coalesces")] : []),
+    ...(id === "ACCT-04" ? [runner("ACCT-04"), integ("legacy-continuation-authority", "ACCT-04|replay|restart|fast repair"), integ("heartbeat-retry-scheduling", "concurrent|coalesces")] : []),
     ...(id === "ACCT-03" ? [ref("integration", "server/test-baselines/lifecycle-heartbeat.test.ts", "LCA-04 LCA-10"), integ("heartbeat-stale-queue-invalidation", "gate|cap|ownership")] : []),
     ref("grading", "tests/runner-e2e/accounting.test.ts"),
   ], live: ["continuation-accounting:accounting-productive-neutral", "continuation-accounting:accounting-productive-noisy", "continuation-accounting:accounting-exhaustion-neutral", "continuation-accounting:accounting-exhaustion-noisy", "continuation-accounting:accounting-repair-stop", "continuation-accounting:accounting-repair-approval"] })),

tests/runner-e2e/accounting-flow.ts

@@ -47,7 +47,7 @@ export async function runAccountingFlow(input: {
         const key = accept(s) ? JSON.stringify([s.issue.status, s.runs.map(r => [r.id, r.status]), s.interactions.map(i => [i.id, i.status]), s.comments.length]) : "";
         const ready = !!key && key === stable; stable = key; return ready;
       },
-      reject: s => s.runs.length > execution.task.expectedRunCount ? "Unexpected additional run exceeds the declared accounting allowance" : s.runs.some(r => ["failed", "timed_out"].includes(r.status)) ? "Provider run failed; inspect retained evidence" : undefined,
+      reject: s => s.runs.length > execution.task.expectedRunCount ? "Unexpected additional run exceeds the declared accounting allowance" : s.runs.some(r => ["failed", "timed_out", "cancelled"].includes(r.status)) ? `Unexpected terminated run: ${s.runs.filter(r => ["failed", "timed_out", "cancelled"].includes(r.status)).map(r => `${r.status}:${r.errorCode ?? "no-code"}`).join(", ")}` : undefined,
     });
   }
   async function checkpoint(phase: string, screenshot = false) {
@@ -117,7 +117,7 @@ export async function runAccountingFlow(input: {
   } finally {
     checks = gradeAccounting({ probe, nonce, agentId: fixtures.agent.id, runtime: execution.profile.expectedRuntimeMode, checkpoints });
     if (issue) input.observe(issue, state?.runs ?? [], checks);
-    await input.evidence("continuation.json", { probe, checkpoints, checks });
+    await input.evidence("continuation.json", { probe, checkpoints, checks, lastObserved: state });
   }
   const failed = checks.filter(c => !c.passed);
   if (failed.length) throw new Error(`Accounting assertions failed: ${failed.map(c => c.id).join(", ")}`);

tests/runner-e2e/catalog.ts

@@ -915,7 +915,7 @@ export const runnerSuites: readonly RunnerSuiteFixture[] = [
     groups: ["local"], environments: [localEnvironment], profiles: codexContinuityProfiles.map(productionStoryProfile),
     tasks: accountingTasks, expectedMatrixSize: 8,
     excludedExecutionIds: accountingTasks.filter(t => !t.id.includes("productive")).map(t => `continuation-accounting.runner-codex.local.${t.id}`),
-    definitionMetadata: { version: 3, grading: "accounting-v3-plain-tokens", scheduling: "explicit-only", providerTurns: "five productive, three repair, two executed plus one cancelled for Stop" },
+    definitionMetadata: { version: 4, grading: "accounting-v4-cancellation-evidence", scheduling: "explicit-only", providerTurns: "five productive, three repair, two executed plus one cancelled for Stop" },
   },
   {
     id: "lifecycle-baseline", label: "Lifecycle authority baseline", manualOnly: true,