fix(recovery): avoid duplicate terminal run status notifications (#17614)

Retain pending status deliveries when Native or legacy terminal recovery projects the same state. Preserve completion time on unchanged Native terminal status. Real state, metadata, error, explicit patch, and retained-file changes still notify. The publisher acknowledgement, accepted results, retry deadlines, recovery holds, ownership checks, and workspace cleanup remain intact.

Correct the queued Retry browser fixture race by releasing its old-process guard at the exact Retry request, requiring a successful response, and retaining fixture cleanup after failure.

Validated by 159 focused tests, 653 wider Native recovery tests, six real Chromium/PostgreSQL browser cases, five queued Retry repeats, build/typecheck, exact-head Apex 5/5, and green required CI. The original detached-button CI timeout is documented in the PR. The additional full local repository Vitest command is still running and is not claimed complete.

Co-Authored-By: Paperclip <noreply@paperclip.ing>

server/src/__tests__/legacy-workspace-restore-recovery.test.ts

@@ -1,6 +1,6 @@
 import { randomUUID } from "node:crypto";
 import { and, eq, sql } from "drizzle-orm";
-import { afterAll, beforeAll, describe, expect, it } from "vitest";
+import { afterAll, beforeAll, describe, expect, it, vi } from "vitest";
 import { agents, companies, createDb, environmentLeases, environments, executionWorkspaces, heartbeatRuns, issueRecoveryActions, issues, projects } from "@paperclipai/db";
 import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js";
 import { recordLegacyWorkspaceRestoreFailure, legacyExecutionNeedsReconciliationWithEvidence, terminalizeLegacyExecution } from "../services/legacy-execution-recovery.js";
@@ -8,6 +8,7 @@ import { hasConversationContinuationPolicy, conversationRecoveryActionPredicate
 import { preserveWorkspaceRestoreRecoveryMetadataSql } from "../services/legacy-workspace-restore-recovery.js";
 import { hasConfirmedSandboxStopAndRetain, prepareSandboxStopAndRetain, settleStopOnlyCleanup } from "../services/sandbox-stop-and-retain.js";
 import { environmentService } from "../services/environments.js";
+import { deliverExecutionStatuses } from "../services/execution-status-delivery.js";
 import { hasRequiredWorkspaceRecovery } from "../services/workspace-restore-recovery-state.js";
 
 const externalTestDatabaseUrl = process.env.PAPERCLIP_TEST_DATABASE_URL?.trim();
@@ -45,6 +46,66 @@ const support = externalTestDatabaseUrl ? { supported: true } : await getEmbedde
   } };
   const readLease = async (id: string) => (await db.select().from(environmentLeases).where(eq(environmentLeases.id, id)))[0];
 
+  it("preserves pending and acknowledged status delivery on unpatched legacy recovery retries", async () => {
+    await deliverExecutionStatuses(db, { publish: vi.fn() });
+    const f = await seed();
+    const failed = (await terminalizeLegacyExecution({ db, run: f.run, status: "failed", patch }))!;
+    const holds = await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.sourceIssueId, f.issueId));
+    expect(holds).toHaveLength(1);
+    expect(hasRequiredWorkspaceRecovery(failed.resultJson)).toBe(true);
+    const repeated = (await terminalizeLegacyExecution({ db, run: failed, status: "failed" }))!;
+    expect(repeated.executionStatusDeliveryId).toBe(failed.executionStatusDeliveryId);
+    const publish = vi.fn();
+    await deliverExecutionStatuses(db, { publish });
+    await terminalizeLegacyExecution({ db, run: failed, status: "failed" });
+    await deliverExecutionStatuses(db, { publish });
+    expect(publish.mock.calls.filter(([event]) => event.payload.runId === f.runId)).toHaveLength(1);
+    const after = await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.sourceIssueId, f.issueId));
+    expect(after).toHaveLength(1);
+    expect(after[0]).toMatchObject({ id: holds[0].id, status: holds[0].status, outcome: holds[0].outcome,
+      ownerType: holds[0].ownerType, wakePolicy: null, evidence: { ...holds[0].evidence,
+        automaticRecovery: expect.objectContaining({ policy: "preserve_without_replay_v1", replay: "blocked",
+          runId: f.runId, actionOutcome: "unknown" }),
+      } });
+    expect(await readLease(f.lease.id)).toMatchObject({ status: "pending_cleanup", leasePolicy: "retain_on_failure" });
+    expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.runId)))[0].resultJson).toEqual(failed.resultJson);
+  });
+
+  it("delivers new retained workspace evidence even when legacy run status is unchanged", async () => {
+    await deliverExecutionStatuses(db, { publish: vi.fn() });
+    const f = await seed();
+    const failed = (await terminalizeLegacyExecution({ db, run: f.run, status: "failed", patch }))!;
+    const publish = vi.fn();
+    await deliverExecutionStatuses(db, { publish });
+    const [added] = await db.insert(environmentLeases).values({ ...f.lease, id: randomUUID(), providerLeaseId: randomUUID() }).returning();
+    const updated = (await terminalizeLegacyExecution({ db, run: failed, status: "failed" }))!;
+    expect(updated.executionStatusDeliveryId).toBeTruthy();
+    expect(updated.executionStatusDeliveryId).not.toBe(failed.executionStatusDeliveryId);
+    expect(updated.resultJson?.workspaceRestoreRecovery).toMatchObject({ leaseIds: expect.arrayContaining([f.lease.id, added.id]) });
+    await deliverExecutionStatuses(db, { publish });
+    expect(publish.mock.calls.filter(([event]) => event.payload.runId === f.runId)).toHaveLength(2);
+    await terminalizeLegacyExecution({ db, run: updated, status: "failed" });
+    await deliverExecutionStatuses(db, { publish });
+    expect(publish.mock.calls.filter(([event]) => event.payload.runId === f.runId)).toHaveLength(2);
+  });
+
+  it("retains write conditions and delivers an explicit patch on unchanged legacy status", async () => {
+    await deliverExecutionStatuses(db, { publish: vi.fn() });
+    const f = await seed();
+    const failed = (await terminalizeLegacyExecution({ db, run: f.run, status: "failed", patch }))!;
+    const publish = vi.fn();
+    await deliverExecutionStatuses(db, { publish });
+    const holds = await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.sourceIssueId, f.issueId));
+    expect(await terminalizeLegacyExecution({ db, run: failed, status: "failed", writeConditions: [sql`false`] })).toBeNull();
+    await deliverExecutionStatuses(db, { publish });
+    expect(publish.mock.calls.filter(([event]) => event.payload.runId === f.runId)).toHaveLength(1);
+    expect(await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.sourceIssueId, f.issueId))).toEqual(holds);
+    const patched = (await terminalizeLegacyExecution({ db, run: failed, status: "failed", patch: { error: "Retained workspace still needs repair" } }))!;
+    expect(patched.error).toBe("Retained workspace still needs repair");
+    await deliverExecutionStatuses(db, { publish });
+    expect(publish.mock.calls.filter(([event]) => event.payload.runId === f.runId)).toHaveLength(2);
+  });
+
   async function stopAndRetain(f: Awaited<ReturnType<typeof seed>>, retry = false) {
     let stopping = (await prepareSandboxStopAndRetain(db, f.lease))!;
     expect(stopping.status).toBe("pending_cleanup");