Merge current master execution recovery into idle ingress

server/src/__tests__/chat-channels.integration.test.ts

@@ -64519,6 +64519,45 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => {
     } finally { await f.service.shutdown(); }
   });
 
+  it("retries saved GitHub input after an endpoint admission lock collision", async () => {
+    const f = await successfulGitHubQueuedInputFixture();
+    const heartbeat = heartbeatService(db);
+    const failures: unknown[] = [];
+    const coordinator = createDeliveryWorkCoordinator({ owner: db, canRun: () => true,
+      onError: error => failures.push(error) });
+    let unlock!: () => void, locked!: () => void;
+    const acquired = new Promise<void>(resolve => { locked = resolve; });
+    const released = new Promise<void>(resolve => { unlock = resolve; });
+    const competingAdmission = db.transaction(async tx => {
+      await tx.select({ id: chatEndpoints.id }).from(chatEndpoints)
+        .where(eq(chatEndpoints.id, f.endpoint.id)).for("no key update");
+      locked();
+      await released;
+    });
+    try {
+      await acquired;
+      await heartbeat.registerDeferredReceiptWork(coordinator, () => true).ready;
+      const priorFailures = failures.length;
+      const retry = await requestTaskRetry(f.input);
+      await expect.poll(() => failures.length, { timeout: 5_000 }).toBeGreaterThan(priorFailures);
+      expect((await readAgentWakeupRequestsById(f.pending!.id))[0].runId).toBeNull();
+      unlock(); await competingAdmission;
+      await expect.poll(async () => (await readAgentWakeupRequestsById(f.pending!.id))[0].runId,
+        { timeout: 40_000 }).toBeTruthy();
+      await expect.poll(async () => (await readAgentWakeupRequestsById(retry.requestId!))[0].status,
+        { timeout: 40_000 }).toBe("coalesced");
+      const [saved] = await readAgentWakeupRequestsById(f.pending!.id);
+      expect((await readHeartbeatRunsByCompanyId(f.fixture.companyId))
+        .filter(run => run.wakeupRequestId === f.pending!.id)).toHaveLength(1);
+      expect((await readAgentWakeupRequestsById(retry.requestId!))[0].runId).toBe(saved.runId);
+      expect((await readHeartbeatRunsById(f.runId))[0]).toMatchObject({ status: "succeeded" });
+    } finally {
+      unlock(); await competingAdmission;
+      await coordinator.stop();
+      await f.service.shutdown();
+    }
+  }, 90_000);
+
   it.each(["reconnected", "newer_input", "reconnected_newer_input", "snapshot_newer_input"] as const)(
     "admits current GitHub task Retry without replaying obsolete input: %s", async kind => {
       const f = await currentGitHubTaskRetryFixture(kind.includes("newer_input"));

server/src/__tests__/deferred-execution-receipt-work.test.ts

@@ -72,14 +72,18 @@ describe("deferred receipt durable dispatch", () => {
     // during actual process execution; accelerated steps only settle idle work.
     vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"], shouldAdvanceTime: true });
     heartbeat = heartbeatService(db);
-    coordinator = createDeliveryWorkCoordinator({ owner: db, canRun: () => !drain, canReconcile: () => true,
+    coordinator = createCoordinator();
+  });
+  function createCoordinator() {
+    const worker = createDeliveryWorkCoordinator({ owner: db, canRun: () => !drain, canReconcile: () => true,
       onError: error => { errors.push(error); errorTimes.push(Date.now()); } });
-    const register = coordinator.register.bind(coordinator);
-    coordinator.register = (queue, task) => register(queue, { ...task, run: signal => {
+    const register = worker.register.bind(worker);
+    worker.register = (queue, task) => register(queue, { ...task, run: signal => {
       const pass = Promise.resolve(task.run(signal)); passes.add(pass);
       void pass.then(() => passes.delete(pass), () => passes.delete(pass)); return pass;
     } });
-  });
+    return worker;
+  }
   afterEach(async () => {
     await coordinator.stop(); await heartbeat.drainActiveRunExecutions();
     for (const stop of extraStops.splice(0)) await stop();
@@ -153,6 +157,78 @@ describe("deferred receipt durable dispatch", () => {
       source: "on_demand", status: "completed", reason: "task_retry", payload: { issueId: f.issue.id } });
     await heartbeat.registerDeferredReceiptWork(coordinator, () => !drain).ready; await quiet();
   });
+  it.each([false, true])("reconsiders every saved receipt during repeated company eligibility changes (expired wait: %s)", async expiredWait => {
+    const f = await fixture();
+    const ids = Array.from({ length: 178 }, (_, index) => `${(index + 1).toString(16).padStart(8, "0")}-0000-4000-8000-000000000000`);
+    await db.insert(agentWakeupRequests).values(ids.map(id => ({ id, companyId: f.companyId, agentId: f.agent.id,
+      source: "on_demand", status: "deferred_issue_execution",
+      updatedAt: new Date(Date.now() - 60_000),
+      payload: { issueId: f.issue.id, _paperclipWakeContext: { wakeCommentIds: [randomUUID()] },
+        ...(expiredWait ? { executionWait: { reason: "recovery" } } : {}) },
+    })));
+    const checked = new Set<string>();
+    await registerDeferredReceiptWork(db, coordinator, {
+      resume: async scope => { checked.add(scope.requestId!); },
+      settleTaskChat: async () => {}, createNativeBatch: () => async () => {}, schedulingSuppressed: async () => false,
+    }, () => !drain).ready;
+    for (let pass = 0; pass < 12; pass++) {
+      await db.transaction(tx => notifyDeferredExecutionEligibility(tx, { companyId: f.companyId }));
+      await vi.advanceTimersByTimeAsync(31_000);
+      await Promise.allSettled([...passes]);
+      await immediate();
+    }
+    expect(checked.size).toBe(ids.length);
+    await quiet();
+  });
+  it("dispatches task Retry past a repeatedly invalidated backlog into real process output", async () => {
+    const f = await fixture();
+    // Hold executor capacity while queue time advances; reconstruct the saved
+    // run with fresh workers at PostgreSQL time before real process execution.
+    const [occupant] = await db.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: f.agent.id,
+      status: "running", startedAt: new Date() }).returning();
+    await db.update(agents).set({ runtimeConfig: { heartbeat: { enabled: true, intervalSec: 0,
+      wakeOnDemand: true, maxConcurrentRuns: 1 } } }).where(eq(agents.id, f.agent.id));
+    const [closed] = await db.insert(issues).values({ companyId: f.companyId, title: "Closed task history",
+      status: "done", assigneeAgentId: f.agent.id }).returning();
+    await db.insert(agentWakeupRequests).values(Array.from({ length: 178 }, (_, index) => ({
+      id: `${(index + 1).toString(16).padStart(8, "0")}-0000-4000-8000-000000000000`,
+      companyId: f.companyId, agentId: f.agent.id, source: "on_demand", status: "deferred_issue_execution",
+      payload: { issueId: closed.id, _paperclipWakeContext: { wakeCommentIds: [randomUUID()] } },
+    })));
+    await heartbeat.registerDeferredReceiptWork(coordinator, () => !drain).ready;
+    await heartbeat.registerScheduledRetryCancellationWork(coordinator, () => !drain).ready;
+    const retry = await requestTaskRetry({ db, companyId: f.companyId, issueId: f.issue.id, actorId: f.actorId,
+      observedRunId: f.source.id, idempotencyKey: randomUUID() });
+    const read = async () => (await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, retry.requestId!)))[0];
+    for (let pass = 0; pass < 12 && !(await read()).runId; pass++) {
+      await db.transaction(tx => notifyDeferredExecutionEligibility(tx, { companyId: f.companyId }));
+      await vi.advanceTimersByTimeAsync(31_000);
+      await Promise.allSettled([...passes]);
+      await immediate();
+    }
+    const receipt = await read();
+    expect(receipt.runId).toBeTruthy();
+    expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, receipt.runId!)))[0].status).toBe("queued");
+    await coordinator.stop();
+    await heartbeat.drainActiveRunExecutions();
+    vi.setSystemTime(vi.getRealSystemTime());
+    heartbeat = heartbeatService(db);
+    coordinator = createCoordinator();
+    await heartbeat.registerDeferredReceiptWork(coordinator, () => !drain).ready;
+    await heartbeat.registerScheduledRetryCancellationWork(coordinator, () => !drain).ready;
+    await db.update(heartbeatRuns).set({ status: "succeeded", finishedAt: new Date() }).where(eq(heartbeatRuns.id, occupant.id));
+    await heartbeat.registerRunDispatchWork(coordinator, () => !drain).ready;
+    await waitForReal(async () => {
+      await heartbeat.drainActiveRunExecutions();
+      const [run] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, receipt.runId!));
+      expect(run.status, JSON.stringify({ error: run.error, code: run.errorCode })).toBe("succeeded");
+      expect(run.stdoutExcerpt).toContain("receipt-proof");
+    }, { timeout: 15_000 });
+    expect(await db.select({ id: heartbeatRuns.id }).from(heartbeatRuns)
+      .where(sql`${heartbeatRuns.contextSnapshot}->>'taskRetryRequestId' = ${retry.requestId}`)).toHaveLength(1);
+    await finish(f.issue.id);
+    await quiet();
+  });
   it("dispatches an ordinary committed task Retry into saved process output", async () => {
     const f = await fixture();
     await heartbeat.registerDeferredReceiptWork(coordinator, () => !drain).ready;

server/src/__tests__/native-run-projection-work.test.ts

@@ -1,3 +1,4 @@
+import * as wakeSettlementNotifications from "../services/external-conversation-settlement-notifications.js";
 import { heartbeatService } from "../services/heartbeat.js";
 import { instanceSettingsService } from "../services/instance-settings.js";
 import { createHeartbeatLifecycle, type HeartbeatLifecycleDependencies } from "../services/heartbeat/run-lifecycle.js";
@@ -8,14 +9,14 @@ import { mkdtemp, rm } from "node:fs/promises";
 import { tmpdir } from "node:os";
 import { join } from "node:path";
 import { setTimeout as delay } from "node:timers/promises";
-import { eq } from "drizzle-orm";
+import { eq, sql } from "drizzle-orm";
 import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
-import { agents, companies, completionContracts, createDb, heartbeatRuns, issues, nativeRunFinalizations, type Db } from "@paperclipai/db";
+import { agentWakeupRequests, agents, companies, completionContracts, createDb, heartbeatRuns, issues, nativeRunFinalizations, nativeRunResults, workspaceOperations, type Db } from "@paperclipai/db";
 import { createDeliveryWorkCoordinator } from "../services/delivery-work-coordinator.js";
 import { nativeRunProjectionWork } from "../services/native-runtime/native-run-projection-work.js";
 import { projectCommittedRun } from "../services/native-runtime/native-run-projection.js";
 import { notifyNativeFinalizationWork } from "../services/native-finalization-notifications.js";
-import { finalizeNativeRun } from "../services/native-runtime/native-run-finalizer.js";
+import { finalizeNativeRun, recordNativeFinalizationFailure } from "../services/native-runtime/native-run-finalizer.js";
 import * as statusCommitter from "../services/native-runtime/status-decision-committer.js";
 import { PaperclipControlPlanePort } from "../services/native-runtime/paperclip-control-plane-port.js";
 import { NativeRunCoordinatorStore } from "../services/native-runtime/native-run-coordinator-store.js";
@@ -120,6 +121,154 @@ describe("committed native run projection lifecycle", () => {
   async function accepted(status: "succeeded" | "failed" | "cancelled" = "succeeded") {
     const binding = await seed(); await complete(binding, "control_plane", db, status); await barrier(binding); return binding;
   }
+  async function claimedWake(binding: Awaited<ReturnType<typeof seed>>) {
+    const [wake] = await db.insert(agentWakeupRequests).values({ companyId, agentId, source: "issue",
+      status: "claimed", runId: binding.runId, claimedAt: new Date(), payload: { issueId: binding.issueId } }).returning();
+    await db.update(heartbeatRuns).set({ wakeupRequestId: wake.id }).where(eq(heartbeatRuns.id, binding.runId));
+    return wake;
+  }
+  const readWake = async (id: string) => (await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, id)))[0];
+
+  it.each(["succeeded", "failed", "cancelled"] as const)("settles the exact wake atomically with a fresh %s projection", async status => {
+    const binding = await accepted(status), wake = await claimedWake(binding);
+    await finalizeNativeRun({ db, runId: binding.runId, workspaceFinalizeStatus: "succeeded", projectRunStatus: true });
+    const run = await read(binding.runId), settled = await readWake(wake.id);
+    expect(run.status).toBe(status);
+    expect(settled).toMatchObject({ status: status === "succeeded" ? "completed" : status, runId: binding.runId,
+      companyId, agentId, finishedAt: run.finishedAt, error: run.error });
+    expect(settled.finishedAt).not.toBeNull();
+    await projectCommittedRun({ db, run, coordinator: await state(binding.runId) });
+    expect(await readWake(wake.id)).toEqual(settled);
+  });
+
+  it.each(["provider", "workspace"] as const)("settles the exact wake when %s finalization retries exhaust", async scope => {
+    const binding = await accepted(), wake = await claimedWake(binding);
+    if (scope === "workspace") await db.update(workspaceOperations).set({ status: "failed" })
+      .where(eq(workspaceOperations.heartbeatRunId, binding.runId));
+    if (scope === "provider") await db.update(nativeRunResults).set({
+      resultJson: sql`jsonb_set(${nativeRunResults.resultJson}, '{terminal,runTerminalState}', '"unknown"'::jsonb)`,
+    }).where(eq(nativeRunResults.runId, binding.runId));
+    for (let attempt = 0; attempt < 3; attempt++) {
+      await db.update(nativeRunFinalizations).set({ nextAttemptAt: new Date(0) })
+        .where(eq(nativeRunFinalizations.runId, binding.runId));
+      if (scope === "provider") await reconcileNativeFinalizations(db, [binding.runId]);
+      else await recordNativeFinalizationFailure({ db, runId: binding.runId, projectRunStatus: true,
+        failureScope: "workspace", error: new Error("native_workspace_finalize_failed") });
+      if (attempt < 2) {
+        expect((await read(binding.runId)).status).toBe("running");
+        expect(await readWake(wake.id)).toMatchObject({ status: "claimed", finishedAt: null });
+      }
+    }
+    const run = await read(binding.runId);
+    expect((await state(binding.runId)).phase).toBe("terminal_failure");
+    expect(run).toMatchObject({ status: scope === "provider" ? "succeeded" : "failed", nativePhase: "terminal_failure" });
+    expect(run.finishedAt).not.toBeNull();
+    expect(await readWake(wake.id)).toMatchObject({ status: scope === "provider" ? "completed" : "failed",
+      runId: binding.runId, finishedAt: run.finishedAt });
+  });
+
+  it("keeps the source wake claimed when exhausted failure does not own terminal projection", async () => {
+    const binding = await accepted(), wake = await claimedWake(binding);
+    await db.update(workspaceOperations).set({ status: "failed" }).where(eq(workspaceOperations.heartbeatRunId, binding.runId));
+    for (let attempt = 0; attempt < 3; attempt++) await recordNativeFinalizationFailure({ db, runId: binding.runId,
+      projectRunStatus: false, failureScope: "workspace", error: new Error("native_workspace_finalize_failed") });
+    expect((await state(binding.runId)).phase).toBe("terminal_failure");
+    expect((await read(binding.runId)).status).toBe("running");
+    expect(await readWake(wake.id)).toEqual(wake);
+  });
+
+  it("rolls back exhausted projection and its claim if wake notification registration fails", async () => {
+    const binding = await accepted(), wake = await claimedWake(binding);
+    await db.update(workspaceOperations).set({ status: "failed" }).where(eq(workspaceOperations.heartbeatRunId, binding.runId));
+    const fail = () => recordNativeFinalizationFailure({ db, runId: binding.runId,
+      projectRunStatus: true, failureScope: "workspace" as const, error: new Error("native_workspace_finalize_failed") });
+    await fail(); await fail();
+    const beforeRun = await read(binding.runId), beforeCoordinator = await state(binding.runId);
+    const notify = wakeSettlementNotifications.notifyExternalConversationSettlement;
+    vi.spyOn(wakeSettlementNotifications, "notifyExternalConversationSettlement").mockImplementation(async (tx, scope) => {
+      const [current] = await tx.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, wake.id));
+      expect(current.status).toBe("claimed");
+      await notify(tx, scope);
+      throw new Error("exhaustion wake notification failure");
+    });
+    await expect(fail()).rejects.toThrow("exhaustion wake notification failure");
+    expect(await read(binding.runId)).toEqual(beforeRun);
+    expect(await state(binding.runId)).toEqual(beforeCoordinator);
+    expect(await readWake(wake.id)).toEqual(wake);
+  });
+
+  it.each(["succeeded", "failed", "cancelled"] as const)("discovers and repairs an old %s run's claimed wake once", async status => {
+    const binding = await accepted(status);
+    await finalizeNativeRun({ db, runId: binding.runId, workspaceFinalizeStatus: "succeeded", projectRunStatus: true });
+    const run = await read(binding.runId), wake = await claimedWake(binding);
+    const work = nativeRunProjectionWork(db);
+    const [entry] = await work.load({ companyId, runId: binding.runId });
+    expect(entry).toMatchObject({ id: binding.runId });
+    await work.check(entry);
+    expect(await readWake(wake.id)).toMatchObject({ status: status === "succeeded" ? "completed" : status,
+      finishedAt: run.finishedAt });
+    expect((await read(binding.runId)).finishedAt).toEqual(run.finishedAt);
+    expect(await work.load({ companyId, runId: binding.runId })).toEqual([]);
+  });
+
+  it.each(["unbound", "newer-run", "queued", "cancelled", "foreign-company", "foreign-agent", "live-owner"] as const)(
+    "does not settle a %s wake claim", async fault => {
+      const binding = await accepted(), wake = await claimedWake(binding);
+      await finalizeNativeRun({ db, runId: binding.runId, workspaceFinalizeStatus: "succeeded" });
+      if (fault === "unbound" || fault === "newer-run") await db.update(agentWakeupRequests)
+        .set({ runId: fault === "unbound" ? null : randomUUID() }).where(eq(agentWakeupRequests.id, wake.id));
+      if (fault === "queued" || fault === "cancelled") await db.update(agentWakeupRequests)
+        .set({ status: fault }).where(eq(agentWakeupRequests.id, wake.id));
+      if (fault === "foreign-company") {
+        const otherCompany = randomUUID();
+        await db.insert(companies).values({ id: otherCompany, name: "Other wake owner", issuePrefix: `OW${randomUUID().slice(0, 6)}` });
+        await db.update(agentWakeupRequests).set({ companyId: otherCompany }).where(eq(agentWakeupRequests.id, wake.id));
+      }
+      if (fault === "foreign-agent") {
+        const otherAgent = randomUUID();
+        await db.insert(agents).values({ id: otherAgent, companyId, name: "Other wake owner", adapterType: "paperclip_runner" });
+        await db.update(agentWakeupRequests).set({ agentId: otherAgent }).where(eq(agentWakeupRequests.id, wake.id));
+      }
+      if (fault === "live-owner") await db.update(heartbeatRuns).set({ status: "running", nativePhase: "terminal_failure",
+        errorCode: "native_execution_ownership_unverified" }).where(eq(heartbeatRuns.id, binding.runId));
+      const before = await readWake(wake.id);
+      await projectCommittedRun({ db, run: await read(binding.runId), coordinator: await state(binding.runId) });
+      expect(await readWake(wake.id)).toEqual(before);
+      if (fault === "live-owner") expect((await read(binding.runId)).status).toBe("running");
+    },
+  );
+
+  it("registers wake settlement intent before mutation and rolls back notification failure", async () => {
+    const binding = await accepted(), wake = await claimedWake(binding);
+    await finalizeNativeRun({ db, runId: binding.runId, workspaceFinalizeStatus: "succeeded" });
+    const before = await read(binding.runId);
+    const notify = wakeSettlementNotifications.notifyExternalConversationSettlement;
+    const notification = vi.spyOn(wakeSettlementNotifications, "notifyExternalConversationSettlement")
+      .mockImplementation(async (tx, scope) => {
+        const [current] = await tx.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, wake.id));
+        expect(current.status).toBe("claimed");
+        await notify(tx, scope);
+        throw new Error("wake intent registration failure");
+      });
+    await expect(projectCommittedRun({ db, run: before, coordinator: await state(binding.runId) }))
+      .rejects.toThrow("wake intent registration failure");
+    expect(notification).toHaveBeenCalledOnce();
+    expect(await read(binding.runId)).toEqual(before);
+    expect(await readWake(wake.id)).toEqual(wake);
+  });
+
+  it("rolls back terminal projection and exact wake settlement together", async () => {
+    const binding = await accepted(), wake = await claimedWake(binding);
+    await finalizeNativeRun({ db, runId: binding.runId, workspaceFinalizeStatus: "succeeded" });
+    const before = await read(binding.runId);
+    await expect(db.transaction(async tx => {
+      await projectCommittedRun({ db: tx as unknown as Db, run: before, coordinator: await state(binding.runId) });
+      throw new Error("projection rollback fixture");
+    })).rejects.toThrow("projection rollback fixture");
+    expect(await read(binding.runId)).toEqual(before);
+    expect(await readWake(wake.id)).toEqual(wake);
+  });
+
   it("reconstructs a committed decision with an unfinished run projection and stays quiet after repair", async () => {
     const binding = await accepted();
     await finalizeNativeRun({ db, runId: binding.runId, workspaceFinalizeStatus: "succeeded" });