fix: wake native response recovery from committed producers

Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
DottaandPaperclip committed 2026-10-25 23:59:18 -05:00
1 parent 85c94ef85d
commit e9e2a7feb9
27 files changed
+390 -63

No files matched your search

@@ -15,7 +15,7 @@ import { accessService } from "../services/access.js";
import { signalDatabaseWork, subscribeDatabaseWork } from "@paperclipai/db";
import { restoreNativeChatReviewPresentationInTransaction } from "../services/native-runtime/native-chat-review-presentation.js";
import { notifyChatEndpointConfiguration, notifyChatCompanyConfiguration } from "../services/chat-channels/configuration-work-notifications.js";
import { configureAgentLifecycle } from "../services/agent-lifecycle.js";
import { configureAgentLifecycle, createAgentLifecycle, processAgentLifecycle } from "../services/agent-lifecycle.js";
import { deleteCompany } from "../services/company-deletion.js";
import { idleWorkSnapshot } from "../services/task-admission.js";
import { readFreshWorkspaceReadiness } from "../services/fresh-workspace-continuation.js";
@@ -63993,6 +63993,19 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => {
const input = { companyId: context.fixture.companyId, issueId: context.issue.id,
agentId: context.fixture.assignedAgentId, failedRunId: context.runId, initiatedByUserId: "owner-user" };
const stage = () => db.transaction(tx => context.service.prepareFailedChatRunRetry(tx, input));
const observedResponseAdmissions: string[] = [];
const responseHints: string[] = [];
await coordinator.register(DELIVERY_QUEUES.nativeResponsePresentation, {
retryMs: 30_000,
onSettled: hints => { responseHints.push(...hints ?? []); },
run: async () => {
const admitted = await db.select({ id: chatActions.id }).from(chatActions).where(and(
eq(chatActions.companyId, context.fixture.companyId), eq(chatActions.kind, "failed_run_retry"), eq(chatActions.status, "processed"),
));
observedResponseAdmissions.push(...admitted.map(row => row.id));
},
hasPending: async () => false,
}).ready;
context.wakeup.mockClear();
try {
let actionId: string;
@@ -64038,6 +64051,8 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => {
expect(action?.result.attemptCount).toBe(mode === "retry" ? 2 : 1);
expect(coordinator.nextWakeAt()).toBeNull();
}, { timeout: 10_000 });
await vi.waitFor(() => expect(observedResponseAdmissions).toContain(actionId), { timeout: 5000 });
expect(responseHints.map(hint => JSON.parse(hint))).toContainEqual({ companyId: context.fixture.companyId, issueId: context.issue.id });
expect(context.wakeup).toHaveBeenCalledTimes(mode === "retry" ? 2 : 1);
const receipts = await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, actionId));
expect(receipts).toHaveLength(1);
@@ -72702,6 +72717,49 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => {
} finally { worker.stop(); await coordinator.stop(); vi.useRealTimers(); await context.service.shutdown(); }
});
it("retains a failed accepted-response attempt across drain and retries after resume", async () => {
const context = await committedChatResponseRecoveryFixture("telegram");
let release!: () => void, observed!: () => void;
const hold = new Promise<void>(resolve => { release = resolve; });
const locked = new Promise<void>(resolve => { observed = resolve; });
const holder = db.transaction(async tx => {
await tx.select().from(chatActions).where(eq(chatActions.id, context.action.id)).for("update");
observed();
await hold;
});
await locked;
let canRun = true;
vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"], shouldAdvanceTime: true });
const errors: unknown[] = [];
const coordinator = createDeliveryWorkCoordinator({ owner: db, canRun: () => canRun, canReconcile: () => true, onError: error => errors.push(error) });
const worker = nativeResponsePresentationWork(db).register(coordinator, () => canRun);
try {
await worker.ready;
await vi.waitFor(async () => {
await vi.advanceTimersByTimeAsync(1000);
expect(errors).toHaveLength(1);
expect(idleWorkSnapshot().active).toBe(0);
}, { timeout: 5000 });
expect(coordinator.nextWakeAt()).toBeGreaterThan(Date.now());
expect(await readChatPublicationsByEndpointId(context.endpoint.id)).toHaveLength(0);
canRun = false;
const reads = vi.spyOn(db, "select"), writes = vi.spyOn(db, "transaction");
try {
await vi.advanceTimersByTimeAsync(120_000);
expect(reads).not.toHaveBeenCalled(); expect(writes).not.toHaveBeenCalled();
} finally { reads.mockRestore(); writes.mockRestore(); }
release(); await holder;
canRun = true; worker.wake();
await vi.waitFor(async () => {
await vi.advanceTimersByTimeAsync(1000);
expect(await readChatPublicationsByEndpointId(context.endpoint.id)).toHaveLength(1);
expect(idleWorkSnapshot().active).toBe(0);
expect(coordinator.nextWakeAt()).toBeNull();
}, { timeout: 5000 });
expect(errors).toHaveLength(1);
} finally { release(); await holder; worker.stop(); await coordinator.stop(); vi.useRealTimers(); await context.service.shutdown(); }
});
it("reconstructs a future accepted-response deadline without early publication", async () => {
const context = await committedChatResponseRecoveryFixture("telegram");
const at = new Date(Date.now() + 60_000);
@@ -72750,6 +72808,88 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => {
} finally { intercept.mockRestore(); vi.useRealTimers(); await context.service.shutdown(); }
});
it("wakes dormant accepted-response recovery after agent resume", async () => {
const context = await committedChatResponseRecoveryFixture("telegram");
await db.update(agents).set({ status: "paused", lifecycleState: "paused", lifecycleHolds: ["manual"], lifecycleOperation: null })
.where(eq(agents.id, context.binding.agentId));
const errors: unknown[] = [];
const coordinator = createDeliveryWorkCoordinator({ owner: db, canRun: () => true, onError: error => errors.push(error) });
const work = nativeResponsePresentationWork(db);
const worker = work.register(coordinator, () => true);
const lifecycle = configureAgentLifecycle(db, {
requiredPluginIds: async () => [], runHost: async () => "complete", runPlugin: async () => "complete",
});
try {
await worker.ready;
expect(coordinator.nextWakeAt()).toBeNull();
expect(await work.load({ companyId: context.fixture.companyId, agentId: randomUUID() })).toEqual([]);
expect(await readChatPublicationsByEndpointId(context.endpoint.id)).toHaveLength(0);
await createAgentLifecycle(db).resumeAgent(context.binding.agentId);
await processAgentLifecycle(db, context.binding.agentId);
await expect.poll(async () => (await db.select().from(agents).where(eq(agents.id, context.binding.agentId)))[0].status).toBe("idle");
await expect.poll(async () => (await readChatPublicationsByEndpointId(context.endpoint.id)).length, { timeout: 5000 }).toBe(1);
await expect.poll(() => coordinator.nextWakeAt(), { timeout: 5000 }).toBeNull();
expect(errors).toEqual([]);
} finally { await lifecycle.stop(); worker.stop(); await coordinator.stop(); await context.service.shutdown(); }
});
it("wakes accepted-response recovery from the real coordinator retry release", async () => {
const context = await committedChatResponseRecoveryFixture("telegram");
const store = new NativeRunCoordinatorStore(db, context.binding);
await store.claimFinalization({ leaseOwner: "response-recovery-fixture", leaseTtlMs: 1000 });
vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"], shouldAdvanceTime: true });
const errors: unknown[] = [];
const coordinator = createDeliveryWorkCoordinator({ owner: db, canRun: () => true, onError: error => errors.push(error) });
const worker = nativeResponsePresentationWork(db).register(coordinator, () => true);
try {
await worker.ready;
expect(coordinator.nextWakeAt()).toBeNull();
await vi.advanceTimersByTimeAsync(2000);
expect(await readChatPublicationsByEndpointId(context.endpoint.id)).toHaveLength(0);
expect(coordinator.nextWakeAt()).toBeNull();
await store.markFinalizationRetry({ leaseOwner: "response-recovery-fixture", failureCode: "response_fixture_retry", retryAfterMs: 1000 });
await vi.waitFor(async () => {
await vi.advanceTimersByTimeAsync(1000);
expect(await readChatPublicationsByEndpointId(context.endpoint.id)).toHaveLength(1);
expect(idleWorkSnapshot().active).toBe(0);
expect(coordinator.nextWakeAt()).toBeNull();
}, { timeout: 5000 });
expect(errors).toEqual([]);
} finally { worker.stop(); await coordinator.stop(); vi.useRealTimers(); await context.service.shutdown(); }
});
it("wakes accepted-response recovery when its authenticated accepted-result journal arrives late", async () => {
const context = await committedChatResponseRecoveryFixture("telegram", "port");
await db.delete(heartbeatRunEvents).where(and(eq(heartbeatRunEvents.runId, context.runId), eq(heartbeatRunEvents.eventType, "run.result.accepted")));
const errors: unknown[] = [];
const coordinator = createDeliveryWorkCoordinator({ owner: db, canRun: () => true, onError: error => errors.push(error) });
const worker = nativeResponsePresentationWork(db).register(coordinator, () => true);
try {
await worker.ready;
expect(coordinator.nextWakeAt()).toBeNull();
expect(await readChatPublicationsByEndpointId(context.endpoint.id)).toHaveLength(0);
const port = new PaperclipControlPlanePort(db, {
companyId: context.binding.companyId, issueId: context.binding.issueId,
runId: context.runId, agentId: context.binding.agentId, sessionId: context.binding.normalizedSessionId,
completionContractId: context.binding.completionContractId,
completionContractSha256: context.binding.completionContractSha256,
sourceInstanceId: context.binding.runnerSourceInstanceId, controlPlaneSourceInstanceId: `control-${context.runId}`,
});
await port.appendEvent({
schema: "paperclip.prp.event.v1", sourceEventId: `accepted-${context.runId}`, sourceSeq: 1,
sourceInstanceId: `control-${context.runId}`, sourceKind: "control_plane",
runId: context.runId, normalizedSessionId: context.binding.normalizedSessionId,
turnId: `turn-${context.runId}`, eventType: "run.result.accepted", schemaVersion: 1,
priority: 0, emittedAt: new Date().toISOString(), payload: { result: context.result },
});
await expect.poll(async () => ({
publications: (await readChatPublicationsByEndpointId(context.endpoint.id)).length,
active: idleWorkSnapshot().active, next: coordinator.nextWakeAt(),
}), { timeout: 5000 }).toEqual({ publications: 1, active: 0, next: null });
expect(errors).toEqual([]);
} finally { worker.stop(); await coordinator.stop(); await context.service.shutdown(); }
});
it("defers scheduled accepted-response recovery until the persisted retry is due", async () => {
const context = await committedChatResponseRecoveryFixture("telegram");
const input = { companyId: context.fixture.companyId, issueId: context.issue.id, runId: context.runId, onlyWhenDue: true };
@@ -1,3 +1,4 @@
import { isNativeResponseEvidence } from "../services/native-response-presentation-notifications.js";
import { describe, expect, it, vi } from "vitest";
import type { Db } from "@paperclipai/db";
import { authorizeCommittedChatResponse, onCommittedChatResponseAuthorityRegistered, registerCommittedChatResponseAuthority } from "../services/durable-chat-wakeup.js";
@@ -39,3 +40,16 @@ describe("committed response authority lifecycle", () => {
} finally { removeSecond(); }
});
});
describe("accepted response evidence selection", () => {
it.each([
{ type: "item.delta", kind: "agentMessage", channel: "final", expected: false },
{ type: "item.completed", kind: "agentMessage", channel: "analysis", expected: false },
{ type: "item.completed", kind: "commandExecution", channel: "final", expected: false },
{ type: "item.completed", kind: "agentMessage", channel: "final", expected: true },
{ type: "run.result.accepted", kind: "agentMessage", channel: "final", expected: true },
])("signals only response evidence: $type/$kind/$channel", ({ type, kind, channel, expected }) => {
expect(isNativeResponseEvidence(type, { payload: { kind, channel, text: "Saved evidence" } })).toBe(expected);
});
});
@@ -1,3 +1,4 @@
import { resolveExternalTestDatabaseUrl } from "../../../__tests__/helpers/external-test-database.js";
import { createAgentLifecycle as createLifecycleCommands } from "../index.js";
import { createAgentLifecycleEffects } from "../../../services/agent-lifecycle.js";
import { deleteCompany } from "../../../services/company-deletion.js";
@@ -28,13 +29,20 @@ import { budgetService } from "../../../services/budgets.js";
const createLifecycleStore = (db: Db) => createStore(db, createAgentLifecycleEffects());
const support = await getEmbeddedPostgresTestSupport();
const external = resolveExternalTestDatabaseUrl("PAPERCLIP_AGENT_LIFECYCLE_TEST_DATABASE_URL");
const support = external ? { supported: true } : await getEmbeddedPostgresTestSupport();
(support.supported ? describe : describe.skip)("agent lifecycle commands", () => {
let database: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>>;
let database: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>> | undefined;
let db: Db;
let companyId: string;
const workers: ReturnType<typeof configureAgentLifecycle>[] = [];
beforeAll(async () => { database = await startEmbeddedPostgresTestDatabase("paperclip-agent-lifecycle-"); db = createDb(database.connectionString); }, 90_000);
beforeAll(async () => {
if (external) db = createDb(external);
else {
database = await startEmbeddedPostgresTestDatabase("paperclip-agent-lifecycle-");
db = createDb(database.connectionString);
}
}, 90_000);
afterAll(async () => { for (const worker of workers) await worker.stop(); await database?.cleanup(); });
beforeEach(async () => {
for (const work of workers) await work.stop();
@@ -7,6 +7,7 @@ import type { ActivityPublication } from "../../../types/activity-publication.js
/** Services supply integrations; lifecycle owns the transaction and state writes. */
export interface LifecycleEffects {
notifyWork(db: Db): Promise<void>;
beforeStatusChange(db: Db, companyId: string, agentId: string, before: string, after: string): Promise<void>;
reportFailure(context: LifecycleFailure, error: unknown): void;
deleteDependencies(db: Db, companyId: string, agentId: string): Promise<void>;
prepareConfiguration(db: Db, current: AgentConfigurationRecord, data: AgentConfigurationPatch, options?: UpdateAgentOptions): Promise<AgentConfigurationPatch>;
@@ -71,6 +71,7 @@ async function changeLockedAgent(tx: Db, effects: LifecycleEffects, agent: Lifec
};
if (next === "ready") operation.resumeState = "ready";
const status = compatibilityStatus(next, agent.status);
if (status !== agent.status) await effects.beforeStatusChange(tx, agent.companyId, agent.id, agent.status, status);
if (isLifecycleWorkPending(next)) await effects.notifyWork(tx);
const [updated] = await tx.update(agents).set({
lifecycleHolds: holds, lifecycleState: next, lifecycleVersion: agent.lifecycleVersion + 1, lifecycleError: null,
+7
View File
@@ -1,3 +1,5 @@
import { isAgentStatusInvokable } from "@paperclipai/shared";
import { notifyNativeResponsePresentation } from "./native-response-presentation-notifications.js";
import { logger } from "../middleware/logger.js";
import type { Db } from "@paperclipai/db";
import * as lifecycle from "../modules/agent-lifecycle/index.js";
@@ -19,6 +21,11 @@ export type { LifecycleDriver, LifecycleAgent } from "../modules/agent-lifecycle
export function createAgentLifecycleEffects(_hooks: BudgetServiceHooks = {}): lifecycle.LifecycleEffects {
return {
notifyWork: tx => notifyDeliveryWork(tx, DELIVERY_QUEUES.agentLifecycle),
async beforeStatusChange(tx, companyId, agentId, before, after) {
if (!isAgentStatusInvokable(before) && isAgentStatusInvokable(after)) {
await notifyNativeResponsePresentation(tx, { companyId, agentId });
}
},
reportFailure(context, error) {
const code = lifecycle.lifecycleFailureCode(error);
logger.warn({ ...context, failureCode: code ?? "unclassified" }, "Agent lifecycle operation failed");
@@ -1,3 +1,4 @@
import { notifyNativeResponsePresentation } from "../native-response-presentation-notifications.js";
import { notifyChatInboundWakeNotices } from "../chat-inbound-wake-notice-notifications.js";
import { createFailedRetryNotices } from "./failed-retry-notices.js";
import { continuingTeamsFilePublication, type ContinuingTeamsFileTransfer } from "./microsoft-teams/file-transfer-source.js";
@@ -1564,6 +1565,9 @@ export function createChatFailedRunRetry(db: Db, options: ChatFailedRunRetryOpti
const settle = async (status: string, result: Record<string, unknown>) => {
const [current] = await db.transaction(async (tx) => {
await notifyChatActionWork(tx, "failed_run_retry");
if (status === "processed") await notifyNativeResponsePresentation(tx, {
companyId: claimed.companyId, issueId: String(claimed.payload.issueId),
});
return tx
.update(chatActions)
.set({ status, result, updatedAt: new Date() })
@@ -1,5 +1,7 @@
import { nativeResponsePresentationWork } from "./native-runtime/native-response-presentation-work.js";
import { resolveExternalTestDatabaseUrl } from "../__tests__/helpers/external-test-database.js";
import { randomUUID } from "node:crypto";
import { and, eq } from "drizzle-orm";
import { and, eq, sql } from "drizzle-orm";
import { afterAll, beforeAll, describe, expect, it } from "vitest";
import {
agents, agentWakeupRequests, authUsers, chatConversations, chatEndpoints, chatMessageLinks,
@@ -16,14 +18,18 @@ import { createHeartbeatQueueTestDependencies } from "../__tests__/helpers/heart
import { createHeartbeatQueue } from "./heartbeat/queue.js";
import { instanceSettingsService } from "./instance-settings.js";
const support = await getEmbeddedPostgresTestSupport();
const external = resolveExternalTestDatabaseUrl("PAPERCLIP_MONITOR_CONTINUATION_TEST_DATABASE_URL");
const support = external ? { supported: true, reason: undefined } : await getEmbeddedPostgresTestSupport();
if (!support.supported) console.warn(`Monitor publication database unavailable: ${support.reason}`);
(support.supported ? describe : describe.skip)("monitor conversation continuation", () => {
let temporary: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>>;
let temporary: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>> | undefined;
let db: ReturnType<typeof createDb>;
beforeAll(async () => {
temporary = await startEmbeddedPostgresTestDatabase("paperclip-monitor-publications-");
db = createDb(temporary.connectionString);
if (external) db = createDb(external);
else {
temporary = await startEmbeddedPostgresTestDatabase("paperclip-monitor-publications-");
db = createDb(temporary.connectionString);
}
await db.insert(authUsers).values({ id: "requester", name: "Requester", email: "requester@example.test", createdAt: new Date(), updatedAt: new Date() });
});
afterAll(async () => { await temporary?.cleanup(); });
@@ -235,7 +241,7 @@ if (!support.supported) console.warn(`Monitor publication database unavailable:
expect(await f.bindings()).toEqual([]);
});
it("recovers the accepted continuation explanation after interrupted finalization without another provider result", async () => {
it.each([false, true])("recovers the accepted continuation explanation without another provider result (missing presentation: %s)", async missingPresentation => {
const f = await fixture();
const [contract] = await db.insert(completionContracts).values({ companyId: f.company.id, issueId: f.issue.id,
revision: 1, schemaVersion: "paperclip.completion-contract.v1", policyVersion: "test", risk: "standard",
@@ -272,6 +278,20 @@ if (!support.supported) console.warn(`Monitor publication database unavailable:
expect(settledRun).toMatchObject({ status: "succeeded", resultJson: { finalizationPhase: "committed" } });
const priorComments = await db.select({ body: issueComments.body }).from(issueComments).where(eq(issueComments.createdByRunId, f.source.id));
expect(priorComments).toEqual([{ body: summary }]);
expect(await nativeResponsePresentationWork(db).load({ companyId: f.company.id, runId: f.source.id })).toEqual([]);
if (missingPresentation) {
await db.transaction(async tx => {
await tx.delete(chatPublications).where(eq(chatPublications.issueId, f.issue.id));
await tx.delete(issueComments).where(eq(issueComments.createdByRunId, f.source.id));
await tx.update(heartbeatRuns).set({ resultJson: sql`${heartbeatRuns.resultJson} - 'presentationDecision'` })
.where(eq(heartbeatRuns.id, f.source.id));
});
const work = nativeResponsePresentationWork(db);
const entries = await work.load({ companyId: f.company.id, runId: f.source.id });
expect(entries).toHaveLength(1);
await expect(work.check(entries[0])).resolves.toBeNull();
expect(await work.load({ companyId: f.company.id, runId: f.source.id })).toEqual([]);
}
expect(await repair()).toBe(false);
expect(await repair()).toBe(false);
const comments = await db.select().from(issueComments).where(eq(issueComments.createdByRunId, f.source.id));
@@ -1,3 +1,4 @@
import { notifyNativeResponsePresentation } from "./native-response-presentation-notifications.js";
import { notifyChatConfirmationWork } from "./chat-confirmation-notifications.js";
import { notifyChatInboundWakeNotices } from "./chat-inbound-wake-notice-notifications.js";
import { notifyChatNativeProgress } from "./chat-native-progress-notifications.js";
@@ -8,7 +9,7 @@ import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notificatio
export type ChatRunMilestoneScope = { companyId: string; issueId?: string };
export async function notifyChatRunMilestones(tx: object, scope: ChatRunMilestoneScope) {
await notifyDeliveryWork(tx, DELIVERY_QUEUES.nativeResponsePresentation, JSON.stringify({ companyId: scope.companyId, issueId: scope.issueId }));
await notifyNativeResponsePresentation(tx, scope);
await notifyDeliveryWork(tx, DELIVERY_QUEUES.chatGitHubReviewChecks, JSON.stringify({ companyId: scope.companyId }));
await notifyDeliveryWork(tx, DELIVERY_QUEUES.chatGitHubTerminalResponses, JSON.stringify({ companyId: scope.companyId, issueId: scope.issueId }));
await notifyDeliveryWork(tx, DELIVERY_QUEUES.chatRunMilestones, JSON.stringify({ companyId: scope.companyId, issueId: scope.issueId }));
@@ -1,4 +1,4 @@
import { notifyNativePendingEffects } from "./native-pending-effect-notifications.js";
import { notifyNativeFinalizationWork } from "./native-finalization-notifications.js";
import { notifyUnsafeWorkspaceExportRunChange } from "./unsafe-workspace-export-notifications.js";
import { notifyCancelledWorkspacePreservation } from "./cancelled-workspace-preservation-notifications.js";
import { notifyInstructionCollection } from "./instruction-collection-notifications.js";
@@ -146,7 +146,7 @@ export async function reconcileAbandonedExecutionControl(
}
// A persisted result belongs to the existing finalizer, not a replacement provider.
if (coordinator?.resultId) {
await notifyNativePendingEffects(tx, { companyId: run.companyId, runId: run.id });
await notifyNativeFinalizationWork(tx, { companyId: run.companyId, runId: run.id });
await tx
.update(nativeRunFinalizations)
.set({
+8 -2
View File
@@ -1,3 +1,4 @@
import { isNativeResponseEvidence, notifyNativeResponsePresentation } from "./native-response-presentation-notifications.js";
import { isSafeNativeChatProgressEventType } from "./safe-native-chat-progress.js";
import { notifyChatNativeProgress } from "./chat-native-progress-notifications.js";
import { boundRunIssueIdSql } from "./run-task-binding.js";
@@ -72,7 +73,8 @@ export async function appendHeartbeatRunEvent(
input: AppendHeartbeatRunEventInput,
): Promise<AppendHeartbeatRunEventResult> {
const projectsProgress = isSafeNativeChatProgressEventType(input.eventType);
if (!input.nativeSource && !input.retryExhaustion && !projectsProgress) {
const projectsResponse = isNativeResponseEvidence(input.eventType, input.payload?.prpEvent);
if (!input.nativeSource && !input.retryExhaustion && !projectsProgress && !projectsResponse) {
const allocated = db.$with("allocated_run_event", {}).as(sql`
update ${heartbeatRuns}
set next_event_seq = ${heartbeatRuns.nextEventSeq} + 1,
@@ -101,7 +103,8 @@ export async function appendHeartbeatRunEvent(
.select({ companyId: heartbeatRuns.companyId, agentId: heartbeatRuns.agentId,
// Only progress needs the task binding. Other native/recovery events
// must not read a potentially large context snapshot to append a log.
...(projectsProgress ? { runtimeMode: heartbeatRuns.runtimeMode, issueId: boundRunIssueIdSql() } : {}) })
...(projectsProgress || projectsResponse ? { runtimeMode: heartbeatRuns.runtimeMode } : {}),
...(projectsProgress ? { issueId: boundRunIssueIdSql() } : {}) })
.from(heartbeatRuns)
.where(eq(heartbeatRuns.id, input.runId))
.for("update")
@@ -175,6 +178,9 @@ export async function appendHeartbeatRunEvent(
if (projectsProgress && run.runtimeMode === "native" && run.issueId) {
await notifyChatNativeProgress(tx, { companyId: run.companyId, issueId: run.issueId });
}
if (projectsResponse && run.runtimeMode === "native") {
await notifyNativeResponsePresentation(tx, { companyId: run.companyId, runId: input.runId });
}
const seq = await allocateHeartbeatRunEventSeq(
tx as unknown as Db,
input.runId,
@@ -0,0 +1,10 @@
import { notifyNativeResponsePresentation } from "./native-response-presentation-notifications.js";
import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js";
export type NativeFinalizationWorkScope = { companyId: string; runId?: string };
export async function notifyNativeFinalizationWork(transaction: object, scope: NativeFinalizationWorkScope) {
await notifyNativeResponsePresentation(transaction, scope);
await notifyDeliveryWork(transaction, DELIVERY_QUEUES.nativePendingEffects,
JSON.stringify({ companyId: scope.companyId, runId: scope.runId }));
}
@@ -1,8 +0,0 @@
import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js";
export type NativePendingEffectScope = { companyId: string; runId?: string };
export async function notifyNativePendingEffects(transaction: object, scope: NativePendingEffectScope) {
await notifyDeliveryWork(transaction, DELIVERY_QUEUES.nativePendingEffects,
JSON.stringify({ companyId: scope.companyId, runId: scope.runId }));
}
@@ -0,0 +1,14 @@
import { readCompletedAssistantMessageCandidate } from "./heartbeat-run-summary.js";
import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js";
export type NativeResponsePresentationScope = { companyId: string; issueId?: string; runId?: string; agentId?: string };
export async function notifyNativeResponsePresentation(tx: object, scope: NativeResponsePresentationScope) {
await notifyDeliveryWork(tx, DELIVERY_QUEUES.nativeResponsePresentation,
JSON.stringify({ companyId: scope.companyId, issueId: scope.issueId, runId: scope.runId, agentId: scope.agentId }));
}
export function isNativeResponseEvidence(eventType: string, prpEvent: unknown): boolean {
return eventType === "run.result.accepted" || eventType === "item.completed" &&
readCompletedAssistantMessageCandidate({ seq: 0, prpEvent })?.channel === "final";
}
@@ -1,4 +1,4 @@
import { notifyNativePendingEffects } from "../native-pending-effect-notifications.js";
import { notifyNativeFinalizationWork } from "../native-finalization-notifications.js";
import { notifyUnsafeWorkspaceExportRecovery } from "../unsafe-workspace-export-notifications.js";
import type { Db } from "@paperclipai/db";
import type { NativeInteractionResponseEnvelope } from "../../vendor/paperclip-runner/index.js";
@@ -633,7 +633,7 @@ export async function routePersistedNativeResultAttention(input: {
const delegation = effects.find((effect) => effect.kind === "create_delegated_issue");
const finalizationError = effects.find((effect) => effect.kind === "record_finalization_error");
await input.db.transaction(async tx => {
await notifyNativePendingEffects(tx, { companyId: binding.companyId, runId: input.runId });
await notifyNativeFinalizationWork(tx, { companyId: binding.companyId, runId: input.runId });
await tx.update(nativeRunFinalizations).set({
phase: finalizationError ? "retryable_failure" : "committed",
assessmentId: existingAssessment.id,
@@ -2,7 +2,7 @@ import { and, asc, eq, exists, gt, inArray, isNotNull, ne } from "drizzle-orm";
import { heartbeatRuns, issues, nativeRunFinalizations, statusDecisionEffects, type Db } from "@paperclipai/db";
import type { createDeliveryWorkCoordinator } from "../delivery-work-coordinator.js";
import { DELIVERY_QUEUES } from "../delivery-work-notifications.js";
import type { NativePendingEffectScope } from "../native-pending-effect-notifications.js";
import type { NativeFinalizationWorkScope } from "../native-finalization-notifications.js";
import { registerScopedDeadlineWork } from "../scoped-deadline-work.js";
import { dispatchPendingNativeStatusEffects, nativePendingEffectDueAt } from "./status-decision-committer.js";
@@ -15,7 +15,7 @@ export function pendingNativeStatusEffectsCondition(db: Db) {
)));
}
function findPendingEffects(db: Db, scope?: NativePendingEffectScope, after?: string) {
function findPendingEffects(db: Db, scope?: NativeFinalizationWorkScope, after?: string) {
return db.select({
id: nativeRunFinalizations.runId, companyId: nativeRunFinalizations.companyId,
issueId: nativeRunFinalizations.issueId, decisionId: nativeRunFinalizations.decisionId,
@@ -36,7 +36,7 @@ function findPendingEffects(db: Db, scope?: NativePendingEffectScope, after?: st
export function registerNativePendingEffectWork(db: Db, coordinator: ReturnType<typeof createDeliveryWorkCoordinator>, canRun: () => boolean) {
type Entry = { id: string; companyId: string; dueAt: number };
async function load(scope?: NativePendingEffectScope, signal?: AbortSignal): Promise<Entry[]> {
async function load(scope?: NativeFinalizationWorkScope, signal?: AbortSignal): Promise<Entry[]> {
const entries: Entry[] = [];
let after: string | undefined;
while (canRun() && !signal?.aborted) {
@@ -49,8 +49,8 @@ export function registerNativePendingEffectWork(db: Db, coordinator: ReturnType<
}
return registerScopedDeadlineWork(coordinator, {
queue: DELIVERY_QUEUES.nativePendingEffects, batchSize: 25,
decodeScope: (hint: string): NativePendingEffectScope | undefined => hint === "*" ? undefined : JSON.parse(hint),
matches: (entry: Entry, scope?: NativePendingEffectScope) => !scope || entry.companyId === scope.companyId && (!scope.runId || entry.id === scope.runId),
decodeScope: (hint: string): NativeFinalizationWorkScope | undefined => hint === "*" ? undefined : JSON.parse(hint),
matches: (entry: Entry, scope?: NativeFinalizationWorkScope) => !scope || entry.companyId === scope.companyId && (!scope.runId || entry.id === scope.runId),
load,
check: async (entry: Entry, signal?: AbortSignal) => {
if (!canRun() || signal?.aborted) throw new Error("Native pending effect recovery stopped");
@@ -2,17 +2,17 @@ import { and, asc, eq, gt, inArray, isNotNull, isNull, notExists, or, sql } from
import { heartbeatRuns, nativeRunFinalizations, nativeRunResults, statusDecisions, statusDecisionEffects, type Db } from "@paperclipai/db";
import type { createDeliveryWorkCoordinator } from "../delivery-work-coordinator.js";
import { DELIVERY_QUEUES } from "../delivery-work-notifications.js";
import type { ChatRunMilestoneScope } from "../chat-run-milestone-notifications.js";
import type { NativeResponsePresentationScope } from "../native-response-presentation-notifications.js";
import { onCommittedChatResponseAuthorityRegistered } from "../durable-chat-wakeup.js";
import { registerScopedDeadlineWork } from "../scoped-deadline-work.js";
import { recoverCommittedNativeChatResponse } from "./native-run-finalizer.js";
type Entry = { id: string; companyId: string; issueId: string; dueAt: number };
type Entry = { id: string; companyId: string; issueId: string; agentId: string; dueAt: number };
export function nativeResponsePresentationWork(db: Db) {
function candidates(scope?: ChatRunMilestoneScope, after?: string, runId?: string) {
function candidates(scope?: NativeResponsePresentationScope, after?: string, runId?: string) {
return db.select({ id: heartbeatRuns.id, companyId: heartbeatRuns.companyId,
issueId: nativeRunFinalizations.issueId, nextAttemptAt: nativeRunFinalizations.nextAttemptAt,
issueId: nativeRunFinalizations.issueId, agentId: heartbeatRuns.agentId, nextAttemptAt: nativeRunFinalizations.nextAttemptAt,
}).from(nativeRunFinalizations)
.innerJoin(heartbeatRuns, and(eq(heartbeatRuns.id, nativeRunFinalizations.runId),
eq(heartbeatRuns.companyId, nativeRunFinalizations.companyId), eq(heartbeatRuns.nativeIssueId, nativeRunFinalizations.issueId)))
@@ -46,15 +46,17 @@ export function nativeResponsePresentationWork(db: Db) {
inArray(statusDecisionEffects.deliveryState, ["pending", "failed"]),
))),
scope ? and(eq(heartbeatRuns.companyId, scope.companyId), scope.issueId ? eq(nativeRunFinalizations.issueId, scope.issueId) : undefined) : undefined,
scope?.runId ? eq(heartbeatRuns.id, scope.runId) : undefined,
scope?.agentId ? eq(heartbeatRuns.agentId, scope.agentId) : undefined,
after ? gt(heartbeatRuns.id, after) : undefined, runId ? eq(heartbeatRuns.id, runId) : undefined,
)).orderBy(asc(heartbeatRuns.id)).limit(100);
}
async function load(scope?: ChatRunMilestoneScope, signal?: AbortSignal, canRun = () => true): Promise<Entry[]> {
async function load(scope?: NativeResponsePresentationScope, signal?: AbortSignal, canRun = () => true): Promise<Entry[]> {
const entries: Entry[] = [];
let after: string | undefined;
while (canRun() && !signal?.aborted) {
const rows = await candidates(scope, after);
for (const row of rows) entries.push({ id: row.id, companyId: row.companyId, issueId: row.issueId, dueAt: row.nextAttemptAt?.getTime() ?? 0 });
for (const row of rows) entries.push({ id: row.id, companyId: row.companyId, issueId: row.issueId, agentId: row.agentId, dueAt: row.nextAttemptAt?.getTime() ?? 0 });
if (rows.length < 100) return entries;
after = rows[rows.length - 1].id;
}
@@ -76,8 +78,8 @@ export function nativeResponsePresentationWork(db: Db) {
function register(coordinator: ReturnType<typeof createDeliveryWorkCoordinator>, canRun: () => boolean) {
const worker = registerScopedDeadlineWork(coordinator, {
queue: DELIVERY_QUEUES.nativeResponsePresentation,
decodeScope: (hint: string): ChatRunMilestoneScope | undefined => hint === "*" ? undefined : JSON.parse(hint),
matches: (entry: Entry, scope?: ChatRunMilestoneScope) => !scope || entry.companyId === scope.companyId && (!scope.issueId || entry.issueId === scope.issueId),
decodeScope: (hint: string): NativeResponsePresentationScope | undefined => hint === "*" ? undefined : JSON.parse(hint),
matches: (entry: Entry, scope?: NativeResponsePresentationScope) => !scope || entry.companyId === scope.companyId && (!scope.issueId || entry.issueId === scope.issueId) && (!scope.runId || entry.id === scope.runId) && (!scope.agentId || entry.agentId === scope.agentId),
load: (scope, signal) => load(scope, signal, canRun),
check: (entry, signal) => {
if (!canRun()) throw new Error("Native response presentation recovery paused");
@@ -1,4 +1,5 @@
import { notifyNativePendingEffects } from "../native-pending-effect-notifications.js";
import { isNativeResponseEvidence, notifyNativeResponsePresentation } from "../native-response-presentation-notifications.js";
import { notifyNativeFinalizationWork } from "../native-finalization-notifications.js";
import { notifyUnsafeWorkspaceExportRecovery } from "../unsafe-workspace-export-notifications.js";
import { isSafeNativeChatProgressEventType } from "../safe-native-chat-progress.js";
import { notifyChatNativeProgress } from "../chat-native-progress-notifications.js";
@@ -297,6 +298,9 @@ export class NativeRunCoordinatorStore {
if (isSafeNativeChatProgressEventType(event.eventType)) {
await notifyChatNativeProgress(tx, { companyId: this.#binding.companyId, issueId: this.#binding.issueId });
}
if (isNativeResponseEvidence(event.eventType, canonicalPayload)) {
await notifyNativeResponsePresentation(tx, this.#binding);
}
const cursor = run.nextEventSeq;
const [inserted] = await tx
.insert(heartbeatRunEvents)
@@ -523,7 +527,7 @@ export class NativeRunCoordinatorStore {
) {
throw new Error("native_finalization_lease_conflict");
}
if (row.decisionId) await notifyNativePendingEffects(tx, this.#binding);
if (row.decisionId) await notifyNativeFinalizationWork(tx, this.#binding);
if (
row.leaseOwner === input.leaseOwner &&
row.leaseExpiresAt &&
@@ -566,7 +570,7 @@ export class NativeRunCoordinatorStore {
throw new Error("native_finalization_retry_invalid");
}
const [updated] = await this.#db.transaction(async tx => {
await notifyNativePendingEffects(tx, this.#binding);
await notifyNativeFinalizationWork(tx, this.#binding);
if (input.failureCode === "native_workspace_sync_out_unsafe_archive") {
await notifyUnsafeWorkspaceExportRecovery(tx, this.#binding);
}
@@ -1,4 +1,4 @@
import { notifyNativePendingEffects } from "../native-pending-effect-notifications.js";
import { notifyNativeFinalizationWork } from "../native-finalization-notifications.js";
import { notifyUnsafeWorkspaceExportRunChange } from "../unsafe-workspace-export-notifications.js";
import { notifyUnsafeWorkspaceExportRecovery } from "../unsafe-workspace-export-notifications.js";
import { notifyCancelledWorkspacePreservation } from "../cancelled-workspace-preservation-notifications.js";
@@ -214,7 +214,7 @@ async function claimCoordinator(input: {
coordinator.leaseOwner !== leaseOwner
)
throw new Error("native_finalization_lease_busy");
if (coordinator.decisionId) await notifyNativePendingEffects(tx, coordinator);
if (coordinator.decisionId) await notifyNativeFinalizationWork(tx, coordinator);
const [updated] = await tx
.update(nativeRunFinalizations)
.set({
@@ -382,7 +382,7 @@ async function recordRetryableFailure(input: {
? "native_workspace_sync_out_retry_exhausted"
: "native_finalization_retry_exhausted"
: input.failureCode;
await notifyNativePendingEffects(tx, { companyId: input.run.companyId, runId: input.run.id });
await notifyNativeFinalizationWork(tx, { companyId: input.run.companyId, runId: input.run.id });
if (phase === "terminal_failure") await notifyDeliveryWork(tx, DELIVERY_QUEUES.accounting, input.run.companyId);
if (phase === "terminal_failure" && failureCode === "native_workspace_sync_out_unsafe_archive") {
await notifyUnsafeWorkspaceExportRecovery(tx, { companyId: input.run.companyId, issueId: input.coordinator.issueId });
@@ -1,4 +1,4 @@
import { notifyNativePendingEffects } from "../native-pending-effect-notifications.js";
import { notifyNativeFinalizationWork } from "../native-finalization-notifications.js";
import { notifyUnsafeWorkspaceExportRecovery } from "../unsafe-workspace-export-notifications.js";
import { notifyCancelledWorkspacePreservation } from "../cancelled-workspace-preservation-notifications.js";
import { notifyInstructionCollection } from "../instruction-collection-notifications.js";
@@ -3814,7 +3814,7 @@ export async function reconcileRetainedNativeSessionCleanup(
renameSync(owned.root, archive);
}
renameSync(copy, owned.root);
await notifyNativePendingEffects(tx, { companyId: owned.run.companyId, runId: owned.run.id });
await notifyNativeFinalizationWork(tx, { companyId: owned.run.companyId, runId: owned.run.id });
await tx
.update(nativeRunFinalizations)
.set({
@@ -3872,7 +3872,7 @@ export async function reconcileRetainedNativeSessionCleanup(
.limit(1)
.then((rows) => rows[0]);
if (!current || current.leaseOwner !== leaseOwner) return;
await notifyNativePendingEffects(tx, { companyId: owned.run.companyId, runId: owned.run.id });
await notifyNativeFinalizationWork(tx, { companyId: owned.run.companyId, runId: owned.run.id });
await tx
.update(nativeRunFinalizations)
.set({
@@ -9477,7 +9477,7 @@ async function executePaperclipNativeSessionWithinScope(
)
.for("update");
await notifyExecutionRunWork(tx);
await notifyNativePendingEffects(tx, input.execution.binding);
await notifyNativeFinalizationWork(tx, input.execution.binding);
const updated = await tx
.update(nativeRunFinalizations)
.set({
@@ -9813,7 +9813,7 @@ async function executePaperclipNativeSessionWithinScope(
}
const releaseNow = new Date();
const released = await timeController("coordinator_release", () => input.db.transaction(async tx => {
await notifyNativePendingEffects(tx, input.execution.binding);
await notifyNativeFinalizationWork(tx, input.execution.binding);
return tx.update(nativeRunFinalizations)
.set({
leaseOwner: null,
@@ -1,4 +1,4 @@
import { notifyNativePendingEffects } from "../native-pending-effect-notifications.js";
import { notifyNativeFinalizationWork } from "../native-finalization-notifications.js";
import { alias } from "drizzle-orm/pg-core";
import type { UnsafeWorkspaceExportScope } from "../unsafe-workspace-export-notifications.js";
import type { createDeliveryWorkCoordinator } from "../delivery-work-coordinator.js";
@@ -96,7 +96,7 @@ async function recoverCandidate(db: Db, candidate: { runId: string; companyId: s
phase: "workspace_finalize", status: "succeeded", exitCode: 0, finishedAt: now,
metadata: { owningService: "native_workspace_finalizer", workspaceSync: { omitted: true, reason: "restore_unsafe_archive", legacy: true } },
});
await notifyNativePendingEffects(tx, { companyId: run.companyId, runId: run.id });
await notifyNativeFinalizationWork(tx, { companyId: run.companyId, runId: run.id });
await tx.update(nativeRunFinalizations).set({ phase: "result_accepted", failureCode: null, nextAttemptAt: null,
failureDetail: { workspaceFinalizeAttempt: 0, legacyUnsafeExportOmitted: { resultId: result.id, recoveredAt: now.toISOString() } }, updatedAt: now,
}).where(eq(nativeRunFinalizations.runId, run.id));
@@ -1,4 +1,4 @@
import { notifyNativePendingEffects } from "../native-pending-effect-notifications.js";
import { notifyNativeFinalizationWork } from "../native-finalization-notifications.js";
import { notifyEnvironmentCleanup } from "../environment-cleanup-notifications.js";
import { randomUUID } from "node:crypto";
import type { Environment, EnvironmentLease } from "@paperclipai/shared";
@@ -135,7 +135,7 @@ export async function retryNativeWorkspaceExport(input: {
const now = new Date();
const retry = { ...receipt, requestId: randomUUID(), actorId: input.actorId, repairNote: input.repairNote, requestedAt: now.toISOString(),
stoppedProvider: initial.lease.metadata?.remoteExecutionTermination };
await notifyNativePendingEffects(tx, { companyId: current.run.companyId, runId: current.run.id });
await notifyNativeFinalizationWork(tx, { companyId: current.run.companyId, runId: current.run.id });
await tx.update(nativeRunFinalizations).set({ phase: "result_accepted", failureCode: null, nextAttemptAt: null,
failureDetail: { ...current.coordinator.failureDetail, workspaceFinalizeAttempt: 0, workspaceExportRetry: retry }, updatedAt: now }).where(eq(nativeRunFinalizations.runId, current.run.id));
await tx.update(heartbeatRuns).set({ status: "running", finishedAt: null, error: null, errorCode: null,
@@ -1,3 +1,5 @@
import { nativeResponsePresentationWork } from "./native-response-presentation-work.js";
import { resolveExternalTestDatabaseUrl } from "../../__tests__/helpers/external-test-database.js";
import { afterAll, beforeAll, describe, expect, it } from "vitest";
import { asc, eq, sql } from "drizzle-orm";
import { generateKeyPairSync, randomUUID } from "node:crypto";
@@ -61,6 +63,8 @@ import {
subscribeCompanyLiveEvents,
} from "../live-events.js";
const external = resolveExternalTestDatabaseUrl("PAPERCLIP_CONTROL_PLANE_TEST_DATABASE_URL");
describe("PaperclipControlPlanePort conformance", () => {
let temporary: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>> | null = null;
let db: ReturnType<typeof createDb>;
@@ -83,8 +87,11 @@ describe("PaperclipControlPlanePort conformance", () => {
const governanceRunnerId = "00000000-0000-4000-8000-000000000021";
beforeAll(async () => {
temporary = await startEmbeddedPostgresTestDatabase("paperclip-native-port-");
db = createDb(temporary.connectionString);
if (external) db = createDb(external);
else {
temporary = await startEmbeddedPostgresTestDatabase("paperclip-native-port-");
db = createDb(temporary.connectionString);
}
const identity = CONTROL_PLANE_CONFORMANCE_OPEN.identity;
await db.insert(companies).values({ id: identity.companyId, name: "Phase 6", issuePrefix: "P6C" });
await db.insert(agents).values({
@@ -609,7 +616,7 @@ describe("PaperclipControlPlanePort conformance", () => {
completionContractSha256: localContractSha,
contextSnapshot: { issueId },
});
const observerDb = createDb(temporary!.connectionString);
const observerDb = createDb(external ?? temporary!.connectionString);
const observed: Array<Record<string, unknown>> = [];
const committedCallbacks: string[] = [];
let visibilityCheck: Promise<Array<{ eventType: string }>> | null = null;
@@ -1054,7 +1061,10 @@ describe("PaperclipControlPlanePort conformance", () => {
.where(eq(heartbeatRuns.id, taskRunId));
expect(await repair()).toBe(false);
await db.update(heartbeatRuns).set({ contextSnapshot: {} }).where(eq(heartbeatRuns.id, taskRunId));
expect(await repair()).toBe(true);
const work = nativeResponsePresentationWork(db);
const [entry] = await work.load({ companyId: identity.companyId, runId: taskRunId });
expect(entry?.id).toBe(taskRunId);
expect(await work.check(entry)).toBeNull();
expect(await repair()).toBe(false);
expect(await db.select({ body: issueComments.body }).from(issueComments)
.where(eq(issueComments.createdByRunId, taskRunId)).orderBy(asc(issueComments.createdAt)))
@@ -1,4 +1,4 @@
import { notifyNativePendingEffects } from "../native-pending-effect-notifications.js";
import { notifyNativeFinalizationWork } from "../native-finalization-notifications.js";
import { notifyUnsafeWorkspaceExportRecovery } from "../unsafe-workspace-export-notifications.js";
import { notifyAutomaticCompletionReviews } from "../automatic-completion-review-notifications.js";
import { notifyObsoletePolicyReviews } from "../obsolete-policy-review-notifications.js";
@@ -151,7 +151,7 @@ export async function dispatchPendingNativeStatusEffects(input: {
targetId: effect.targetId,
});
}
await notifyNativePendingEffects(tx, input);
await notifyNativeFinalizationWork(tx, input);
await notifyChatRunMilestones(tx, input);
await tx
.update(statusDecisionEffects)
@@ -1238,7 +1238,7 @@ async function materializeDecisionEffect(input: {
)
.for("update");
if (pending.length === 0) throw new Error("native_pending_effect_missing");
await notifyNativePendingEffects(input.tx, { companyId: input.companyId });
await notifyNativeFinalizationWork(input.tx, { companyId: input.companyId });
await input.tx
.update(statusDecisionEffects)
.set({
@@ -1888,7 +1888,7 @@ export async function commitNativeStatusDecision(input: {
effect => effect.kind === "record_finalization_error",
);
await notifyDeliveryWork(tx, DELIVERY_QUEUES.accounting, input.companyId);
await notifyNativePendingEffects(tx, input);
await notifyNativeFinalizationWork(tx, input);
await tx
.update(nativeRunFinalizations)
.set({
@@ -1,4 +1,4 @@
import { notifyNativePendingEffects } from "./native-pending-effect-notifications.js";
import { notifyNativeFinalizationWork } from "./native-finalization-notifications.js";
import { and, desc, eq, gte, ne, or, sql } from "drizzle-orm";
import { agentWakeupRequests, heartbeatRuns, issueRecoveryActions, issues, nativeRunFinalizations, nativeRunResults, type Db } from "@paperclipai/db";
import { readSystemRecovery } from "./system-recovery-state.js";
@@ -46,7 +46,7 @@ export async function resumePreservedResultFinalization(db: Db, companyId: strin
const generationId = intent?.id ?? recovery.generationId;
if (coordinator.recoveryHistory.some(entry => entry.kind === "preserved_result_finalization" &&
entry.generationId === generationId)) return false;
await notifyNativePendingEffects(tx, { companyId, runId });
await notifyNativeFinalizationWork(tx, { companyId, runId });
await tx.update(nativeRunFinalizations).set({
phase: coordinator.assessmentId ? "arbitrating" : "workspace_finalizing",
nextAttemptAt: new Date(), failureCode: null, failureDetail: null, updatedAt: new Date(),