fix(server): wake native chat progress from committed work

Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
DottaandPaperclip committed 2026-10-25 12:19:41 -05:00
1 parent b212886949
commit bcf2f38ff5
17 files changed
+711 -281

No files matched your search

@@ -25,7 +25,6 @@ describe("createChatReconciliationCoordinator", () => {
processFailedGitHubWebhookDeliveries,
processPendingDeliveries,
flushPublications,
projectNativeProgress: async () => 0,
processPendingSlackFileUploadReceipts: async () => undefined,
processPendingSlackSessionSyncs: async () => undefined,
onError,
@@ -78,6 +77,8 @@ describe("createChatReconciliationCoordinator", () => {
const maintenance = source.slice(source.indexOf("const chatReconciliation ="), source.indexOf("const unsubscribeChatPublicationSignals"));
expect(maintenance).not.toContain("chatChannels.reconcileProviderRuntimes()");
expect(maintenance).not.toContain("chatChannels.voice.");
expect(maintenance).not.toContain("enqueueSafeNativeChatProgress");
expect(source).toContain("nativeChatProgressWork(db).register(deliveryWork,");
expect(source).toContain("chatChannels.registerReceiverWork(deliveryWork,");
expect(source).toContain("chatChannels.registerTeamsConsentExpiryWork(deliveryWork,");
const teams = readFileSync(new URL("../services/chat-channels/microsoft-teams/file-delivery.ts", import.meta.url), "utf8");
@@ -141,117 +142,6 @@ describe("createChatReconciliationCoordinator", () => {
expect(actionMaintenance).not.toContain("processPendingGitHubWebhookIngress(");
});
it.each([false, true])(
"isolates a blocked native progress projector and joins its completion (rejects: %s)",
async (rejects) => {
let release!: () => void;
const held = new Promise<void>((resolve) => {
release = resolve;
});
const failure = new Error("native progress projection failed");
const projectNativeProgress = vi
.fn(async () => 0)
.mockImplementationOnce(async () => {
await held;
if (rejects) throw failure;
return 1;
});
const flushPublications = vi.fn(async () => undefined);
const onError = vi.fn();
const coordinator = createChatReconciliationCoordinator({
projectNativeProgress,
processPendingDeliveries: async () => undefined,
flushPublications,
processPendingSlackFileUploadReceipts: async () => undefined,
processPendingSlackSessionSyncs: async () => undefined,
onError,
});
try {
coordinator.reconcile();
await vi.waitFor(() => {
expect(projectNativeProgress).toHaveBeenCalledTimes(1);
expect(flushPublications).toHaveBeenCalledTimes(1);
});
// A newly committed final/question must dispatch even while a different
// issue's projection is blocked. Repeated signals leave one dirty retry.
coordinator.notifyPublications();
coordinator.notifyPublications();
await vi.waitFor(() =>
expect(flushPublications).toHaveBeenCalledTimes(2),
);
expect(projectNativeProgress).toHaveBeenCalledTimes(1);
let drained = false;
const draining = coordinator.drain().then(() => {
drained = true;
});
await new Promise<void>((resolve) => setImmediate(resolve));
expect(drained).toBe(false);
release();
await draining;
expect(projectNativeProgress).toHaveBeenCalledTimes(2);
expect(onError).toHaveBeenCalledTimes(rejects ? 1 : 0);
if (rejects)
expect(onError).toHaveBeenCalledWith("native progress", failure);
const completedCounts = [
projectNativeProgress.mock.calls.length,
flushPublications.mock.calls.length,
];
await new Promise<void>((resolve) => setImmediate(resolve));
expect([
projectNativeProgress.mock.calls.length,
flushPublications.mock.calls.length,
]).toEqual(completedCounts);
} finally {
release();
coordinator.stop();
await coordinator.drain();
}
},
);
it("joins a blocked native progress projector at shutdown without starting dirty follow-ups", async () => {
let release!: () => void;
const held = new Promise<void>((resolve) => {
release = resolve;
});
const projectNativeProgress = vi.fn(async () => {
await held;
return 1;
});
const flushPublications = vi.fn(async () => undefined);
const coordinator = createChatReconciliationCoordinator({
projectNativeProgress,
flushPublications,
processPendingDeliveries: async () => undefined,
processPendingSlackFileUploadReceipts: async () => undefined,
processPendingSlackSessionSyncs: async () => undefined,
onError: vi.fn(),
});
try {
coordinator.reconcile();
await vi.waitFor(() => {
expect(projectNativeProgress).toHaveBeenCalledOnce();
expect(flushPublications).toHaveBeenCalledOnce();
});
coordinator.notifyPublications();
coordinator.stop();
let drained = false;
const draining = coordinator.drain().then(() => {
drained = true;
});
await new Promise<void>((resolve) => setImmediate(resolve));
expect(drained).toBe(false);
release();
await draining;
expect(projectNativeProgress).toHaveBeenCalledOnce();
expect(flushPublications).toHaveBeenCalledOnce();
} finally {
release();
coordinator.stop();
await coordinator.drain();
}
});
it("keeps slow optional recovery from suppressing later publication sweeps", async () => {
let releaseDelivery!: () => void;
let releaseSlackStatus!: () => void;
@@ -269,7 +159,6 @@ describe("createChatReconciliationCoordinator", () => {
);
const onError = vi.fn();
const coordinator = createChatReconciliationCoordinator({
projectNativeProgress: async () => 0,
processPendingDeliveries,
flushPublications,
processPendingSlackFileUploadReceipts,
@@ -317,7 +206,6 @@ describe("createChatReconciliationCoordinator", () => {
const processPendingSlackSessionSyncs = vi.fn(async () => undefined);
const onError = vi.fn();
const coordinator = createChatReconciliationCoordinator({
projectNativeProgress: async () => 0,
processPendingDeliveries,
flushPublications,
processPendingSlackFileUploadReceipts,
@@ -1,3 +1,4 @@
import { nativeChatProgressWork } from "../services/chat-native-progress-work.js";
import { cloudActorHeaderSourceFromHeaders, resolveCloudTenantActor } from "../middleware/auth.js";
import { cloudTenantPrimaryCompanyId } from "../services/cloud-instance.js";
import { accessService } from "../services/access.js";
@@ -52781,6 +52782,57 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => {
},
);
it.each(["identity", "resource", "visibility"] as const)(
"wakes saved post-close native progress when %s authority returns", async owner => {
const f = await reopenedFixture();
const onError = vi.fn();
const coordinator = createDeliveryWorkCoordinator({ owner: db, canRun: () => true, onError });
const progress = () => db.select().from(chatPublications).where(and(
eq(chatPublications.issueId, f.conversation.issueId), like(chatPublications.idempotencyKey, "%:native:%"),
));
try {
await db.insert(chatPublications).values({
companyId: f.fixture.companyId, endpointId: f.endpoint.id,
conversationId: f.conversation.id, issueId: f.conversation.issueId,
idempotencyKey: `run:${f.fresh.runId}:working:${f.endpoint.id}`,
payload: { text: "Maya is working…", progressState: "working" }, state: "published",
createdAt: new Date(Date.now() - 60_000),
});
await db.insert(heartbeatRunEvents).values({
companyId: f.fixture.companyId, agentId: f.fixture.assignedAgentId,
runId: f.fresh.runId, seq: 1, eventType: "item.completed",
});
let restore: () => Promise<unknown>;
if (owner === "identity") {
await f.service.update(f.endpoint.id, { allowUnlinkedPeople: false }, "owner-user");
await f.service.revokeLink(f.endpoint.id, f.fresh.action.principalId!);
const intent = await f.service.createLinkIntent(f.endpoint.id, f.fresh.action.principalId!, 1800);
restore = () => f.service.confirmIdentityLink(new URL(intent.confirmationUrl).searchParams.get("token")!, "owner-user");
} else if (owner === "resource") {
const [conversation] = await db.select().from(chatConversations).where(eq(chatConversations.id, f.conversation.id));
expect(conversation.resourceId).not.toBeNull();
await f.service.replaceResources(f.endpoint.id, [{ id: conversation.resourceId!, enabled: false }], "owner-user");
restore = () => f.service.replaceResources(f.endpoint.id, [{ id: conversation.resourceId!, enabled: true }], "owner-user");
} else {
await issueService(db).update(f.conversation.issueId, { hiddenAt: new Date() });
restore = () => issueService(db).update(f.conversation.issueId, { hiddenAt: null });
}
await nativeChatProgressWork(db).register(coordinator, () => true).ready;
await vi.waitFor(() => { expect(coordinator.nextWakeAt()).toBeNull(); expect(idleWorkSnapshot().active).toBe(0); });
expect(await progress()).toEqual([]);
await restore();
await expect.poll(progress, { timeout: 5000 }).toEqual([
expect.objectContaining({ idempotencyKey: `run:${f.fresh.runId}:working:${f.endpoint.id}:native:making_progress:1` }),
]);
await vi.waitFor(() => { expect(coordinator.nextWakeAt()).toBeNull(); expect(idleWorkSnapshot().active).toBe(0); });
const select = vi.spyOn(db, "select");
try { await new Promise(resolve => setTimeout(resolve, 1200)); expect(select).not.toHaveBeenCalled(); }
finally { select.mockRestore(); }
expect(onError).not.toHaveBeenCalled();
} finally { await coordinator.stop(); await f.close(); }
},
);
it("publishes working and the exact fresh final after pinned close without resurrecting the old source", async () => {
const f = await reopenedFixture();
try {
@@ -63567,6 +63619,45 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => {
};
}
it.each(["replay", "operator retry"] as const)(
"wakes saved native progress when %s restores eligibility", async owner => {
const context = await safeNativeProgressFixture("telegram", `12${["replay", "operator retry"].indexOf(owner)}`, "channel", "42", true);
const onError = vi.fn();
const coordinator = createDeliveryWorkCoordinator({ owner: db, canRun: () => true, onError });
const progress = () => db.select().from(chatPublications).where(and(
eq(chatPublications.issueId, context.conversation.issueId),
like(chatPublications.idempotencyKey, "%:native:%"),
));
const settled = () => vi.waitFor(() => {
expect(coordinator.nextWakeAt()).toBeNull();
expect(idleWorkSnapshot().active).toBe(0);
}, { timeout: 5000 });
try {
const run = await context.createRun("saved progress awaiting eligibility");
await context.addEvent(run);
const [base] = await db.update(chatPublications).set({
state: owner === "replay" ? "failed" : "delivery_unknown",
}).where(eq(chatPublications.idempotencyKey, `run:${run.runId}:working:${context.endpoint.id}`)).returning();
const restore = () => owner === "replay"
? context.service.replayPublication(context.endpoint.id, base.id)
: context.service.resolvePublication(context.endpoint.id, base.id, "retry_anyway", "owner-user");
await nativeChatProgressWork(db).register(coordinator, () => true).ready;
await settled(); expect(await progress()).toEqual([]);
await restore();
await expect.poll(progress, { timeout: 5000 }).toEqual([
expect.objectContaining({ idempotencyKey: `run:${run.runId}:working:${context.endpoint.id}:native:making_progress:1` }),
]);
await settled();
const select = vi.spyOn(db, "select"), transaction = vi.spyOn(db, "transaction");
try {
await new Promise(resolve => setTimeout(resolve, 1200));
expect(select).not.toHaveBeenCalled(); expect(transaction).not.toHaveBeenCalled();
} finally { select.mockRestore(); transaction.mockRestore(); }
expect(onError).not.toHaveBeenCalled();
} finally { await coordinator.stop(); await retirePublicationFixture(context.service, context.endpoint.id); }
},
);
it("advances native progress past suppressed candidates with microsecond timestamps", async () => {
const context = await safeNativeProgressFixture("telegram", "119");
try {
@@ -0,0 +1,350 @@
import { enqueueChatRunLifecycleMilestones } from "../services/chat-run-publications.js";
import { persistReprojectedWorkspaceDiffs, projectCodexWorkspaceDiffsFromTrace } from "../services/provider-trace-workspace-diff-reprojection.js";
import { randomUUID } from "node:crypto";
import { eq, like, and } from "drizzle-orm";
import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
import { chatPublications, createDb, heartbeatRunEvents, heartbeatRuns, issues, subscribeDatabaseWork } from "@paperclipai/db";
import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js";
import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js";
import { createChatMilestoneFixtures } from "./helpers/chat-milestone-fixtures.js";
import { createDeliveryWorkCoordinator } from "../services/delivery-work-coordinator.js";
import { DELIVERY_QUEUES } from "../services/delivery-work-notifications.js";
import { nativeChatProgressWork } from "../services/chat-native-progress-work.js";
import { appendHeartbeatRunEvent } from "../services/heartbeat-run-events.js";
import { NativeRunCoordinatorStore } from "../services/native-runtime/native-run-coordinator-store.js";
import { PaperclipControlPlanePort } from "../services/native-runtime/paperclip-control-plane-port.js";
import type { PrpEvent } from "../vendor/paperclip-runner/index.js";
import { idleWorkSnapshot } from "../services/task-admission.js";
const external = resolveExternalTestDatabaseUrl("PAPERCLIP_NATIVE_PROGRESS_TEST_DATABASE_URL");
const support = external ? { supported: true } : await getEmbeddedPostgresTestSupport();
(support.supported ? describe : describe.skip)("native chat progress work", () => {
let db: ReturnType<typeof createDb>;
let temporary: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>> | undefined;
let fixtures: ReturnType<typeof createChatMilestoneFixtures>;
let f: Awaited<ReturnType<typeof fixtures.create>>;
let runId: string;
let coordinator: ReturnType<typeof createDeliveryWorkCoordinator> | undefined;
let canRun = true;
const errors: unknown[] = [];
beforeAll(async () => {
if (!external) temporary = await startEmbeddedPostgresTestDatabase("native-chat-progress-");
db = createDb(external ?? temporary!.connectionString, { idleTimeoutSeconds: 0, maxLifetimeSeconds: 0 });
fixtures = createChatMilestoneFixtures(db);
}, 60_000);
beforeEach(async () => {
errors.length = 0; canRun = true; f = await fixtures.create(); runId = randomUUID();
await db.insert(heartbeatRuns).values({ id: runId, companyId: f.companyId, agentId: f.agentId,
nativeIssueId: f.issueId, runtimeMode: "native", status: "running", responsibleUserId: f.userId,
contextSnapshot: { issueId: f.issueId, source: "chat:slack", endpointId: f.endpointId, wakeCommentId: f.commentId },
});
await db.insert(chatPublications).values({ companyId: f.companyId, endpointId: f.endpointId,
conversationId: f.conversationId, issueId: f.issueId, idempotencyKey: `run:${runId}:working:${f.endpointId}`,
payload: { text: "Maya is working…", progressState: "working" }, state: "published",
createdAt: new Date(Date.now() - 60_000),
});
vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"], shouldAdvanceTime: true });
});
afterEach(async () => {
await coordinator?.stop(); coordinator = undefined;
vi.useRealTimers(); vi.restoreAllMocks(); await fixtures.cleanup();
});
afterAll(async () => { await db?.$client.end({ timeout: 1 }); await temporary?.cleanup(); });
const publications = () => db.select().from(chatPublications).where(and(
eq(chatPublications.issueId, f.issueId), like(chatPublications.idempotencyKey, "%:native:%"),
));
async function start() {
coordinator = createDeliveryWorkCoordinator({ owner: db, canRun: () => canRun, canReconcile: () => true,
onError: error => errors.push(error) });
await nativeChatProgressWork(db).register(coordinator, () => canRun).ready;
}
async function settle() {
await vi.waitFor(async () => {
await vi.advanceTimersByTimeAsync(1000);
expect(idleWorkSnapshot().active).toBe(0); expect(coordinator!.nextWakeAt()).toBeNull();
}, { timeout: 10_000 });
}
async function quiet() {
const select = vi.spyOn(db, "select"), transaction = vi.spyOn(db, "transaction");
try { await vi.advanceTimersByTimeAsync(120_000); expect(select).not.toHaveBeenCalled(); expect(transaction).not.toHaveBeenCalled(); }
finally { select.mockRestore(); transaction.mockRestore(); }
}
function event(nativeSource = false, eventType = "item.completed") {
return { companyId: f.companyId, agentId: f.agentId, runId, eventType,
message: "PRIVATE event text", payload: { token: "PRIVATE token" },
...(nativeSource ? { nativeSource: { sourceInstanceId: runId, sourceEventId: "event-1", sourceSeq: 1,
protocolSchemaVersion: 1, canonicalPayload: { eventType } } } : {}),
};
}
it.each([false, true])("wakes from a real committed event after idle (native source: %s)", async nativeSource => {
await start(); await settle(); await quiet(); expect(await publications()).toEqual([]);
const input = event(nativeSource);
const save = (rollback: boolean) => db.transaction(async tx => {
await appendHeartbeatRunEvent(tx as unknown as typeof db, input);
await vi.advanceTimersByTimeAsync(1000);
expect(await publications()).toEqual([]);
if (rollback) throw new Error("outer rollback");
});
await expect(save(true)).rejects.toThrow("outer rollback");
await settle(); expect(await publications()).toEqual([]); await quiet();
await save(false);
await vi.waitFor(async () => {
await vi.advanceTimersByTimeAsync(1000);
expect(await publications()).toEqual([expect.objectContaining({ payload: expect.objectContaining({ text: "Maya is making progress…" }) })]);
}, { timeout: 5000 });
await settle(); await quiet(); expect(errors).toEqual([]);
if (nativeSource) {
const signals: string[] = [];
const unsubscribe = subscribeDatabaseWork(db, DELIVERY_QUEUES.chatNativeProgress, kind => signals.push(kind));
try { expect((await appendHeartbeatRunEvent(db, input)).disposition).toBe("duplicate"); expect(signals).toEqual([]); }
finally { unsubscribe(); }
await quiet();
}
});
it("wakes a saved event when its first working publication commits after event selection", async () => {
await db.delete(chatPublications).where(eq(chatPublications.issueId, f.issueId));
await appendHeartbeatRunEvent(db, event(true));
await start(); await settle(); expect(await publications()).toEqual([]); await quiet();
await db.transaction(async tx => {
expect(await enqueueChatRunLifecycleMilestones(tx as unknown as typeof db, {
scope: { companyId: f.companyId, issueId: f.issueId },
})).toBe(1);
// Model a base transaction that began before the event but committed
// afterwards, without a wall-clock delay in the suite. PostgreSQL's
// default now() is transaction-start time, not commit time.
await tx.update(chatPublications).set({ createdAt: new Date(Date.now() - 300_000) })
.where(eq(chatPublications.idempotencyKey, `run:${runId}:working:${f.endpointId}`));
await vi.advanceTimersByTimeAsync(1000); expect(await publications()).toEqual([]);
});
await settle(); expect(await publications()).toHaveLength(1); await quiet(); expect(errors).toEqual([]);
});
it("reconstructs saved progress after restart and becomes quiet with published history", async () => {
await appendHeartbeatRunEvent(db, event(true));
await start(); await settle(); expect(await publications()).toHaveLength(1); await quiet();
await coordinator!.stop(); await start(); await settle();
expect(await publications()).toHaveLength(1); await quiet(); expect(errors).toEqual([]);
});
it.each(["issue", "run"] as const)("retries a contended %s without another event", async owner => {
await appendHeartbeatRunEvent(db, event(true));
let release!: () => void, entered!: () => void;
const hold = new Promise<void>(resolve => { release = resolve; });
const ready = new Promise<void>(resolve => { entered = resolve; });
const lock = db.transaction(async tx => {
if (owner === "issue") await tx.select().from(issues).where(eq(issues.id, f.issueId)).for("update");
else await tx.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, runId)).for("update");
entered(); await hold;
});
try {
await ready; await start(); expect(await publications()).toEqual([]);
expect(coordinator!.nextWakeAt()).not.toBeNull();
release(); await lock;
await settle(); expect(await publications()).toHaveLength(1); await quiet(); expect(errors).toEqual([]);
} finally { release(); await lock; }
});
it.each(["port", "coordinator"] as const)("wakes from the real %s event owner after outer commit, with quiet replay", async kind => {
const sessionId = randomUUID(), runnerId = randomUUID();
await db.update(heartbeatRuns).set({ nativeSessionId: sessionId }).where(eq(heartbeatRuns.id, runId));
const binding = { companyId: f.companyId, issueId: f.issueId, runId, agentId: f.agentId,
sessionId, completionContractId: randomUUID(), completionContractSha256: "fixture",
sourceInstanceId: runnerId, controlPlaneSourceInstanceId: "control",
normalizedSessionId: sessionId, runnerSourceInstanceId: runnerId,
completionContractRevision: "1", completionContractCriterionIds: [],
};
const source: PrpEvent = { schema: "paperclip.prp.event.v1", schemaVersion: 1,
sourceEventId: "event-1", sourceSeq: 1, sourceInstanceId: runnerId, sourceKind: "runner",
runId, normalizedSessionId: sessionId, turnId: "turn-1", itemId: "item-1",
eventType: "item.completed", priority: 1, emittedAt: new Date().toISOString(), payload: {},
};
const append = (client: typeof db) => (kind === "port"
? new PaperclipControlPlanePort(client, binding)
: new NativeRunCoordinatorStore(client, binding)).appendEvent(source);
await start(); await settle(); await quiet();
await expect(db.transaction(async tx => {
await append(tx as unknown as typeof db); throw new Error("rollback owner");
})).rejects.toThrow("rollback owner");
await settle(); expect(await publications()).toEqual([]); await quiet();
await db.transaction(async tx => {
await append(tx as unknown as typeof db); await vi.advanceTimersByTimeAsync(1000);
expect(await publications()).toEqual([]);
});
await settle(); expect(await publications()).toHaveLength(1); await quiet();
const signals: string[] = [];
const unsubscribe = subscribeDatabaseWork(db, DELIVERY_QUEUES.chatNativeProgress, kind => signals.push(kind));
try { expect((await append(db)).disposition).toBe("duplicate"); expect(signals).toEqual([]); }
finally { unsubscribe(); }
await quiet(); expect(errors).toEqual([]);
});
it.each(["issue", "run"] as const)("disarms after terminalization wins during a contended %s", async owner => {
await appendHeartbeatRunEvent(db, event(true));
let release!: () => void, entered!: () => void;
const hold = new Promise<void>(resolve => { release = resolve; });
const ready = new Promise<void>(resolve => { entered = resolve; });
const lock = db.transaction(async tx => {
if (owner === "issue") await tx.select().from(issues).where(eq(issues.id, f.issueId)).for("update");
else await tx.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, runId)).for("update");
entered(); await hold;
await tx.update(heartbeatRuns).set({ status: "failed" }).where(eq(heartbeatRuns.id, runId));
});
try {
await ready; await start(); expect(coordinator!.nextWakeAt()).not.toBeNull();
release(); await lock; await settle(); expect(await publications()).toEqual([]); await quiet();
expect(errors).toEqual([]);
} finally { release(); await lock; }
});
it.each(["first write", "startup"] as const)("retains %s through drain without passive SQL", async kind => {
if (kind === "first write") { await start(); await settle(); }
canRun = false;
await appendHeartbeatRunEvent(db, event(true));
if (kind === "startup") await start();
await vi.advanceTimersByTimeAsync(1000);
await vi.waitFor(() => expect(idleWorkSnapshot().active).toBe(0));
expect(await publications()).toEqual([]); await quiet();
canRun = true; await vi.advanceTimersByTimeAsync(30_000); await settle();
expect(await publications()).toHaveLength(1); await quiet(); expect(errors).toEqual([]);
});
it("retains failed startup selection through drain and retries after resume", async () => {
await appendHeartbeatRunEvent(db, event(true));
const select = vi.spyOn(db, "select").mockImplementationOnce(() => { throw new Error("temporary read failure"); });
try { await start(); } finally { select.mockRestore(); }
expect(errors.splice(0)).toHaveLength(1); canRun = false;
await vi.advanceTimersByTimeAsync(30_000);
await vi.waitFor(() => expect(idleWorkSnapshot().active).toBe(0));
await quiet(); expect(await publications()).toEqual([]);
canRun = true; await vi.advanceTimersByTimeAsync(30_000); await settle();
expect(await publications()).toHaveLength(1); await quiet(); expect(errors).toEqual([]);
});
it("leaves unrelated direct log events on their atomic statement path without projection intent", async () => {
await start(); await settle();
const signals: string[] = [];
const unsubscribe = subscribeDatabaseWork(db, DELIVERY_QUEUES.chatNativeProgress, kind => signals.push(kind));
const transaction = vi.spyOn(db, "transaction");
try {
await appendHeartbeatRunEvent(db, event(false, "item.delta"));
expect(transaction).not.toHaveBeenCalled(); expect(signals).toEqual([]);
} finally { transaction.mockRestore(); unsubscribe(); }
await quiet(); expect(await publications()).toEqual([]); expect(errors).toEqual([]);
});
it("wakes saved trace reprojection without live runner ingress and leaves duplicate recovery quiet", async () => {
await start(); await settle(); await quiet();
const projection = projectCodexWorkspaceDiffsFromTrace([{
kind: "frame", direction: "provider_to_client", frameId: 1,
rawBase64: Buffer.from(JSON.stringify({ method: "turn/diff/updated", params: {
turnId: "turn-1", diff: "diff --git a/private.txt b/private.txt\n--- a/private.txt\n+++ b/private.txt\n@@ -1 +1 @@\n-private\n+PRIVATE replacement\n",
} })).toString("base64"),
}]);
expect(projection.turns).toHaveLength(1);
const input = { traceId: randomUUID(), companyId: f.companyId, agentId: f.agentId, runId, projection };
expect(await persistReprojectedWorkspaceDiffs(db, input)).toMatchObject({ created: 1 });
await settle();
expect(await publications()).toEqual([expect.objectContaining({ payload: expect.objectContaining({ text: "Maya is working with files…" }) })]);
await quiet();
const signals: string[] = [];
const unsubscribe = subscribeDatabaseWork(db, DELIVERY_QUEUES.chatNativeProgress, kind => signals.push(kind));
try { expect(await persistReprojectedWorkspaceDiffs(db, input)).toMatchObject({ created: 0 }); expect(signals).toEqual([]); }
finally { unsubscribe(); }
await quiet(); expect(errors).toEqual([]);
});
it("retains a committed event that arrives before the current projection pass returns", async () => {
const [source] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, runId));
const secondRunId = randomUUID();
await db.insert(heartbeatRuns).values({
id: secondRunId, companyId: f.companyId, agentId: f.agentId, nativeIssueId: f.issueId,
runtimeMode: "native", status: "running", responsibleUserId: f.userId, contextSnapshot: source.contextSnapshot,
});
await db.insert(chatPublications).values({
companyId: f.companyId, endpointId: f.endpointId, conversationId: f.conversationId,
issueId: f.issueId, idempotencyKey: `run:${secondRunId}:working:${f.endpointId}`,
payload: { text: "Maya is working…", progressState: "working" }, state: "published",
createdAt: new Date(Date.now() - 60_000),
});
await start(); await settle();
let held = false, release!: () => void;
const pending = new Promise<void>(resolve => { release = resolve; });
const transaction = db.transaction.bind(db);
const barrier = vi.spyOn(db, "transaction").mockImplementation(async (callback, options) => {
const result = await transaction(callback, options);
// Pause after the first projection commits, before its pass can disarm.
// Producer transactions return event records, not an inserted row count.
if (result === 1 && !held) { held = true; await pending; }
return result;
});
try {
await appendHeartbeatRunEvent(db, event(true));
await vi.waitFor(() => expect(held).toBe(true), { timeout: 5000 });
await appendHeartbeatRunEvent(db, { ...event(false), runId: secondRunId });
release(); await settle();
expect(await publications()).toHaveLength(2); barrier.mockRestore();
await quiet(); expect(errors).toEqual([]);
} finally { release(); barrier.mockRestore(); }
});
it("limits an event wake to its issue and reconstructs other saved scopes on restart", async () => {
await start(); await settle();
// Seed rows without signals to expose an accidentally global event wake.
const others = [await fixtures.create(f.companyId), await fixtures.create()];
for (const other of others) {
const id = randomUUID();
await db.insert(heartbeatRuns).values({
id, companyId: other.companyId, agentId: other.agentId, nativeIssueId: other.issueId,
runtimeMode: "native", status: "running", responsibleUserId: other.userId,
contextSnapshot: { issueId: other.issueId, source: "chat:slack", endpointId: other.endpointId, wakeCommentId: other.commentId },
});
await db.insert(chatPublications).values({
companyId: other.companyId, endpointId: other.endpointId, conversationId: other.conversationId,
issueId: other.issueId, idempotencyKey: `run:${id}:working:${other.endpointId}`,
payload: { text: "Maya is working…", progressState: "working" }, state: "published",
createdAt: new Date(Date.now() - 60_000),
});
await db.insert(heartbeatRunEvents).values({ companyId: other.companyId, agentId: other.agentId,
runId: id, seq: 1, eventType: "item.completed" });
}
const saved = (issueId: string) => db.select().from(chatPublications).where(and(
eq(chatPublications.issueId, issueId), like(chatPublications.idempotencyKey, "%:native:%"),
));
await appendHeartbeatRunEvent(db, event(true)); await settle();
expect(await publications()).toHaveLength(1);
for (const other of others) expect(await saved(other.issueId)).toEqual([]);
await quiet(); await coordinator!.stop(); await start(); await settle();
for (const other of others) expect(await saved(other.issueId)).toHaveLength(1);
await quiet(); expect(errors).toEqual([]);
});
it("drains more than one issue batch and becomes quiet with all progress retained", async () => {
const [source] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, runId));
const ids = Array.from({ length: 105 }, () => randomUUID());
await db.insert(heartbeatRuns).values(ids.map(id => ({
id, companyId: f.companyId, agentId: f.agentId, nativeIssueId: f.issueId,
runtimeMode: "native", status: "running", responsibleUserId: f.userId, contextSnapshot: source.contextSnapshot,
})));
await db.insert(chatPublications).values(ids.map(id => ({
companyId: f.companyId, endpointId: f.endpointId, conversationId: f.conversationId,
issueId: f.issueId, idempotencyKey: `run:${id}:working:${f.endpointId}`,
payload: { text: "Maya is working…", progressState: "working" }, state: "published",
createdAt: new Date(Date.now() - 60_000),
})));
await db.insert(heartbeatRunEvents).values(ids.map(id => ({
companyId: f.companyId, agentId: f.agentId, runId: id, seq: 1, eventType: "item.completed",
})));
await start(); await settle();
expect(await publications()).toHaveLength(ids.length); await quiet(); expect(errors).toEqual([]);
});
it("unsubscribes at shutdown and reconstructs a later event on restart", async () => {
await start(); await settle(); await coordinator!.stop();
await appendHeartbeatRunEvent(db, event(true));
await quiet(); expect(await publications()).toEqual([]);
await start(); await settle(); expect(await publications()).toHaveLength(1); expect(errors).toEqual([]);
});
});
@@ -173,7 +173,7 @@ const support = external ? { supported: true } : await getEmbeddedPostgresTestSu
await db.update(chatVoiceInboundCalls).set({ expiresAt: new Date(0), updatedAt: new Date(0) }).where(eq(chatVoiceInboundCalls.id, f.call.id));
const onError = vi.fn();
const maintenance = createChatReconciliationCoordinator({
processPendingDeliveries: async () => undefined, projectNativeProgress: async () => 0,
processPendingDeliveries: async () => undefined,
flushPublications: async () => undefined, onError,
});
try {
+4 -22
View File
@@ -195,7 +195,7 @@ import { createToolGatewayService } from "./services/tool-gateway.js";
import { toolAccessService } from "./services/tool-access.js";
import { chatChannelService } from "./services/chat-channels.js";
import { deliverNativeQuestionResponse } from "./services/native-runtime/native-question-bridge.js";
import { enqueueSafeNativeChatProgress } from "./services/chat-run-publications.js";
import { nativeChatProgressWork } from "./services/chat-native-progress-work.js";
import { chatRunMilestoneWork } from "./services/chat-run-milestone-work.js";
import {
createCoalescedAsyncTrigger,
@@ -324,7 +324,6 @@ export function shouldEnablePrivateHostnameGuard(opts: {
type ChatReconciliationLane =
| "deliveries"
| "GitHub webhook recovery"
| "native progress"
| "publications"
| "Slack file receipts"
| "Slack session status";
@@ -337,7 +336,6 @@ type ChatReconciliationLane =
export function createChatReconciliationCoordinator(input: {
processPendingDeliveries: () => Promise<unknown>;
processFailedGitHubWebhookDeliveries?: () => Promise<unknown>;
projectNativeProgress: () => Promise<number>;
flushPublications: () => Promise<unknown>;
processPendingSlackFileUploadReceipts?: () => Promise<unknown>;
processPendingSlackSessionSyncs?: () => Promise<unknown>;
@@ -349,16 +347,6 @@ export function createChatReconciliationCoordinator(input: {
run: input.flushPublications,
onError: (error) => input.onError("publications", error),
});
const progressReconciliation = createCoalescedAsyncTrigger({
run: async () => {
const inserted = await input.projectNativeProgress();
// Existing final/question publications never wait on this optional
// projection. Newly committed progress gets a bounded dispatch wake;
// an empty/contended pass does not create a self-sustaining loop.
if (inserted > 0) publicationReconciliation.notify();
},
onError: (error) => input.onError("native progress", error),
});
const start = (
lane: ChatReconciliationLane,
task: () => Promise<unknown>,
@@ -383,27 +371,21 @@ export function createChatReconciliationCoordinator(input: {
input.processFailedGitHubWebhookDeliveries,
);
}
progressReconciliation.poll();
publicationReconciliation.poll();
if (input.processPendingSlackFileUploadReceipts) start("Slack file receipts", input.processPendingSlackFileUploadReceipts);
if (input.processPendingSlackSessionSyncs) start("Slack session status", input.processPendingSlackSessionSyncs);
},
notifyPublications() {
progressReconciliation.notify();
publicationReconciliation.notify();
},
stop() {
stopped = true;
progressReconciliation.stop();
publicationReconciliation.stop();
},
async drain() {
await Promise.allSettled([
...inFlight.values(),
progressReconciliation.drain(),
]);
// Projecting the final batch can notify dispatch after an earlier drain
// would have returned. Join dispatch only after its producer has drained.
await publicationReconciliation.drain();
},
};
@@ -1305,6 +1287,7 @@ export async function createApp(
chatChannels.registerTeamsConsentExpiryWork(deliveryWork, () => !isWarmStandby() && !isIdleTaskDrainActive());
chatChannels.registerTeamsAttemptRecoveryWork(deliveryWork, () => !isWarmStandby() && !isIdleTaskDrainActive());
chatChannels.registerCommandWork(deliveryWork, () => !isWarmStandby() && !isIdleTaskDrainActive());
nativeChatProgressWork(db).register(deliveryWork, () => !isWarmStandby() && !isIdleTaskDrainActive());
chatRunMilestoneWork(db, opts.authPublicBaseUrl).register(
deliveryWork, () => !isWarmStandby() && !isIdleTaskDrainActive(),
);
@@ -1318,7 +1301,6 @@ export async function createApp(
processPendingDeliveries: async () => undefined,
processFailedGitHubWebhookDeliveries: () =>
chatChannels.processFailedGitHubWebhookDeliveries(),
projectNativeProgress: () => enqueueSafeNativeChatProgress(db),
flushPublications: () => reconcileChatPublicationMaintenance(),
onError: (lane, err) => {
logger.error({ err, lane }, `Failed to reconcile chat ${lane}`);
@@ -1330,8 +1312,8 @@ export async function createApp(
chatReconciliation.notifyPublications();
},
);
// Native progress and remaining publication maintenance still use this cadence.
// Lifecycle milestones use committed changes and their own recovery worker.
// Remaining publication maintenance still uses this cadence. Lifecycle and
// native progress projection have independent commit/deadline workers.
// Inbox, outbox, and file receipt queues have independent commit/deadline wakes.
let chatPublicationTimer: ReturnType<typeof setInterval> | null = setInterval(
() => {
@@ -113,7 +113,7 @@ export function createChatPublicationResolution(db: Db, options: ResolutionOptio
code: "chat_file_transfer_resolution_required",
},
);
await notifyChatPublicationWork(tx);
await notifyChatPublicationWork(tx, current);
return tx
.update(chatPublications)
.set({
@@ -218,7 +218,7 @@ export function createChatPublicationResolution(db: Db, options: ResolutionOptio
action === "retry_anyway"
) { await recordPhotonPublicationRetry(tx, endpointId, publication, userId); }
const now = new Date();
await notifyChatPublicationWork(tx, action === "retry_anyway" ? undefined : publication);
await notifyChatPublicationWork(tx, publication);
await notifyVoicePublication(tx, initialRecord.endpoint);
await tx
.update(chatPublications)
@@ -0,0 +1,6 @@
import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js";
import type { ChatRunProjectionScope } from "./chat-run-publication-policy.js";
export function notifyChatNativeProgress(tx: object, scope: ChatRunProjectionScope) {
return notifyDeliveryWork(tx, DELIVERY_QUEUES.chatNativeProgress, JSON.stringify(scope));
}
@@ -0,0 +1,34 @@
import type { Db } from "@paperclipai/db";
import type { createDeliveryWorkCoordinator } from "./delivery-work-coordinator.js";
import { DELIVERY_QUEUES } from "./delivery-work-notifications.js";
import { registerScopedDeadlineWork } from "./scoped-deadline-work.js";
import type { ChatRunProjectionScope } from "./chat-run-publication-policy.js";
import { loadNativeChatProgressScopes, reconcileNativeChatProgress } from "./chat-native-progress.js";
type Entry = { id: string; companyId: string; issueId: string; dueAt: number };
const batchSize = 100;
export function nativeChatProgressWork(db: Db) {
async function load(scope?: ChatRunProjectionScope, signal?: AbortSignal): Promise<Entry[]> {
return (await loadNativeChatProgressScopes(db, scope, signal)).map(row => ({
...row, id: `${row.companyId}:${row.issueId}`, dueAt: 0,
}));
}
async function check(entry: Entry, signal?: AbortSignal): Promise<Entry | null> {
const result = await reconcileNativeChatProgress(db, {
scope: { companyId: entry.companyId, issueId: entry.issueId }, limit: batchSize, signal,
});
if (result.inserted === batchSize) return { ...entry, dueAt: Date.now() };
return result.contended ? { ...entry, dueAt: Date.now() + 1000 } : null;
}
function register(coordinator: ReturnType<typeof createDeliveryWorkCoordinator>, canRun: () => boolean) {
return registerScopedDeadlineWork(coordinator, {
queue: DELIVERY_QUEUES.chatNativeProgress,
decodeScope: (hint: string): ChatRunProjectionScope | undefined => hint === "*" ? undefined : JSON.parse(hint),
matches: (entry: Entry, scope?: ChatRunProjectionScope) => !scope ||
(entry.companyId === scope.companyId && (!scope.issueId || entry.issueId === scope.issueId)),
load, check,
}, canRun);
}
return { load, check, register };
}
+56 -22
View File
@@ -7,7 +7,7 @@ import { projectSafeChatPublication } from "./chat-publication-projection.js";
import { hasChatRunOwnedProviderInteraction } from "./chat-interaction-arbitration.js";
import { resolveChatOriginPublicationBindings } from "./issues.js";
import { boundRunIssueIdSql } from "./run-task-binding.js";
import { allowsAutomaticRunPublication, chatRunProjectionWindow } from "./chat-run-publication-policy.js";
import { allowsAutomaticRunPublication, chatRunProjectionWindow, type ChatRunProjectionScope } from "./chat-run-publication-policy.js";
import { SAFE_NATIVE_CHAT_PROGRESS_EVENT_TYPES, safeNativeChatProgressForEvent } from "./safe-native-chat-progress.js";
type SafeNativeChatProgressCandidate = {
@@ -26,12 +26,10 @@ type SafeNativeChatProgressCandidate = {
const SAFE_NATIVE_CHAT_PROGRESS_CADENCE_MS = 20_000;
/** Only closed event types become coarse progress; event payloads stay local. */
export async function enqueueSafeNativeChatProgress(
db: Db,
options: { since?: Date; limit?: number } = {},
): Promise<number> {
const input = chatRunProjectionWindow(options);
type NativeProgressCursor = Pick<SafeNativeChatProgressCandidate, "eventCreatedAt" | "runId" | "conversationId">;
type NativeProgressInput = { since?: Date; limit?: number; scope?: ChatRunProjectionScope; signal?: AbortSignal };
function nativeProgressPages(db: Db, since: Date, scope?: ChatRunProjectionScope) {
const basePublication = alias(
chatPublications,
"safe_native_progress_base_publication",
@@ -41,17 +39,7 @@ export async function enqueueSafeNativeChatProgress(
"safe_native_progress_later_event",
);
const issueIdFromContext = boundRunIssueIdSql();
let inserted = 0;
let cursor: {
eventCreatedAt: string;
runId: string;
conversationId: string;
} | null = null;
while (inserted < input.limit) {
const pageSize = Math.max(25, Math.min(200, input.limit - inserted));
const pageCursor: typeof cursor = cursor;
const rows: SafeNativeChatProgressCandidate[] = await db
return async (pageSize: number, pageCursor: NativeProgressCursor | null): Promise<SafeNativeChatProgressCandidate[]> => await db
.select({
eventId: heartbeatRunEvents.id,
eventSeq: heartbeatRunEvents.seq,
@@ -118,7 +106,9 @@ export async function enqueueSafeNativeChatProgress(
inArray(heartbeatRunEvents.eventType, [
...SAFE_NATIVE_CHAT_PROGRESS_EVENT_TYPES,
]),
gte(heartbeatRunEvents.createdAt, input.since),
gte(heartbeatRunEvents.createdAt, since),
scope ? eq(heartbeatRuns.companyId, scope.companyId) : undefined,
scope?.issueId ? eq(chatConversations.issueId, scope.issueId) : undefined,
sql`${heartbeatRunEvents.createdAt} >= ${basePublication.createdAt} + interval '20 seconds'`,
notExists(
db
@@ -147,6 +137,31 @@ export async function enqueueSafeNativeChatProgress(
asc(chatConversations.id),
)
.limit(pageSize);
}
export async function loadNativeChatProgressScopes(db: Db, scope?: ChatRunProjectionScope, signal?: AbortSignal) {
const pages = nativeProgressPages(db, chatRunProjectionWindow({}).since, scope);
const scopes = new Map<string, { companyId: string; issueId: string }>();
let cursor: NativeProgressCursor | null = null;
for (;;) {
signal?.throwIfAborted();
const rows = await pages(200, cursor);
for (const row of rows) scopes.set(`${row.companyId}:${row.issueId}`, { companyId: row.companyId, issueId: row.issueId });
if (rows.length < 200) return [...scopes.values()];
cursor = rows[rows.length - 1];
}
}
/** A suppressed event is settled; lock contention still owes another attempt. */
export async function reconcileNativeChatProgress(db: Db, options: NativeProgressInput = {}) {
const input = chatRunProjectionWindow(options);
const pages = nativeProgressPages(db, input.since, options.scope);
let inserted = 0, contended = false;
let cursor: NativeProgressCursor | null = null;
while (inserted < input.limit) {
options.signal?.throwIfAborted();
const pageSize = Math.max(25, Math.min(200, input.limit - inserted));
const rows = await pages(pageSize, cursor);
if (rows.length === 0) break;
const lastRow = rows.at(-1)!;
cursor = {
@@ -156,6 +171,7 @@ export async function enqueueSafeNativeChatProgress(
};
for (const row of rows) {
options.signal?.throwIfAborted();
if (inserted >= input.limit) break;
const result = await db.transaction(async (tx) => {
// Keep the normal issue -> run order used by native finalization. The
@@ -174,7 +190,12 @@ export async function enqueueSafeNativeChatProgress(
)
.for("update", { skipLocked: true })
.then((currentRows) => currentRows[0] ?? null);
if (!currentIssue) return 0;
if (!currentIssue) {
const [existing] = await tx.select({ id: issues.id }).from(issues)
.where(and(eq(issues.id, row.issueId), eq(issues.companyId, row.companyId)));
contended ||= Boolean(existing);
return 0;
}
// Native event admission serializes on the run row. Taking the same
// short lock makes terminal status, a later event, or a final comment
// win before this non-terminal projection can be inserted.
@@ -192,7 +213,15 @@ export async function enqueueSafeNativeChatProgress(
)
.for("update", { skipLocked: true })
.then((currentRows) => currentRows[0] ?? null);
if (!currentRun) return 0;
if (!currentRun) {
const [existing] = await tx.select({ id: heartbeatRuns.id }).from(heartbeatRuns).where(and(
eq(heartbeatRuns.id, row.runId), eq(heartbeatRuns.companyId, row.companyId),
eq(heartbeatRuns.agentId, row.agentId), eq(heartbeatRuns.runtimeMode, "native"),
eq(heartbeatRuns.status, "running"),
));
contended ||= Boolean(existing);
return 0;
}
const destination = await tx
.select({ id: chatConversations.id })
@@ -389,5 +418,10 @@ export async function enqueueSafeNativeChatProgress(
}
if (rows.length < pageSize) break;
}
return inserted;
return { inserted, contended };
}
/** Explicit/manual callers retain the original count-only contract. */
export async function enqueueSafeNativeChatProgress(db: Db, options: NativeProgressInput = {}): Promise<number> {
return (await reconcileNativeChatProgress(db, options)).inserted;
}
@@ -1,9 +1,13 @@
import { notifyChatNativeProgress } from "./chat-native-progress-notifications.js";
import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js";
/** Lineage changes can authorize descendants, so invalidate the issue rather
* than just the run whose row changed. Configuration may invalidate a company. */
export type ChatRunMilestoneScope = { companyId: string; issueId?: string };
export function notifyChatRunMilestones(tx: object, scope: ChatRunMilestoneScope) {
return notifyDeliveryWork(tx, DELIVERY_QUEUES.chatRunMilestones, JSON.stringify({ companyId: scope.companyId, issueId: scope.issueId }));
export async function notifyChatRunMilestones(tx: object, scope: ChatRunMilestoneScope) {
await notifyDeliveryWork(tx, DELIVERY_QUEUES.chatRunMilestones, JSON.stringify({ companyId: scope.companyId, issueId: scope.issueId }));
// Lifecycle and coarse native progress share lineage and authority. Extend
// this eligibility boundary once rather than adding wrappers to its callers.
await notifyChatNativeProgress(tx, scope);
}
@@ -17,3 +17,5 @@ export function chatRunProjectionWindow(input: { since?: Date; limit?: number })
limit: Math.max(1, Math.min(input.limit ?? 200, 1_000)),
};
}
export type ChatRunProjectionScope = { companyId: string; issueId?: string };
@@ -1,3 +1,4 @@
import { notifyChatNativeProgress } from "./chat-native-progress-notifications.js";
import { allowsAutomaticRunPublication, chatRunProjectionWindow } from "./chat-run-publication-policy.js";
import { enqueueSafeNativeChatProgress } from "./chat-native-progress.js";
export { enqueueSafeNativeChatProgress } from "./chat-native-progress.js";
@@ -646,6 +647,9 @@ export async function enqueueChatRunLifecycleMilestones(
if (explicitlyAuthoredPublication.length > 0) continue;
}
const result = await db.transaction(async (tx) => {
if (milestone === "working") {
await notifyChatNativeProgress(tx, { companyId: row.companyId, issueId: row.issueId });
}
await notifyChatPublicationWork(tx);
return tx
.insert(chatPublications)
@@ -54,6 +54,7 @@ export const DELIVERY_QUEUES = {
chatVerification: "chat-verification",
chatInbound: "chat-inbound",
chatPublications: "chat-publications",
chatNativeProgress: "chat-native-progress",
chatRunMilestones: "chat-run-milestones",
chatReceipts: "chat-slack-receipts",
browser: "browser-use",
+10 -2
View File
@@ -1,3 +1,6 @@
import { isSafeNativeChatProgressEventType } from "./safe-native-chat-progress.js";
import { notifyChatNativeProgress } from "./chat-native-progress-notifications.js";
import { boundRunIssueIdSql } from "./run-task-binding.js";
import { and, asc, eq, getTableColumns, or, sql } from "drizzle-orm";
import type { Db } from "@paperclipai/db";
import { heartbeatRunEvents, heartbeatRuns } from "@paperclipai/db";
@@ -68,7 +71,8 @@ export async function appendHeartbeatRunEvent(
db: Db,
input: AppendHeartbeatRunEventInput,
): Promise<AppendHeartbeatRunEventResult> {
if (!input.nativeSource && !input.retryExhaustion) {
const projectsProgress = isSafeNativeChatProgressEventType(input.eventType);
if (!input.nativeSource && !input.retryExhaustion && !projectsProgress) {
const allocated = db.$with("allocated_run_event", {}).as(sql`
update ${heartbeatRuns}
set next_event_seq = ${heartbeatRuns.nextEventSeq} + 1,
@@ -94,7 +98,8 @@ export async function appendHeartbeatRunEvent(
}
return db.transaction(async (tx) => {
const run = await tx
.select({ companyId: heartbeatRuns.companyId, agentId: heartbeatRuns.agentId })
.select({ companyId: heartbeatRuns.companyId, agentId: heartbeatRuns.agentId,
runtimeMode: heartbeatRuns.runtimeMode, issueId: boundRunIssueIdSql() })
.from(heartbeatRuns)
.where(eq(heartbeatRuns.id, input.runId))
.for("update")
@@ -165,6 +170,9 @@ export async function appendHeartbeatRunEvent(
}
}
if (projectsProgress && run.runtimeMode === "native" && run.issueId) {
await notifyChatNativeProgress(tx, { companyId: run.companyId, issueId: run.issueId });
}
const seq = await allocateHeartbeatRunEventSeq(
tx as unknown as Db,
input.runId,
@@ -1,3 +1,5 @@
import { isSafeNativeChatProgressEventType } from "../safe-native-chat-progress.js";
import { notifyChatNativeProgress } from "../chat-native-progress-notifications.js";
import { createHash } from "node:crypto";
import { and, desc, eq, inArray, lt } from "drizzle-orm";
@@ -290,6 +292,9 @@ export class NativeRunCoordinatorStore {
throw new Error("native_event_source_gap");
}
if (isSafeNativeChatProgressEventType(event.eventType)) {
await notifyChatNativeProgress(tx, { companyId: this.#binding.companyId, issueId: this.#binding.issueId });
}
const cursor = run.nextEventSeq;
const [inserted] = await tx
.insert(heartbeatRunEvents)