fix(server): schedule inbound voice cleanup from durable obligations

Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
DottaandPaperclip committed 2026-10-25 04:44:37 -05:00
1 parent 95f439ab49
commit 4b4bf2d8af
20 files changed
+791 -23

No files matched your search

@@ -86,6 +86,7 @@ describe("createChatReconciliationCoordinator", () => {
const source = readFileSync(new URL("../app.ts", import.meta.url), "utf8");
expect(source).toContain("chatChannels.voice.registerRecoveryWork(deliveryWork,");
expect(source).toContain("chatChannels.voice.registerReplyWork(deliveryWork,");
expect(source).toContain("chatChannels.voice.inbound.registerCleanupWork(deliveryWork,");
expect(source).toContain("chatChannels.voice.reconcileMaintenance()");
expect(source).not.toContain("chatChannels.voice.reconcile()");
});
@@ -0,0 +1,194 @@
import { readFile } from "node:fs/promises";
import { chatCompanyDeletion } from "../services/chat-channels/company-deletion.js";
import { randomUUID } from "node:crypto";
import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
import { eq, sql } from "drizzle-orm";
import { agents, chatConversations, chatEndpoints, chatVoiceInboundCalls, chatVoiceSessions, companies, createDb, issues, toolApplications, toolConnections } from "@paperclipai/db";
import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js";
import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js";
import { createDeliveryWorkCoordinator } from "../services/delivery-work-coordinator.js";
import { idleWorkSnapshot } from "../services/task-admission.js";
import { voiceInboundCleanup } from "../services/voice/voice-inbound-cleanup.js";
import { notifyVoiceInbound } from "../services/voice/voice-inbound-notifications.js";
import { notifyVoiceChange } from "../services/voice/voice-recovery-notifications.js";
const external = resolveExternalTestDatabaseUrl("PAPERCLIP_VOICE_INBOUND_TEST_DATABASE_URL");
const support = external ? { supported: true } : await getEmbeddedPostgresTestSupport();
(support.supported ? describe : describe.skip)("inbound voice cleanup lifecycle", () => {
let temporary: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>> | undefined;
let db: ReturnType<typeof createDb>, companyId: string, endpointId: string, agentId: string, issueId: string, conversationId: string;
let coordinator: ReturnType<typeof createDeliveryWorkCoordinator> | undefined;
let enabled = true;
const errors: unknown[] = [], fixtureCompanies: string[] = [];
const terminate = vi.fn(async (_call: typeof chatVoiceInboundCalls.$inferSelect) => true);
beforeAll(async () => {
if (!external) temporary = await startEmbeddedPostgresTestDatabase("voice-inbound-cleanup-");
db = createDb(external ?? temporary!.connectionString, { idleTimeoutSeconds: 0, maxLifetimeSeconds: 0 });
}, 60_000);
async function fixture() {
const companyId = randomUUID(), endpointId = randomUUID(), agentId = randomUUID(), issueId = randomUUID(), conversationId = randomUUID();
fixtureCompanies.push(companyId);
const applicationId = randomUUID(), connectionId = randomUUID();
await db.insert(companies).values({ id: companyId, name: "Inbound cleanup", issuePrefix: `V${companyId.slice(0, 7)}` });
await db.insert(agents).values({ id: agentId, companyId, name: "Voice", adapterType: "paperclip_runner" });
await db.insert(issues).values({ id: issueId, companyId, title: "Voice", assigneeAgentId: agentId, status: "in_progress" });
await db.insert(toolApplications).values({ id: applicationId, companyId, name: "Voice", type: "chat" });
await db.insert(toolConnections).values({ id: connectionId, companyId, applicationId, name: "Voice", uid: connectionId, transport: "voice", connectionPurpose: "channel", enabled: true, status: "active" });
await db.insert(chatEndpoints).values({ id: endpointId, companyId, connectionId, assignedAgentId: agentId, publicId: endpointId, provider: "speko", status: "active", setup: { step: "complete", runtimeGeneration: 1 } });
await db.insert(chatConversations).values({ id: conversationId, companyId, endpointId, issueId, externalConversationId: conversationId, externalThreadId: conversationId, externalLabel: "Voice", isDirectMessage: true });
return { companyId, endpointId, agentId, issueId, conversationId };
}
beforeEach(async () => {
enabled = true; errors.length = 0; terminate.mockReset().mockResolvedValue(true);
({ companyId, endpointId, agentId, issueId, conversationId } = await fixture());
vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"], shouldAdvanceTime: true });
});
afterEach(async () => {
await coordinator?.stop(); coordinator = undefined; vi.useRealTimers(); vi.restoreAllMocks();
for (const companyId of fixtureCompanies.splice(0)) {
await db.delete(chatVoiceInboundCalls).where(eq(chatVoiceInboundCalls.companyId, companyId));
await db.delete(chatVoiceSessions).where(eq(chatVoiceSessions.companyId, companyId));
await db.delete(chatConversations).where(eq(chatConversations.companyId, companyId));
await db.delete(chatEndpoints).where(eq(chatEndpoints.companyId, companyId));
await db.delete(issues).where(eq(issues.companyId, companyId));
await db.delete(agents).where(eq(agents.companyId, companyId));
await db.delete(companies).where(eq(companies.id, companyId));
}
expect(errors).toEqual([]);
});
afterAll(async () => { await db?.$client.end({ timeout: 1 }); await temporary?.cleanup(); });
async function admission(patch: Partial<typeof chatVoiceInboundCalls.$inferInsert> = {}, writer = db) {
const id = randomUUID();
return writer.transaction(async tx => {
await notifyVoiceInbound(tx, { kind: "admission", id });
return (await tx.insert(chatVoiceInboundCalls).values({ id, companyId, endpointId, providerSessionId: randomUUID(), state: "awaiting_approval", approvalCode: "123456", generation: 1, credentialFingerprint: "test", toolTokenHash: "a".repeat(64), requestFingerprint: id, expiresAt: new Date(Date.now() - 1), ...patch }).returning())[0];
});
}
async function owner(call: typeof chatVoiceInboundCalls.$inferSelect, patch: Partial<typeof chatVoiceSessions.$inferInsert> = {}) {
return (await db.insert(chatVoiceSessions).values({ companyId, endpointId, conversationId, issueId, assignedAgentId: agentId, callerId: call.approvedByUserId ?? `guest:${call.id}`, callerAuthority: call.callerAuthority === "member" ? "member" : "guest_intake", approvedByUserId: call.approvedByUserId,
mode: "inbound_phone", state: "active", generation: call.generation, credentialFingerprint: call.credentialFingerprint, providerSessionId: call.providerSessionId, toolTokenHash: call.toolTokenHash, idempotencyKey: `inbound:${call.id}`, requestFingerprint: call.requestFingerprint, expiresAt: new Date(Date.now() + 600_000), ...patch }).returning())[0];
}
async function start(service = voiceInboundCleanup(db, terminate)) {
coordinator = createDeliveryWorkCoordinator({ owner: db, canRun: () => enabled, canReconcile: () => true, onError: error => errors.push(error) });
await service.register(coordinator, () => enabled).ready;
return service;
}
async function settled() {
await vi.waitFor(async () => { await vi.advanceTimersByTimeAsync(1000); expect(idleWorkSnapshot().active).toBe(0); expect(coordinator!.nextWakeAt()).toBeNull(); }, { timeout: 5000 });
}
async function quiet(ms = 120_000) {
const reads = vi.spyOn(db, "select"), writes = vi.spyOn(db, "transaction");
try { await vi.advanceTimersByTimeAsync(ms); expect(reads).not.toHaveBeenCalled(); expect(writes).not.toHaveBeenCalled(); }
finally { reads.mockRestore(); writes.mockRestore(); }
}
it("leaves configured idle and confirmed history quiet", async () => {
await admission({ state: "ended", cleanupConfirmedAt: new Date() });
await start(); await settled(); await quiet(); expect(terminate).not.toHaveBeenCalled();
});
it("prevents company deletion from erasing an unfinished provider cleanup", async () => {
const call = await admission();
await expect(db.transaction(tx => chatCompanyDeletion.deleteCompanyData(tx, companyId))).rejects.toMatchObject({ status: 409 });
expect(await db.select().from(chatVoiceInboundCalls).where(eq(chatVoiceInboundCalls.id, call.id))).toHaveLength(1);
await start(); await settled();
await db.transaction(tx => chatCompanyDeletion.deleteCompanyData(tx, companyId));
expect(await db.select().from(chatVoiceInboundCalls).where(eq(chatVoiceInboundCalls.id, call.id))).toHaveLength(0);
});
it("backfills only provable legacy termination and preserves rejected cleanup debt", async () => {
const confirmed = await admission({ state: "ended" });
const uncertain = await admission({ state: "ended", approvedByUserId: randomUUID(), callerAuthority: "member" });
const migration = await readFile(new URL("../../../packages/db/src/migrations/0388_lying_trish_tilby.sql", import.meta.url), "utf8");
const backfill = migration.split("--> statement-breakpoint").at(-1)!;
await db.execute(sql.raw(backfill)); await db.execute(sql.raw(backfill));
expect((await db.select().from(chatVoiceInboundCalls).where(eq(chatVoiceInboundCalls.id, confirmed.id)))[0].cleanupConfirmedAt).not.toBeNull();
expect((await db.select().from(chatVoiceInboundCalls).where(eq(chatVoiceInboundCalls.id, uncertain.id)))[0].cleanupConfirmedAt).toBeNull();
await start(); await settled(); expect(terminate).toHaveBeenCalledOnce();
expect(terminate.mock.calls[0][0].id).toBe(uncertain.id); await quiet();
});
it("joins provider cleanup at shutdown and saves a restart deadline for uncertainty", async () => {
const call = await admission(); let release!: (confirmed: boolean) => void;
terminate.mockImplementationOnce(() => new Promise<boolean>(resolve => { release = resolve; }));
const service = voiceInboundCleanup(db, terminate);
coordinator = createDeliveryWorkCoordinator({ owner: db, canRun: () => true, onError: error => errors.push(error) });
const worker = service.register(coordinator, () => true);
await vi.waitFor(() => expect(terminate).toHaveBeenCalledOnce());
let stopped = false;
const stopping = coordinator.stop().then(() => { stopped = true; });
try {
await vi.advanceTimersByTimeAsync(1000); expect(stopped).toBe(false);
release(false); await stopping; await worker.ready;
expect((await db.select().from(chatVoiceInboundCalls).where(eq(chatVoiceInboundCalls.id, call.id)))[0].cleanupNextAttemptAt).not.toBeNull();
await start(); await settled(); expect(terminate).toHaveBeenCalledTimes(2);
} finally { release(false); await stopping; }
});
it("waits for the original expiry and reconstructs it on restart", async () => {
const due = Date.now() + 60_000; await admission({ expiresAt: new Date(due) });
await start(); expect(coordinator!.nextWakeAt()).toBe(due); await quiet(10_000);
await coordinator!.stop(); await start(); expect(coordinator!.nextWakeAt()).toBe(due);
await vi.advanceTimersByTimeAsync(due - Date.now()); await settled(); expect(terminate).toHaveBeenCalledOnce(); await quiet();
});
it("waits for outer commit and ignores a rolled-back nested producer", async () => {
await start(); await settled();
await expect(db.transaction(async tx => { await admission({}, tx as unknown as typeof db); await vi.advanceTimersByTimeAsync(2000); expect(terminate).not.toHaveBeenCalled(); throw new Error("rollback"); })).rejects.toThrow("rollback");
await settled(); await quiet();
await db.transaction(async tx => { await admission({}, tx as unknown as typeof db); await vi.advanceTimersByTimeAsync(2000); expect(terminate).not.toHaveBeenCalled(); });
await settled(); expect(terminate).toHaveBeenCalledOnce(); await quiet();
});
it("retains unconfirmed termination across restart without early requests", async () => {
terminate.mockResolvedValueOnce(false);
const call = await admission(); await start();
const [pending] = await db.select().from(chatVoiceInboundCalls).where(eq(chatVoiceInboundCalls.id, call.id));
expect(pending.cleanupConfirmedAt).toBeNull(); expect(pending.cleanupNextAttemptAt).not.toBeNull();
await coordinator!.stop(); await start(); await quiet(10_000); expect(terminate).toHaveBeenCalledOnce();
await vi.advanceTimersByTimeAsync(pending.cleanupNextAttemptAt!.getTime() - Date.now()); await settled();
expect(terminate).toHaveBeenCalledTimes(2); await quiet();
});
it("rechecks ownership after discovery rather than ending a newly bound call", async () => {
const call = await admission({ state: "approving", approvedByUserId: randomUUID(), callerAuthority: "member", requestedIssueId: issueId });
const service = voiceInboundCleanup(db, terminate), [candidate] = await service.load();
const session = await owner(call);
await db.update(chatVoiceInboundCalls).set({ state: "approved", sessionId: session.id }).where(eq(chatVoiceInboundCalls.id, call.id));
expect(await service.check(candidate.id)).toBeNull(); expect(terminate).not.toHaveBeenCalled();
});
it.each([false, true])("repairs older interrupted handoffs (existing guest binding=%s)", async bound => {
const call = await admission({ state: "approving", approvedByUserId: randomUUID(), callerAuthority: "member", requestedIssueId: issueId });
const session = await owner(call);
if (bound) await db.update(chatVoiceInboundCalls).set({ sessionId: session.id }).where(eq(chatVoiceInboundCalls.id, call.id));
await start(); await settled(); expect(terminate).not.toHaveBeenCalled();
expect((await db.select().from(chatVoiceInboundCalls).where(eq(chatVoiceInboundCalls.id, call.id)))[0]).toMatchObject({ state: "approved", sessionId: session.id });
await quiet();
});
it("hands rejected bound calls to live recovery and settles from its confirmation", async () => {
const call = await admission({ state: "denied" }), session = await owner(call);
await db.update(chatVoiceInboundCalls).set({ sessionId: session.id }).where(eq(chatVoiceInboundCalls.id, call.id));
await start(); await settled(); expect(terminate).not.toHaveBeenCalled();
expect((await db.select().from(chatVoiceSessions).where(eq(chatVoiceSessions.id, session.id)))[0].state).toBe("ending");
await quiet();
await db.transaction(async tx => {
await notifyVoiceChange(tx, { kind: "session", id: session.id });
await tx.update(chatVoiceSessions).set({ state: "ended", endedAt: new Date() }).where(eq(chatVoiceSessions.id, session.id));
});
await settled();
expect((await db.select().from(chatVoiceInboundCalls).where(eq(chatVoiceInboundCalls.id, call.id)))[0].cleanupConfirmedAt).not.toBeNull();
await quiet(); expect(terminate).not.toHaveBeenCalled();
});
it("retains first work committed during drain without passive SQL", async () => {
await start(); await settled(); enabled = false;
await admission(); await quiet(60_000); expect(terminate).not.toHaveBeenCalled();
enabled = true; await settled(); expect(terminate).toHaveBeenCalledOnce(); await quiet();
});
it("pages backlog fairly while a failed provider retains its own retry", async () => {
const broken = await admission();
for (let i = 0; i < 104; i++) await admission();
const other = await fixture(), last = await admission({ companyId: other.companyId, endpointId: other.endpointId });
let failed = true;
terminate.mockImplementation(async call => { if (call.id === broken.id && failed) throw new Error("provider unavailable"); return true; });
await start();
expect(terminate.mock.calls.slice(0, 25).some(([call]) => call.id === last.id)).toBe(true);
await vi.waitFor(async () => {
await vi.advanceTimersByTimeAsync(1000);
expect(terminate.mock.calls.filter(([call]) => call.id !== broken.id)).toHaveLength(105);
}, { timeout: 5000 });
expect(errors).toHaveLength(1); failed = false;
await settled(); errors.length = 0; await quiet();
});
});
@@ -78,6 +78,91 @@ const support = external ? { supported: true } : await getEmbeddedPostgresTestSu
return { companyId, endpointId, agentId, connectionId, issueId, conversationId, callerId, sessionId, store, request, reserved, accept, toolInput, publication };
}
it("wakes inbound cleanup from a real signed pre-call and retains an unconfirmed end", async () => {
const f = await inboundFixture();
const ended = f.sign({ type: "call.status", call_id: f.call.providerSessionId, status: "ended" });
await f.service.inbound.lifecycle(f.endpoint.publicId, ended.body, ended.headers);
vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"], shouldAdvanceTime: true });
const onError = vi.fn(), coordinator = createDeliveryWorkCoordinator({ owner: db, canRun: () => true, onError });
const settle = () => vi.waitFor(async () => {
await vi.advanceTimersByTimeAsync(1000); expect(idleWorkSnapshot().active).toBe(0);
}, { timeout: 5000 });
const quiet = async (ms: number) => {
const reads = vi.spyOn(db, "select"), transactions = vi.spyOn(db, "transaction");
try { await vi.advanceTimersByTimeAsync(ms); expect(reads).not.toHaveBeenCalled(); expect(transactions).not.toHaveBeenCalled(); }
finally { reads.mockRestore(); transactions.mockRestore(); }
};
try {
await f.service.inbound.registerCleanupWork(coordinator, () => true).ready;
await settle(); expect(coordinator.nextWakeAt()).toBeNull(); await quiet(10_000);
const providerSessionId = randomUUID(), incoming = f.sign({ ...f.event, call_id: providerSessionId, session_id: providerSessionId });
await f.service.inbound.lifecycle(f.endpoint.publicId, incoming.body, incoming.headers);
await settle();
const [call] = await db.select().from(chatVoiceInboundCalls).where(eq(chatVoiceInboundCalls.providerSessionId, providerSessionId));
expect(coordinator.nextWakeAt()).toBe(call.expiresAt.getTime());
await quiet(60_000); expect(f.transport.inspectSession).not.toHaveBeenCalled();
// Disabling new voice work must not erase an existing physical cleanup.
await instanceSettingsService(db).updateExperimental({ enableChatConnectors: false });
await settle();
f.credentials.mockResolvedValue({ apiKey: "rotated-key-in-original-account", agentId: "agent_test" });
f.transport.endSession.mockResolvedValueOnce({ confirmed: false });
await vi.advanceTimersByTimeAsync(call.expiresAt.getTime() - Date.now()); await settle();
expect(f.transport.endSession).toHaveBeenCalledWith(providerSessionId);
const [pending] = await db.select().from(chatVoiceInboundCalls).where(eq(chatVoiceInboundCalls.id, call.id));
expect(pending.cleanupConfirmedAt).toBeNull();
expect(coordinator.nextWakeAt()).toBe(pending.cleanupNextAttemptAt!.getTime());
await quiet(10_000); expect(f.transport.endSession).toHaveBeenCalledOnce();
await vi.advanceTimersByTimeAsync(pending.cleanupNextAttemptAt!.getTime() - Date.now()); await settle();
await vi.waitFor(async () => { await vi.advanceTimersByTimeAsync(1000); expect(coordinator.nextWakeAt()).toBeNull(); });
expect(f.transport.endSession).toHaveBeenCalledTimes(2); await quiet(120_000);
expect(onError).not.toHaveBeenCalled();
expect(f.transport.createPhoneSession).not.toHaveBeenCalled();
} finally { await coordinator.stop(); vi.useRealTimers(); await instanceSettingsService(db).updateExperimental({ enableChatConnectors: true }); }
});
it("retains rejected inbound cleanup across provider failure and history retention", async () => {
const f = await inboundFixture(true);
f.transport.endSession.mockRejectedValueOnce(new Error("provider unavailable"));
const result = await f.service.inbound.decide(f.companyId, f.endpointId, f.call.id, f.input.caller,
{ approve: false, approvalCode: f.call.approvalCode });
expect(result.state).toBe("ended");
const [pending] = await db.select().from(chatVoiceInboundCalls).where(eq(chatVoiceInboundCalls.id, f.call.id));
expect(pending.cleanupConfirmedAt).toBeNull(); expect(pending.cleanupNextAttemptAt).not.toBeNull();
await db.update(chatVoiceInboundCalls).set({ expiresAt: new Date(0), updatedAt: new Date(0) }).where(eq(chatVoiceInboundCalls.id, f.call.id));
await f.service.inbound.pruneHistory();
expect(await db.select().from(chatVoiceInboundCalls).where(eq(chatVoiceInboundCalls.id, f.call.id))).toHaveLength(1);
vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"], shouldAdvanceTime: true });
const onError = vi.fn(), coordinator = createDeliveryWorkCoordinator({ owner: db, canRun: () => true, onError });
try {
await f.service.inbound.registerCleanupWork(coordinator, () => true).ready;
expect(coordinator.nextWakeAt()).toBe(pending.cleanupNextAttemptAt!.getTime());
await vi.advanceTimersByTimeAsync(pending.cleanupNextAttemptAt!.getTime() - Date.now());
await vi.waitFor(async () => { await vi.advanceTimersByTimeAsync(1000); expect(idleWorkSnapshot().active).toBe(0); expect(coordinator.nextWakeAt()).toBeNull(); });
expect(f.transport.endSession).toHaveBeenCalledTimes(2);
expect((await db.select().from(chatVoiceInboundCalls).where(eq(chatVoiceInboundCalls.id, f.call.id)))[0].cleanupConfirmedAt).not.toBeNull();
expect(onError).not.toHaveBeenCalled();
} finally { await coordinator.stop(); vi.useRealTimers(); }
});
it("does not clean up an inbound call through a different provider account", async () => {
const f = await inboundFixture();
f.credentials.mockResolvedValue({ apiKey: "key-for-another-account", agentId: "other-agent" });
f.transport.verifyAgent.mockResolvedValue({ id: "other-agent", organizationId: "another-account", name: "Other voice" });
await f.service.inbound.decide(f.companyId, f.endpointId, f.call.id, f.input.caller,
{ approve: false, approvalCode: f.call.approvalCode });
expect(f.transport.inspectSession).not.toHaveBeenCalled(); expect(f.transport.endSession).not.toHaveBeenCalled();
const [pending] = await db.select().from(chatVoiceInboundCalls).where(eq(chatVoiceInboundCalls.id, f.call.id));
expect(pending.cleanupConfirmedAt).toBeNull(); expect(pending.cleanupNextAttemptAt).not.toBeNull();
// A usable key for the saved account restores cleanup without redialing.
f.credentials.mockResolvedValue({ apiKey: "restored-account-key", agentId: "agent_test" });
f.transport.verifyAgent.mockResolvedValue({ id: "agent_test", organizationId: `org_${f.companyId}`, name: "Test voice" });
await db.update(chatVoiceInboundCalls).set({ cleanupNextAttemptAt: new Date(0) }).where(eq(chatVoiceInboundCalls.id, f.call.id));
await f.service.inbound.reconcile(1000);
expect(f.transport.endSession).toHaveBeenCalledWith(f.call.providerSessionId);
expect((await db.select().from(chatVoiceInboundCalls).where(eq(chatVoiceInboundCalls.id, f.call.id)))[0].cleanupConfirmedAt).not.toBeNull();
expect(f.transport.createPhoneSession).not.toHaveBeenCalled();
});
it("restores blocked replies through the production live-call recovery wiring", async () => {
const f = await pushFixture();
await db.update(companyMemberships).set({ status: "suspended" }).where(eq(companyMemberships.principalId, f.callerId));
@@ -1085,6 +1170,7 @@ const support = external ? { supported: true } : await getEmbeddedPostgresTestSu
await instanceSettingsService(db).updateExperimental({enableChatConnectors: true, enableIsolatedWorkspaces: guestIntake});
const signingSecret = `whsec_${Buffer.alloc(32, 7).toString("base64")}`;
f.credentials.mockResolvedValue({apiKey: "server-only-key", agentId: "agent_test", signingSecret} as never);
f.transport.verifyAgent.mockResolvedValue({ id: "agent_test", organizationId: `org_${f.companyId}`, name: "Test voice" });
await db.update(chatEndpoints).set({providerAccountId: `org_${f.companyId}`, botExternalId: "agent_test"}).where(eq(chatEndpoints.id, f.endpointId));
await db.insert(chatVoicePhoneLines).values({companyId: f.companyId, endpointId: f.endpointId, providerNumberId: randomUUID(), phoneNumber: "+12015550123", enabled: true, guestIntake, lowTrustEnvironmentId: sandboxId ?? null});
const [line] = await db.select().from(chatVoicePhoneLines).where(eq(chatVoicePhoneLines.endpointId, f.endpointId));