fix(server): schedule voice call reports from saved deadlines

Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
DottaandPaperclip committed 2026-10-25 05:33:49 -05:00
1 parent 53aa3bd603
commit f271cd1bda
13 files changed
+579 -86

No files matched your search

@@ -42,6 +42,7 @@ export const DELIVERY_QUEUES = {
chatTelegramMaintenance: "chat-telegram-maintenance",
chatReceivers: "chat-receivers",
voiceSessionRecovery: "voice-session-recovery",
voiceReports: "voice-call-reports",
voiceInboundRetention: "voice-inbound-retention",
voiceInboundCleanup: "voice-inbound-cleanup",
voiceReplies: "voice-reply-delivery",
@@ -0,0 +1,126 @@
import { and, eq, gt, gte, inArray, isNotNull, lte, ne, or, sql } from "drizzle-orm";
import { activityLog, chatEndpoints, chatVoiceReports, chatVoiceSessions, issues, type Db } from "@paperclipai/db";
import { logger } from "../../middleware/logger.js";
import { documentService } from "../documents.js";
import { registerScopedDeadlineWork } from "../scoped-deadline-work.js";
import type { createDeliveryWorkCoordinator } from "../delivery-work-coordinator.js";
import { DELIVERY_QUEUES } from "../delivery-work-notifications.js";
import { voiceRecoveryScope, type VoiceRecoveryScope } from "./voice-recovery-notifications.js";
import { notifyVoiceReports } from "./voice-report-notifications.js";
import type { VoiceSessionRow } from "./voice-session-store.js";
import type { SpekoCallReport } from "./speko-call-report.js";
import { SpekoProviderError } from "./speko-provider.js";
type Entry = { id: string; companyId: string; endpointId: string; connectionId: string; dueAt: number };
type Report = SpekoCallReport;
const WINDOW_MS = 24 * 60 * 60 * 1000;
const eligible = (now: number) => and(inArray(chatVoiceSessions.state, ["ended", "failed", "expired"]),
isNotNull(chatVoiceSessions.providerSessionId), gte(chatVoiceSessions.endedAt, new Date(now - WINDOW_MS)));
/** Reports may change for 24 hours after a call. Available is a presentation
* state, not a promise of immutable provider data. Keep that finite obligation. */
export function voiceCallReports(db: Db, fetchReport: (session: VoiceSessionRow) => Promise<Report>) {
async function load(scope?: VoiceRecoveryScope, signal?: AbortSignal): Promise<Entry[]> {
const entries: Entry[] = []; let cursor: string | undefined;
for (;;) {
signal?.throwIfAborted();
const rows = await db.select({ id: chatVoiceSessions.id, companyId: chatVoiceSessions.companyId,
endpointId: chatVoiceSessions.endpointId, connectionId: chatEndpoints.connectionId,
dueAt: sql<string>`least(coalesce(ceil(extract(epoch from ${chatVoiceReports.nextCheckAt}) * 1000), 0), floor(extract(epoch from ${chatVoiceSessions.endedAt}) * 1000) + ${WINDOW_MS} + 1)`,
}).from(chatVoiceSessions)
.innerJoin(chatEndpoints, and(eq(chatEndpoints.id, chatVoiceSessions.endpointId), eq(chatEndpoints.companyId, chatVoiceSessions.companyId)))
.leftJoin(chatVoiceReports, and(eq(chatVoiceReports.sessionId, chatVoiceSessions.id), eq(chatVoiceReports.companyId, chatVoiceSessions.companyId)))
.where(and(eligible(Date.now()),
scope?.kind === "session" ? eq(chatVoiceSessions.id, scope.id) : undefined,
scope?.kind === "endpoint" ? eq(chatVoiceSessions.endpointId, scope.id) : undefined,
scope?.kind === "connection" ? eq(chatEndpoints.connectionId, scope.id) : undefined,
scope?.kind === "company" ? eq(chatVoiceSessions.companyId, scope.id) : undefined,
cursor ? gt(chatVoiceSessions.id, cursor) : undefined)).orderBy(chatVoiceSessions.id).limit(100);
entries.push(...rows.map(row => ({ ...row, dueAt: Number(row.dueAt) })));
if (rows.length < 100) return entries;
cursor = rows[rows.length - 1].id;
}
}
async function projectTranscript(session: VoiceSessionRow) {
// Keep both actual spoken sides on this public call's own task. This
// does not publish a document or grant access to any other task.
if (session.callerAuthority !== "guest_intake") return;
await db.transaction(async tx => {
await tx.execute(sql`select pg_advisory_xact_lock(hashtextextended(${`voice-transcript:${session.id}`}, 0))`);
const [retained] = await tx.select().from(chatVoiceReports).where(and(eq(chatVoiceReports.companyId, session.companyId), eq(chatVoiceReports.sessionId, session.id)));
if (!retained?.transcript.length) return;
const [task] = await tx.select({sourceTrust: issues.sourceTrust}).from(issues).where(and(eq(issues.companyId, session.companyId), eq(issues.id, session.issueId)));
if (task?.sourceTrust?.preset !== "low_trust_review" || task.sourceTrust.disposition !== "quarantined") return;
const key = `voice-transcript-${session.id}`;
const body = retained.transcript.map(turn => `**${turn.speaker === "caller" ? "Caller" : "Agent"}** · ${turn.startedAt}${turn.interrupted ? " · interrupted" : ""}\n\n${turn.text}`).join("\n\n---\n\n");
const docs = documentService(tx as unknown as Db), previous = await docs.getIssueDocumentByKey(session.issueId, key);
if (previous?.body === body) return;
await docs.upsertIssueDocument({issueId: session.issueId, key, baseRevisionId: previous?.latestRevisionId, title: "Phone conversation transcript", format: "markdown", body, sourceTrust: task.sourceTrust, changeSummary: "Speko spoken conversation, including caller and agent turns"});
await tx.insert(activityLog).values({companyId: session.companyId, actorType: "system", actorId: "speko-report", action: "voice.transcript.updated", entityType: "issue", entityId: session.issueId, details: {sessionId: session.id}});
});
}
async function check(id: string, signal?: AbortSignal): Promise<Entry | null> {
signal?.throwIfAborted();
const claim = await db.transaction(async tx => {
const [session] = await tx.select().from(chatVoiceSessions).where(and(eq(chatVoiceSessions.id, id), eligible(Date.now()))).for("update");
if (!session) return null;
await notifyVoiceReports(tx, { kind: "session", id });
await tx.insert(chatVoiceReports).values({ sessionId: id, companyId: session.companyId, nextCheckAt: new Date() }).onConflictDoNothing();
const [report] = await tx.select().from(chatVoiceReports).where(and(eq(chatVoiceReports.sessionId, id),
eq(chatVoiceReports.companyId, session.companyId), lte(chatVoiceReports.nextCheckAt, new Date()))).for("update");
if (!report) return null;
signal?.throwIfAborted();
const attempts = report.attempts + 1;
const retryAt = new Date(Date.now() + Math.min(3600, 30 * 2 ** Math.min(attempts, 7)) * 1000);
await tx.update(chatVoiceReports).set({ attempts, nextCheckAt: retryAt }).where(eq(chatVoiceReports.sessionId, id));
return { session, attempts, retryAt };
});
if (claim) {
const { session, attempts, retryAt } = claim;
// A callback can advance nextCheckAt while this GET is in flight.
// That callback or a newer attempt owns settlement; never overwrite it.
const owned = () => and(eq(chatVoiceReports.sessionId, id), eq(chatVoiceReports.companyId, session.companyId),
eq(chatVoiceReports.attempts, attempts), eq(chatVoiceReports.nextCheckAt, retryAt));
signal?.throwIfAborted();
try {
const result = await fetchReport(session);
logger.info({ event: "voice.call.report_received", companyId: session.companyId, endpointId: session.endpointId,
issueId: session.issueId, sessionId: id, providerSessionId: session.providerSessionId, attempts,
complete: result.complete, transcriptTurns: result.transcript.length, providerUpdatedAt: result.providerUpdatedAt?.toISOString() ?? null }, "Voice call report received for reconciliation");
await db.transaction(async tx => {
await notifyVoiceReports(tx, { kind: "session", id });
await tx.update(chatVoiceReports).set({ transcript: result.transcript, costMicroUsd: result.costMicroUsd,
durationSeconds: result.durationSeconds, providerUpdatedAt: result.providerUpdatedAt,
status: result.complete ? "available" : "pending", updatedAt: new Date(),
}).where(and(owned(), result.complete ? undefined : ne(chatVoiceReports.status, "available"),
result.providerUpdatedAt ? or(sql`${chatVoiceReports.providerUpdatedAt} is null`, lte(chatVoiceReports.providerUpdatedAt, result.providerUpdatedAt)) : sql`${chatVoiceReports.providerUpdatedAt} is null`));
});
// Project the retained authoritative report, including when a response
// was stale. A projection failure retains the already saved retry.
await projectTranscript(session);
} catch (error) {
logger.warn({ event: "voice.call.report_failed", companyId: session.companyId, endpointId: session.endpointId,
issueId: session.issueId, sessionId: id, providerSessionId: session.providerSessionId, attempts,
errorCode: error instanceof SpekoProviderError ? error.code : "report_reconciliation_failed",
httpStatus: error instanceof SpekoProviderError ? error.httpStatus ?? null : null, retryAt: retryAt.toISOString() }, "Voice call report reconciliation deferred");
if (attempts >= 8) await db.transaction(async tx => {
await notifyVoiceReports(tx, { kind: "session", id });
await tx.update(chatVoiceReports).set({ status: "unavailable" }).where(and(owned(), ne(chatVoiceReports.status, "available")));
});
}
}
return (await load({ kind: "session", id }, signal))[0] ?? null;
}
async function reconcile(limit: number) {
const rows = (await load()).filter(row => row.dueAt <= Date.now()).sort((a, b) => a.dueAt - b.dueAt || a.id.localeCompare(b.id)).slice(0, limit);
for (const row of rows) await check(row.id);
return rows.length;
}
function register(coordinator: ReturnType<typeof createDeliveryWorkCoordinator>, canRun: () => boolean) {
return registerScopedDeadlineWork(coordinator, { queue: DELIVERY_QUEUES.voiceReports, decodeScope: voiceRecoveryScope,
matches: (entry, scope) => !scope || (scope.kind === "session" ? entry.id : scope.kind === "endpoint" ? entry.endpointId : scope.kind === "connection" ? entry.connectionId : entry.companyId) === scope.id,
load, check: (entry, signal) => check(entry.id, signal),
}, canRun);
}
return { load, check, reconcile, register };
}
@@ -1,3 +1,4 @@
import { notifyVoiceReports } from "./voice-report-notifications.js";
import { notifyVoiceInbound } from "./voice-inbound-notifications.js";
import { notifyVoiceReplies } from "./voice-reply-notifications.js";
import type { Db } from "@paperclipai/db";
@@ -20,6 +21,7 @@ export async function notifyVoiceChange(tx: object, scope?: VoiceRecoveryScope)
await notifyVoiceRecovery(tx, scope);
await notifyVoiceReplies(tx, scope);
await notifyVoiceInbound(tx, scope);
await notifyVoiceReports(tx, scope);
}
export function withVoiceSessionChange<T>(db: Db, id: string, write: (tx: Parameters<Parameters<Db["transaction"]>[0]>[0]) => PromiseLike<T>, options: { replyDelivery?: boolean } = {}): Promise<T> {
return db.transaction(async tx => {
@@ -1,3 +1,4 @@
import { notifyVoiceReports } from "./voice-report-notifications.js";
import { notifyVoiceInbound } from "./voice-inbound-notifications.js";
import { notifyVoiceReplies } from "./voice-reply-notifications.js";
import { recoverVoiceReplyDispatches } from "./voice-reply-dispatch-recovery.js";
@@ -84,7 +85,10 @@ export async function pushVoiceReplies(db: Db, options: {
await notifyVoiceReplies(tx, { kind: "session", id: candidate.id });
await tx.update(chatActions).set({status, result: {errorCode: providerError?.code ?? "voice_authority_unavailable", ...(status === "retry" ? {retryAt: Date.now() + 10_000} : {})}, updatedAt: new Date()}).where(eq(chatActions.id, claim.action.id));
if (status === "call_ended" || status === "unknown") await notifyVoiceRecovery(tx, { kind: "session", id: candidate.id });
if (status === "call_ended") await notifyVoiceInbound(tx, { kind: "session", id: candidate.id });
if (status === "call_ended") {
await notifyVoiceInbound(tx, { kind: "session", id: candidate.id });
await notifyVoiceReports(tx, { kind: "session", id: candidate.id });
}
if (status === "call_ended") await tx.update(chatVoiceSessions).set({state: "ended", endedAt: new Date(), updatedAt: new Date()}).where(and(eq(chatVoiceSessions.id, candidate.id), eq(chatVoiceSessions.companyId, candidate.companyId)));
if (status === "unknown") await tx.update(chatVoiceSessions).set({errorCode: "message_delivery_unknown", updatedAt: new Date()}).where(eq(chatVoiceSessions.id, candidate.id));
});
@@ -0,0 +1,6 @@
import { DELIVERY_QUEUES, notifyDeliveryWork } from "../delivery-work-notifications.js";
import type { VoiceRecoveryScope } from "./voice-recovery-notifications.js";
export function notifyVoiceReports(tx: object, scope?: VoiceRecoveryScope) {
return notifyDeliveryWork(tx, DELIVERY_QUEUES.voiceReports, scope ? `${scope.kind}:${scope.id}` : "*");
}
@@ -5,17 +5,17 @@ import { registerVoiceRecoveryWork } from "./voice-recovery-work.js";
import { notifyVoiceChange, withVoiceSessionChange } from "./voice-recovery-notifications.js";
import type { createDeliveryWorkCoordinator } from "../delivery-work-coordinator.js";
import { logger } from "../../middleware/logger.js";
import { voiceCallReports } from "./voice-call-reports.js";
import { captureVoiceDeliveryDiagnostics } from "./voice-delivery-diagnostics.js";
import { pushVoiceReplies } from "./voice-reply-push.js";
import { voiceInboundService } from "./voice-inbound-service.js";
import { createHash, randomUUID } from "node:crypto";
import { and, asc, desc, eq, inArray, lte, notInArray, or, sql } from "drizzle-orm";
import { chatVoiceReports, chatVoiceCallbacks, chatConversations, chatEndpointResources, chatEndpoints, chatVoiceSessions, issues, toolConnections, activityLog, type Db } from "@paperclipai/db";
import { and, desc, eq, inArray, notInArray, or, sql } from "drizzle-orm";
import { chatVoiceReports, chatVoiceCallbacks, chatConversations, chatEndpointResources, chatEndpoints, chatVoiceSessions, issues, activityLog, type Db } from "@paperclipai/db";
import { VOICE_RESULT_NOTIFICATION, VOICE_REPEAT_NOTIFICATION, type VoiceSession, type VoiceSessionMedia, type VoiceCallHistoryEntry, type VoiceCallReport } from "@paperclipai/shared";
import { conflict, forbidden, notFound, HttpError } from "../../errors.js";
import { instanceSettingsService } from "../instance-settings.js";
import { buildChatCommunicationGuidance } from "../chat-communication-guidance.js";
import { documentService } from "../documents.js";
import { issueService } from "../issues.js";
import { createSpekoProvider, SpekoProviderError } from "./speko-provider.js";
import { serializeVoiceSession, voiceSessionStore, type VoiceCaller, type VoiceSessionRow } from "./voice-session-store.js";
@@ -229,8 +229,16 @@ export function voiceSessionService(db: Db, options: {
function registerReplyWork(coordinator: ReturnType<typeof createDeliveryWorkCoordinator>, canRun: () => boolean) {
return registerVoiceReplyWork(coordinator, replies, canRun);
}
async function reconcileMaintenance(limit = 25) {
await reconcileReports(limit);
const reports = voiceCallReports(db, async session => {
const endpoint = await endpointFor(session.companyId, session.endpointId);
const credentials = await options.credentials(endpoint, session.credentialFingerprint);
const client = provider(credentials.apiKey);
const result = await client.callReport(session.providerSessionId!, [credentials.signingSecret]);
await captureVoiceDeliveryDiagnostics(db, session, client, result.complete);
return result;
});
function registerReportWork(coordinator: ReturnType<typeof createDeliveryWorkCoordinator>, canRun: () => boolean) {
return reports.register(coordinator, canRun);
}
/** Explicit one-shot recovery for diagnostics/tests; never register on a recurring lane. */
async function reconcile(limit = 25) {
@@ -238,7 +246,7 @@ export function voiceSessionService(db: Db, options: {
for (const row of due) await recovery.check(row.id).catch(() => undefined);
await pushReplies(limit);
await inbound.reconcile(limit);
await reconcileReports(limit);
await reports.reconcile(limit);
return due.length;
}
async function pushReplies(limit = 25, companyId?: string, endpointId?: string) {
@@ -270,65 +278,6 @@ export function voiceSessionService(db: Db, options: {
const [report] = await db.select().from(chatVoiceReports).where(and(eq(chatVoiceReports.companyId, companyId), eq(chatVoiceReports.sessionId, session.id)));
return { session: serializeVoiceSession(session), report: reportProjection(report) };
}
async function reconcileReports(limit: number) {
// Repair missed lifecycle callbacks, including later recording-ready reports.
// GET the same call; never create a call or download provider recordings.
const recent = new Date(Date.now() - 24 * 60 * 60 * 1000);
const rows = await db.select({ session: chatVoiceSessions, report: chatVoiceReports }).from(chatVoiceSessions)
.leftJoin(chatVoiceReports, and(eq(chatVoiceReports.sessionId, chatVoiceSessions.id), eq(chatVoiceReports.companyId, chatVoiceSessions.companyId)))
.where(and(inArray(chatVoiceSessions.state, terminalStates), sql`${chatVoiceSessions.providerSessionId} is not null`,
sql`${chatVoiceSessions.endedAt} >= ${recent.toISOString()}`, or(sql`${chatVoiceReports.sessionId} is null`, lte(chatVoiceReports.nextCheckAt, new Date()))))
.orderBy(asc(sql`coalesce(${chatVoiceReports.nextCheckAt}, ${chatVoiceSessions.endedAt})`)).limit(limit);
for (const {session, report} of rows) {
const attempts = (report?.attempts ?? 0) + 1;
const retryAt = new Date(Date.now() + Math.min(3600, 30 * 2 ** Math.min(attempts, 7)) * 1000);
await db.insert(chatVoiceReports).values({ sessionId: session.id, companyId: session.companyId }).onConflictDoNothing();
try {
const endpoint = await endpointFor(session.companyId, session.endpointId);
const credentials = await options.credentials(endpoint, session.credentialFingerprint);
const client = provider(credentials.apiKey);
const result = await client.callReport(session.providerSessionId!, [credentials.signingSecret]);
logger.info({event: "voice.call.report_received", companyId: session.companyId, endpointId: session.endpointId,
issueId: session.issueId, sessionId: session.id, providerSessionId: session.providerSessionId, attempts,
complete: result.complete, transcriptTurns: result.transcript.length, providerUpdatedAt: result.providerUpdatedAt?.toISOString() ?? null}, "Speko call report received for reconciliation");
await captureVoiceDeliveryDiagnostics(db, session, client, result.complete);
if (!result.complete && report?.status === "available") {
await db.update(chatVoiceReports).set({ attempts, nextCheckAt: retryAt }).where(eq(chatVoiceReports.sessionId, session.id));
continue;
}
await db.update(chatVoiceReports).set({
transcript: result.transcript, costMicroUsd: result.costMicroUsd, durationSeconds: result.durationSeconds,
providerUpdatedAt: result.providerUpdatedAt, status: result.complete ? "available" : "pending", attempts, nextCheckAt: retryAt, updatedAt: new Date(),
}).where(and(eq(chatVoiceReports.sessionId, session.id), eq(chatVoiceReports.companyId, session.companyId),
// A stale report must never replace a newer completed report.
result.providerUpdatedAt ? or(sql`${chatVoiceReports.providerUpdatedAt} is null`, lte(chatVoiceReports.providerUpdatedAt, result.providerUpdatedAt)) : sql`${chatVoiceReports.providerUpdatedAt} is null`));
// Keep both actual spoken sides on this public call's own task. This
// does not publish a document or grant access to any other task.
if (session.callerAuthority === "guest_intake") await db.transaction(async tx => {
await tx.execute(sql`select pg_advisory_xact_lock(hashtextextended(${`voice-transcript:${session.id}`}, 0))`);
const [retained] = await tx.select().from(chatVoiceReports).where(and(eq(chatVoiceReports.companyId, session.companyId), eq(chatVoiceReports.sessionId, session.id)));
if (!retained?.transcript.length) return;
const [task] = await tx.select({sourceTrust: issues.sourceTrust}).from(issues).where(and(eq(issues.companyId, session.companyId), eq(issues.id, session.issueId)));
if (task?.sourceTrust?.preset !== "low_trust_review" || task.sourceTrust.disposition !== "quarantined") return;
const key = `voice-transcript-${session.id}`;
const body = retained.transcript.map(turn => `**${turn.speaker === "caller" ? "Caller" : "Agent"}** · ${turn.startedAt}${turn.interrupted ? " · interrupted" : ""}\n\n${turn.text}`).join("\n\n---\n\n");
const docs = documentService(tx as unknown as Db), previous = await docs.getIssueDocumentByKey(session.issueId, key);
if (previous?.body === body) return;
await docs.upsertIssueDocument({issueId: session.issueId, key, title: "Phone conversation transcript", format: "markdown", body, sourceTrust: task.sourceTrust, changeSummary: "Speko spoken conversation, including caller and agent turns"});
await tx.insert(activityLog).values({companyId: session.companyId, actorType: "system", actorId: "speko-report", action: "voice.transcript.updated", entityType: "issue", entityId: session.issueId, details: {sessionId: session.id}});
});
// Even a stale provider reply must back off instead of spinning.
await db.update(chatVoiceReports).set({ attempts, nextCheckAt: retryAt }).where(eq(chatVoiceReports.sessionId, session.id));
} catch (error) {
logger.warn({event: "voice.call.report_failed", companyId: session.companyId, endpointId: session.endpointId,
issueId: session.issueId, sessionId: session.id, providerSessionId: session.providerSessionId, attempts,
errorCode: error instanceof SpekoProviderError ? error.code : "report_reconciliation_failed",
httpStatus: error instanceof SpekoProviderError ? error.httpStatus ?? null : null, retryAt: retryAt.toISOString()}, "Speko call report reconciliation deferred");
await db.update(chatVoiceReports).set({ attempts, nextCheckAt: retryAt, ...(report?.status !== "available" && attempts >= 8 ? { status: "unavailable" as const } : {}) }).where(eq(chatVoiceReports.sessionId, session.id));
}
}
return rows.length;
}
async function callbackPreference(companyId: string, endpointId: string, caller: VoiceCaller) {
await endpointFor(companyId, endpointId);
await db.transaction(tx => store.authorizeCaller(tx, companyId, caller));
@@ -351,7 +300,7 @@ export function voiceSessionService(db: Db, options: {
const result = await start({ ...binding, mode: "outbound_phone", phoneNumber: preference.phoneNumber, idempotencyKey: invocationId, maxDurationSeconds: 600 });
return { session: result.session };
});
return { start, inspect, end, tool, reconcile, reconcileMaintenance, registerRecoveryWork, registerReplyWork, pushReplies, inbound, notification: store.notification, callbackPreference, saveCallbackPreference, history, callDetail, unregisterAgentTools };
return { start, inspect, end, tool, reconcile, registerReportWork, registerRecoveryWork, registerReplyWork, pushReplies, inbound, notification: store.notification, callbackPreference, saveCallbackPreference, history, callDetail, unregisterAgentTools };
}
export type VoiceSessionService = ReturnType<typeof voiceSessionService>;