Jannik/PaperClipAI
1
0 Fork 0
mirror of https://github.com/paperclipai/paperclip.git synced 2026-11-11 13:24:55 +01:00
Code Issues Packages Projects Releases Wiki Activity

fix(server): schedule tool health from connection changes and deadlines

Browse Source 
Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
DottaandPaperclip committed 2026-10-25 00:27:46 -05:00
1 parent 9bdc52e093
commit 2ca12f8063
44 files changed
+889 -231
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

server/src/services/access.ts
+2
View File
@@ -1,3 +1,4 @@
import { notifyToolConnectionHealth } from "./tool-connection-health-notifications.js";
import { and, eq, inArray, ne, notInArray, sql } from "drizzle-orm";
import type { Db } from "@paperclipai/db";
import {
@@ -323,6 +324,7 @@ export function accessService(db: Db) {
&& grant.credentialSecretRefs.length > 0
&& grant.credentialSecretRefs.every((ref) => !connectionSecretIdsToRemove.has(ref.secretId)),
);
if (!hasUnaffectedActiveGrant) await notifyToolConnectionHealth(tx, companyId);
await tx.update(toolConnections).set({
credentialRefs,
credentialSecretRefs,
server/src/services/chat-channels.ts
+2
View File
@@ -1,3 +1,4 @@
import { notifyChatConnectionHealth } from "./chat-channels/connection-health-notifications.js";
import { createChatSdkMessageReceiver } from "./chat-channels/runtime/sdk-messages.js";
import { admitDiscordSdkMessage } from "./chat-channels/discord/messages.js";
import { resolveInboundConversationBinding } from "./chat-channels/inbound/conversation-binding.js";
@@ -2299,6 +2300,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) {
updatedAt: new Date(),
})
.where(eq(chatConversations.id, conversation!.id));
await notifyChatConnectionHealth(taskTx, taskEndpoint);
await taskTx
.update(toolConnections)
.set({
server/src/services/chat-channels/configuration.ts
+7
View File
@@ -1,3 +1,4 @@
import { notifyChatConnectionHealth } from "./connection-health-notifications.js";
import { preparePhotonConfiguration, createPhotonConfiguration } from "./imessage-photon/setup.js";
import { slackRuntimeConfig } from "./slack/setup.js";
import { createTelegramSetup } from "./telegram/setup.js";
@@ -193,6 +194,7 @@ export function createChatConfiguration(db: Db, options: ChatConfigurationOption
updatedAt: pausedAt,
})
.where(eq(chatEndpoints.id, endpoint.id));
await notifyChatConnectionHealth(tx, endpoint);
await tx
.update(toolConnections)
.set({ status: "disabled", enabled: false, updatedAt: new Date() })
@@ -336,6 +338,7 @@ export function createChatConfiguration(db: Db, options: ChatConfigurationOption
})
.where(eq(chatEndpoints.id, endpoint.id))
.returning();
await notifyChatConnectionHealth(tx, endpoint);
await tx
.update(toolConnections)
.set({
@@ -380,6 +383,7 @@ export function createChatConfiguration(db: Db, options: ChatConfigurationOption
sql`coalesce((${chatEndpoints.setup}->>'runtimeGeneration')::integer, 0) = ${runtimeGeneration(activatedEndpoint.setup)}`,
),
);
await notifyChatConnectionHealth(tx, endpoint);
await tx
.update(toolConnections)
.set({
@@ -484,6 +488,7 @@ export function createChatConfiguration(db: Db, options: ChatConfigurationOption
.update(chatConversations)
.set({ state: "endpoint_removed", updatedAt: new Date() })
.where(eq(chatConversations.endpointId, endpoint.id));
await notifyChatConnectionHealth(tx, endpoint);
await tx
.update(toolConnections)
.set({ status: "archived", enabled: false, updatedAt: new Date() })
@@ -812,6 +817,7 @@ export function createChatConfiguration(db: Db, options: ChatConfigurationOption
updatedAt,
})
.where(eq(chatEndpoints.id, endpoint.id));
await notifyChatConnectionHealth(tx, endpoint);
await tx
.update(toolConnections)
.set({
@@ -873,6 +879,7 @@ export function createChatConfiguration(db: Db, options: ChatConfigurationOption
updatedAt: new Date(),
})
.where(eq(chatEndpoints.id, endpoint.id));
await notifyChatConnectionHealth(tx, endpoint);
await tx
.update(toolConnections)
.set({
server/src/services/chat-channels/connection-health-notifications.ts
+8
View File
@@ -0,0 +1,8 @@
import type { ChatProvider } from "@paperclipai/shared";
import { notifyToolConnectionHealth } from "../tool-connection-health-notifications.js";
import { chatConnectionTransport } from "./provider-catalog.js";
export async function notifyChatConnectionHealth(transaction: object, endpoint: { companyId: string; provider: ChatProvider }) {
if (chatConnectionTransport(endpoint.provider) === "chat_sdk") return;
await notifyToolConnectionHealth(transaction, endpoint.companyId);
}
server/src/services/chat-channels/endpoints.ts
+2 -1
View File
@@ -1,3 +1,4 @@
import { chatConnectionTransport } from "./provider-catalog.js";
import { slackSetupState } from "./slack/setup.js";
import { assertChatExecutionDefaultsAccess } from "../chat-execution-defaults.js";
@@ -326,7 +327,7 @@ function providerSetupState(
connectionKind: "managed",
connectionPurpose: "channel",
ownership: "customer",
transport: input.provider === "speko" ? "voice" : "chat_sdk",
transport: chatConnectionTransport(input.provider),
authKind: "api_key",
credentialPolicy: "shared",
status: "draft",
server/src/services/chat-channels/interactions/callbacks.ts
+2
View File
@@ -1,3 +1,4 @@
import { notifyChatConnectionHealth } from "../connection-health-notifications.js";
import { createDiscordQuestionForms } from "../discord/question-forms.js";
import { createProviderEffects, type ProviderEffectPayload } from "../provider-effects.js";
@@ -420,6 +421,7 @@ export function createChatInteractionCallbacks(db: Db, options: InteractionCallb
updatedAt: processedAt,
})
.where(eq(chatEndpoints.id, endpoint.id));
await notifyChatConnectionHealth(tx, currentEndpoint);
await tx
.update(toolConnections)
.set({
server/src/services/chat-channels/provider-catalog.ts
+6
View File
@@ -106,3 +106,9 @@ const CAPABILITIES: Record<ChatProvider, ChatAdapterCapabilities> = {
},
};
export { PROVIDER_LABELS, CAPABILITIES };
export function chatConnectionTransport(provider: ChatProvider): "voice" | "rest_api" | "chat_sdk" {
if (provider === "speko") return "voice";
if (provider === "agentmail") return "rest_api";
return "chat_sdk";
}
server/src/services/chat-channels/provider-effects.ts
+2
View File
@@ -1,3 +1,4 @@
import { notifyChatConnectionHealth } from "./connection-health-notifications.js";
import { createTelegramCallbackNoticeAuthority } from "./telegram/callback-notices.js";
import { slackThreadChannelId } from "./slack/messages.js";
@@ -325,6 +326,7 @@ export function createProviderEffects(db: Db, options: ProviderEffectsOptions) {
updatedAt: new Date(),
})
.where(eq(chatEndpoints.id, currentEndpoint.id));
await notifyChatConnectionHealth(tx, currentEndpoint);
await tx
.update(toolConnections)
.set({
server/src/services/chat-channels/publications/failures.ts
+2
View File
@@ -1,3 +1,4 @@
import { notifyChatConnectionHealth } from "../connection-health-notifications.js";
import { isOutboundAttachmentHydrationError, isOutboundAttachmentValidationError } from "../outbound-attachments.js";
import type { EndpointRow, DbTransaction, LifecycleRuntimeFence } from "../runtime-types.js";
@@ -197,6 +198,7 @@ export function createChatPublicationFailures(db: Db, options: PublicationFailur
updatedAt: new Date(),
})
.where(eq(chatEndpoints.id, currentEndpoint.id));
await notifyChatConnectionHealth(tx, currentEndpoint);
await tx
.update(toolConnections)
.set({
server/src/services/chat-channels/resource-reconciliation.ts
+3
View File
@@ -1,3 +1,4 @@
import { notifyChatConnectionHealth } from "./connection-health-notifications.js";
import { imessageResourceParticipants } from "./imessage-photon/resources.js";
import { createGitHubResources, reconcileGitHubRepositoryCoordinates } from "./github/resources.js";
import { baseTeamsConversationId } from "./microsoft-teams/resources.js";
@@ -522,6 +523,7 @@ export function createChatResourceReconciliation(db: Db, options: ResourceReconc
updatedAt: now,
})
.where(eq(chatEndpoints.id, currentEndpoint.id));
await notifyChatConnectionHealth(tx, currentEndpoint);
await tx
.update(toolConnections)
.set({
@@ -551,6 +553,7 @@ export function createChatResourceReconciliation(db: Db, options: ResourceReconc
updatedAt: now,
})
.where(eq(chatEndpoints.id, currentEndpoint.id));
await notifyChatConnectionHealth(tx, currentEndpoint);
await tx
.update(toolConnections)
.set({
server/src/services/chat-channels/setup-probes.ts
+2
View File
@@ -1,3 +1,4 @@
import { notifyChatConnectionHealth } from "./connection-health-notifications.js";
import type { EndpointRow, DbOrTransaction } from "./runtime-types.js";
import { notifyChatEndpointWork } from "../chat-work-notifications.js";
@@ -326,6 +327,7 @@ export function createSetupProbes(db: Db, options: SetupProbesOptions) {
code: "chat_endpoint_test_superseded",
});
}
await notifyChatConnectionHealth(tx, endpoint);
await tx
.update(toolConnections)
.set({
server/src/services/company-deletion.ts
+2
View File
@@ -1,3 +1,4 @@
import { notifyToolConnectionHealth } from "./tool-connection-health-notifications.js";
import { notifyCompanyRetentionWork } from "./retention-work-notifications.js";
import { notifyDecisionTargetWork } from "./decision-target-work.js";
import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js";
@@ -118,6 +119,7 @@ export async function deleteCompany(db: Db, id: string) {
await tx.delete(executionWorkspaces).where(eq(executionWorkspaces.companyId, id));
await tx.delete(projects).where(eq(projects.companyId, id));
await agentLifecycleCompanyDeletion.deleteCompanyData(tx, id);
await notifyToolConnectionHealth(tx, id);
const rows = await tx
.delete(companies)
.where(eq(companies.id, id))
server/src/services/delivery-work-notifications.ts
+1
View File
@@ -16,6 +16,7 @@ export const DELIVERY_QUEUES = {
decisionExpiry: "decision-expiry",
decisionContinuation: "decision-continuation",
decisionArchiveDelivery: "decision-archive-delivery",
toolConnectionHealth: "tool-connection-health",
toolReviewProjection: "tool-review-projection",
toolReviewInterruption: "tool-review-interruption",
toolReviewRecovery: "tool-review-recovery",
server/src/services/email-channels.ts
+8 -4
View File
@@ -1,3 +1,4 @@
import { notifyToolConnectionHealth, writeToolConnectionHealthChange } from "./tool-connection-health-notifications.js";
import { createDeliveryWorkCoordinator } from "./delivery-work-coordinator.js";
import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js";
import { fastResponseService, enqueueFastResponse, fastResponseSourceCurrent, fastResponseTurnQueued } from "./fast-responses.js";
@@ -913,6 +914,7 @@ export function emailChannelService(db: Db, options: EmailChannelOptions) {
updatedAt: now,
})
.where(eq(chatEndpoints.id, endpoint.id));
await notifyToolConnectionHealth(tx, endpoint.companyId);
await tx
.update(toolConnections)
.set({ status: "active", enabled: true, healthStatus: "ok" })
@@ -1984,7 +1986,7 @@ export function emailChannelService(db: Db, options: EmailChannelOptions) {
.update(chatEndpoints)
.set({ lastError: null, healthMessage: "Connected" })
.where(eq(chatEndpoints.id, endpoint.id));
await db
await writeToolConnectionHealthChange(db, endpoint.companyId, tx => tx
.update(toolConnections)
.set({
healthStatus: "ok",
@@ -1992,7 +1994,7 @@ export function emailChannelService(db: Db, options: EmailChannelOptions) {
lastError: null,
healthCheckedAt: new Date(),
})
.where(eq(toolConnections.id, endpoint.connectionId));
.where(eq(toolConnections.id, endpoint.connectionId)));
}
}
async function maintainSocket(endpoint: Endpoint) {
@@ -2092,7 +2094,7 @@ export function emailChannelService(db: Db, options: EmailChannelOptions) {
});
}
async function markError(endpoint: Endpoint, error: string) {
await db
await writeToolConnectionHealthChange(db, endpoint.companyId, tx => tx
.update(toolConnections)
.set({
healthStatus: "degraded",
@@ -2100,7 +2102,7 @@ export function emailChannelService(db: Db, options: EmailChannelOptions) {
lastError: error,
healthCheckedAt: new Date(),
})
.where(eq(toolConnections.id, endpoint.connectionId));
.where(eq(toolConnections.id, endpoint.connectionId)));
await db
.update(chatEndpoints)
.set({ lastError: error, healthMessage: error, updatedAt: new Date() })
@@ -2362,6 +2364,7 @@ export function emailChannelService(db: Db, options: EmailChannelOptions) {
.update(chatEndpoints)
.set({ status: "paused" })
.where(eq(chatEndpoints.id, endpoint.id));
await notifyToolConnectionHealth(tx, endpoint.companyId);
await tx
.update(toolConnections)
.set({ enabled: false, status: "disabled" })
@@ -2455,6 +2458,7 @@ export function emailChannelService(db: Db, options: EmailChannelOptions) {
updatedAt: new Date(),
})
.where(eq(chatEndpoints.id, id));
await notifyToolConnectionHealth(tx, endpoint.companyId);
await tx
.update(toolConnections)
.set({
server/src/services/email-connections.ts
+3 -2
View File
@@ -1,3 +1,4 @@
import { writeToolConnectionHealthChange } from "./tool-connection-health-notifications.js";
import { randomUUID } from "node:crypto";
import { and, eq, isNull, ne, sql } from "drizzle-orm";
import {
@@ -257,7 +258,7 @@ export function emailConnectionService(
actorSource: actor.localImplicit ? "local_implicit" : "session",
},
);
await db
await writeToolConnectionHealthChange(db, companyId, tx => tx
.update(toolConnections)
.set({
uid: `agentmail-account-${input.idempotencyKey}`,
@@ -265,7 +266,7 @@ export function emailConnectionService(
healthMessage: "Connected",
healthCheckedAt: new Date(),
})
.where(eq(toolConnections.id, connection.id));
.where(eq(toolConnections.id, connection.id)));
if (input.grantKind === "user")
await db
.update(connectionGrants)
server/src/services/github-connection-events.ts
+2
View File
@@ -1,3 +1,4 @@
import { notifyToolConnectionHealth } from "./tool-connection-health-notifications.js";
import { dispatchGitHubBotCloudEvent } from "./chat-github-cloud-ingress.js";
import { chatEndpoints } from "@paperclipai/db";
import { randomUUID } from "node:crypto";
@@ -326,6 +327,7 @@ export function githubConnectionEventService(
};
await database.update(connectionGrants).set({ providerTenant, updatedAt: now() })
.where(and(eq(connectionGrants.id, binding.grantId), eq(connectionGrants.companyId, binding.companyId)));
await notifyToolConnectionHealth(database, binding.companyId);
await database.update(toolConnections).set({
healthStatus: unavailable ? "failed" : "ok",
healthMessage: unavailable
server/src/services/instance-settings.ts
+2
View File
@@ -1,3 +1,4 @@
import { notifyToolConnectionHealth } from "./tool-connection-health-notifications.js";
import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js";
import type { Db } from "@paperclipai/db";
import { companies, instanceSettings } from "@paperclipai/db";
@@ -613,6 +614,7 @@ export function instanceSettingsService(db: Db, options: InstanceSettingsService
await notifyDeliveryWork(tx, DELIVERY_QUEUES.mcpEvents, "*");
await notifyDeliveryWork(tx, DELIVERY_QUEUES.dotEvents, "*");
await notifyDeliveryWork(tx, DELIVERY_QUEUES.grokDispatch);
await notifyToolConnectionHealth(tx, "*");
return tx
.update(instanceSettings)
.set({
server/src/services/smoke-lab.ts
+44 -40
View File
@@ -1,3 +1,4 @@
import { writeToolConnectionHealthChange } from "./tool-connection-health-notifications.js";
import { spawn, type ChildProcess } from "node:child_process";
import { createHash, randomUUID } from "node:crypto";
import { existsSync } from "node:fs";
@@ -690,14 +691,14 @@ export function smokeLabService(db: Db, options: {
));
const now = new Date();
if (existing) {
const [updated] = await db.update(toolApplications).set({
const [updated] = await writeToolConnectionHealthChange(db, input.companyId, tx => tx.update(toolApplications).set({
name: input.name,
description: input.description,
type: input.type,
status: "active",
metadata: { smokeLab: true },
updatedAt: now,
}).where(eq(toolApplications.id, existing.id)).returning();
}).where(eq(toolApplications.id, existing.id)).returning());
return { row: updated ?? existing, created: false };
}
const [created] = await db.insert(toolApplications).values({
@@ -742,41 +743,43 @@ export function smokeLabService(db: Db, options: {
lastHealthAt: now,
updatedAt: now,
};
const ensureDefaultOrganizationGrant = async (connectionId: string) => {
const [existingGrant] = await db.select({ id: connectionGrants.id }).from(connectionGrants).where(and(
eq(connectionGrants.companyId, input.companyId),
eq(connectionGrants.connectionId, connectionId),
eq(connectionGrants.kind, "organization"),
eq(connectionGrants.isDefault, true),
));
if (existingGrant) return;
await db.insert(connectionGrants).values({
return writeToolConnectionHealthChange(db, input.companyId, async tx => {
const ensureDefaultOrganizationGrant = async (connectionId: string) => {
const [existingGrant] = await tx.select({ id: connectionGrants.id }).from(connectionGrants).where(and(
eq(connectionGrants.companyId, input.companyId),
eq(connectionGrants.connectionId, connectionId),
eq(connectionGrants.kind, "organization"),
eq(connectionGrants.isDefault, true),
));
if (existingGrant) return;
await tx.insert(connectionGrants).values({
companyId: input.companyId,
connectionId,
kind: "organization",
status: "active",
isDefault: true,
credentialSecretRefs: [],
createdAt: now,
updatedAt: now,
});
};
if (existing) {
const [updated] = await tx.update(toolConnections).set(values).where(eq(toolConnections.id, existing.id)).returning();
await ensureDefaultOrganizationGrant(existing.id);
return { row: updated ?? existing, created: false };
}
const [created] = await tx.insert(toolConnections).values({
companyId: input.companyId,
connectionId,
kind: "organization",
status: "active",
isDefault: true,
credentialSecretRefs: [],
name: input.name,
uid: `smoke-lab/${input.name.toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-|-$/g, "")}`,
...values,
createdByAgentId: input.actor?.actorType === "agent" ? input.actor.agentId : null,
createdByUserId: input.actor?.actorType === "user" ? input.actor.actorId : null,
createdAt: now,
updatedAt: now,
});
};
if (existing) {
const [updated] = await db.update(toolConnections).set(values).where(eq(toolConnections.id, existing.id)).returning();
await ensureDefaultOrganizationGrant(existing.id);
return { row: updated ?? existing, created: false };
}
const [created] = await db.insert(toolConnections).values({
companyId: input.companyId,
name: input.name,
uid: `smoke-lab/${input.name.toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-|-$/g, "")}`,
...values,
createdByAgentId: input.actor?.actorType === "agent" ? input.actor.agentId : null,
createdByUserId: input.actor?.actorType === "user" ? input.actor.actorId : null,
createdAt: now,
}).returning();
await ensureDefaultOrganizationGrant(created.id);
return { row: created, created: true };
}).returning();
await ensureDefaultOrganizationGrant(created.id);
return { row: created, created: true };
});
}
async function ensureStdioTemplate(companyId: string, actor?: SmokeLabActorInfo) {
@@ -963,20 +966,21 @@ export function smokeLabService(db: Db, options: {
eq(toolConnections.name, HTTP_CONNECTION_NAME),
));
if (!connection || !httpSidecar) return;
const sidecarUrl = httpSidecar.url;
const now = new Date();
await db.update(toolConnections).set({
await writeToolConnectionHealthChange(db, companyId, tx => tx.update(toolConnections).set({
config: {
...asRecord(connection.config),
url: `${httpSidecar.url}/mcp`,
catalogUrl: `${httpSidecar.url}/catalog`,
toolCallUrl: `${httpSidecar.url}/tools/call`,
url: `${sidecarUrl}/mcp`,
catalogUrl: `${sidecarUrl}/catalog`,
toolCallUrl: `${sidecarUrl}/tools/call`,
},
healthStatus: "ok",
healthMessage: "Smoke Lab HTTP fixture sidecar is running.",
healthCheckedAt: now,
lastHealthAt: now,
updatedAt: now,
}).where(eq(toolConnections.id, connection.id));
}).where(eq(toolConnections.id, connection.id)));
}
async function startHttpSidecar(companyId?: string) {
server/src/services/tool-access.ts
+102 -83
View File
@@ -1,3 +1,5 @@
import { notifyToolConnectionHealth, writeToolConnectionHealthChange } from "./tool-connection-health-notifications.js";
import { connectionHealthSchedule } from "./tool-access/catalog/health-schedule.js";
import { restConnectorFor } from "./connectors/rest-registry.js";
import { createConnectionCredentialValidation } from "./tool-access/connections/credential-validation.js";
import { createConnectionGrants } from "./tool-access/connections/grants.js";
@@ -1217,7 +1219,7 @@ export function toolAccessService(
safeDefault: true,
};
if (existing) {
const [updated] = await db
const [updated] = await writeToolConnectionHealthChange(db, companyId, tx => tx
.update(toolApplications)
.set({
name: definition.applicationName,
@@ -1229,7 +1231,7 @@ export function toolAccessService(
updatedAt: new Date(),
})
.where(eq(toolApplications.id, existing.id))
.returning();
.returning());
return { row: updated, created: false };
}
const [created] = await db
@@ -1260,11 +1262,46 @@ export function toolAccessService(
quarantineNewEntries: true,
};
if (existing) {
const [updated] = await db
.update(toolConnections)
.set({
const updated = await writeToolConnectionHealthChange(db, companyId, async tx => {
const [updated] = await tx
.update(toolConnections)
.set({
applicationId,
name: definition.connectionName,
transport: "local_stdio",
status: "active",
enabled: true,
config,
transportConfig: config,
credentialRefs: [],
credentialSecretRefs: [],
updatedAt: new Date(),
})
.where(eq(toolConnections.id, existing.id))
.returning();
await ensureDefaultOrganizationGrant(updated, tx);
await syncCredentialBindings(updated, [], tx);
await createToolRuntimeSlots(tx, runtimeSupervisor).ensureRuntimeSlot(updated);
return updated;
});
emitConnectionUpdated(updated, existing, "example");
return { row: updated, created: false };
}
const connectionId = randomUUID();
const created = await writeToolConnectionHealthChange(db, companyId, async tx => {
const [created] = await tx
.insert(toolConnections)
.values({
id: connectionId,
companyId,
applicationId,
name: definition.connectionName,
uid: connectionUid(
"paperclip",
definition.connectionName,
connectionId,
),
connectionKind: "managed",
transport: "local_stdio",
status: "active",
enabled: true,
@@ -1272,42 +1309,13 @@ export function toolAccessService(
transportConfig: config,
credentialRefs: [],
credentialSecretRefs: [],
updatedAt: new Date(),
})
.where(eq(toolConnections.id, existing.id))
.returning();
await ensureDefaultOrganizationGrant(updated);
await syncCredentialBindings(updated);
await ensureRuntimeSlot(updated);
emitConnectionUpdated(updated, existing, "example");
return { row: updated, created: false };
}
const connectionId = randomUUID();
const [created] = await db
.insert(toolConnections)
.values({
id: connectionId,
companyId,
applicationId,
name: definition.connectionName,
uid: connectionUid(
"paperclip",
definition.connectionName,
connectionId,
),
connectionKind: "managed",
transport: "local_stdio",
status: "active",
enabled: true,
config,
transportConfig: config,
credentialRefs: [],
credentialSecretRefs: [],
})
.returning();
await ensureDefaultOrganizationGrant(created);
await syncCredentialBindings(created);
await ensureRuntimeSlot(created);
await ensureDefaultOrganizationGrant(created, tx);
await syncCredentialBindings(created, [], tx);
await createToolRuntimeSlots(tx, runtimeSupervisor).ensureRuntimeSlot(created);
return created;
});
emitConnectionCreated(created, "example");
return { row: created, created: true };
}
@@ -2478,7 +2486,7 @@ export function toolAccessService(
personalIdentityUserId || dedicatedAgentId ? [] : credentialSecretRefs;
if (revivedConnectionPrevious) {
forgetMcpHttpSessions(revivedConnectionPrevious.id);
[connectionRow] = await db
[connectionRow] = await writeToolConnectionHealthChange(db, companyId, tx => tx
.update(toolConnections)
.set({
name,
@@ -2500,7 +2508,7 @@ export function toolAccessService(
updatedAt: new Date(),
})
.where(eq(toolConnections.id, revivedConnectionPrevious.id))
.returning();
.returning());
} else {
const connectionId = randomUUID();
[connectionRow] = await db
@@ -2923,6 +2931,7 @@ export function toolAccessService(
connectionSetupMutationFingerprint(attemptedConnection),
);
if (connectionMutationIsStillCurrent) {
await notifyToolConnectionHealth(tx, companyId);
await tx
.update(toolConnections)
.set({
@@ -2998,7 +3007,7 @@ export function toolAccessService(
// surface the restoration failure instead of returning only the
// original provider error.
try {
await db
await writeToolConnectionHealthChange(db, companyId, tx => tx
.update(toolConnections)
.set({
status: "draft",
@@ -3014,7 +3023,7 @@ export function toolAccessService(
eq(toolConnections.id, revivedConnectionPrevious.id),
eq(toolConnections.companyId, companyId),
),
);
));
} catch (quarantineError) {
identityRollbackError = new AggregateError(
[rollbackError, quarantineError],
@@ -3023,9 +3032,10 @@ export function toolAccessService(
}
}
} else if (connectionRow) {
await db
const failedConnectionId = connectionRow.id;
await writeToolConnectionHealthChange(db, companyId, tx => tx
.delete(toolConnections)
.where(eq(toolConnections.id, connectionRow.id))
.where(eq(toolConnections.id, failedConnectionId)))
.catch(() => undefined);
}
if (
@@ -3138,6 +3148,7 @@ export function toolAccessService(
await tx.select({ id: toolApplications.id }).from(toolApplications)
.where(and(eq(toolApplications.id, connection.applicationId), eq(toolApplications.companyId, companyId)))
.for("update");
await notifyToolConnectionHealth(tx, companyId);
const [row] = await tx.update(toolConnections).set({ config, transportConfig: config, status: "active", enabled: true, ...(input.agentInstructions !== undefined ? { agentInstructions: input.agentInstructions } : {}), updatedAt: new Date() })
.where(and(eq(toolConnections.id, connection.id), eq(toolConnections.companyId, companyId))).returning();
await tx.update(toolApplications).set({ status: "active", updatedAt: new Date() }).where(and(eq(toolApplications.id, connection.applicationId), eq(toolApplications.companyId, companyId)));
@@ -3494,6 +3505,7 @@ export function toolAccessService(
},
tx,
);
await notifyToolConnectionHealth(tx, companyId);
const [updatedConnection] = await tx
.update(toolConnections)
.set({ status: "active", enabled: true, ...(input.agentInstructions !== undefined ? { agentInstructions: input.agentInstructions } : {}), updatedAt: new Date() })
@@ -4514,41 +4526,44 @@ export function toolAccessService(
}
const connectionId = randomUUID();
const binding = actorBinding(actor);
const [row] = await db
.insert(toolConnections)
.values({
id: connectionId,
companyId,
applicationId,
name: input.name,
uid: connectionUid(applicationNamespace, input.name, connectionId),
connectionKind: input.connectionKind ?? "managed",
ownership: input.ownership ?? "customer",
transport,
authKind: input.authKind ?? "none",
credentialPolicy:
input.credentialPolicy ??
(input.authKind === "oauth" ? "per_user" : "shared"),
status: input.status ?? "draft",
enabled: input.enabled ?? false,
agentInstructions: input.agentInstructions !== undefined ? input.agentInstructions : defaultConnectionAgentInstructions(
typeof config.sourceTemplateKey === "string" ? getConnectableAppDefinition(config.sourceTemplateKey)?.agentInstructions : null,
),
config,
transportConfig: isGoogleSheetsConnectionConfig(config)
? config
: (input.transportConfig ?? config),
credentialRefs: input.credentialRefs ?? [],
credentialSecretRefs: input.credentialSecretRefs ?? [],
createdByAgentId:
binding.actorType === "agent" ? binding.actorId : null,
createdByUserId:
binding.actorType === "user" ? binding.actorId : null,
})
.returning();
await ensureDefaultOrganizationGrant(row);
await syncCredentialBindings(row);
await ensureRuntimeSlot(row);
const row = await writeToolConnectionHealthChange(db, companyId, async tx => {
const [row] = await tx
.insert(toolConnections)
.values({
id: connectionId,
companyId,
applicationId,
name: input.name,
uid: connectionUid(applicationNamespace, input.name, connectionId),
connectionKind: input.connectionKind ?? "managed",
ownership: input.ownership ?? "customer",
transport,
authKind: input.authKind ?? "none",
credentialPolicy:
input.credentialPolicy ??
(input.authKind === "oauth" ? "per_user" : "shared"),
status: input.status ?? "draft",
enabled: input.enabled ?? false,
agentInstructions: input.agentInstructions !== undefined ? input.agentInstructions : defaultConnectionAgentInstructions(
typeof config.sourceTemplateKey === "string" ? getConnectableAppDefinition(config.sourceTemplateKey)?.agentInstructions : null,
),
config,
transportConfig: isGoogleSheetsConnectionConfig(config)
? config
: (input.transportConfig ?? config),
credentialRefs: input.credentialRefs ?? [],
credentialSecretRefs: input.credentialSecretRefs ?? [],
createdByAgentId:
binding.actorType === "agent" ? binding.actorId : null,
createdByUserId:
binding.actorType === "user" ? binding.actorId : null,
})
.returning();
await ensureDefaultOrganizationGrant(row, tx);
await syncCredentialBindings(row, [], tx);
await createToolRuntimeSlots(tx, runtimeSupervisor).ensureRuntimeSlot(row);
return row;
});
emitConnectionCreated(row, "api");
return toConnection(row);
},
@@ -4633,9 +4648,12 @@ export function toolAccessService(
await syncCredentialBindings(rows[0]!, [], tx);
return rows;
})
: await write(db);
if (existing.transport !== "chat_sdk") await syncCredentialBindings(row);
await ensureRuntimeSlot(row);
: await writeToolConnectionHealthChange(db, existing.companyId, async tx => {
const rows = await write(tx);
await syncCredentialBindings(rows[0]!, [], tx);
await createToolRuntimeSlots(tx, runtimeSupervisor).ensureRuntimeSlot(rows[0]!);
return rows;
});
emitConnectionUpdated(row, existing, "api");
return toConnection(row);
},
@@ -4685,6 +4703,7 @@ export function toolAccessService(
listAppsNeedingAttention,
sweepConnectionHealth,
scheduledHealth: connectionHealthSchedule(db, checkConnectionHealth),
sweepGitHubConnectionContinuity,
...catalogReader,
server/src/services/tool-access/catalog/health-schedule.ts
+48
View File
@@ -0,0 +1,48 @@
import { and, eq, ne, sql } from "drizzle-orm";
import { toolApplications, toolConnections, type Db } from "@paperclipai/db";
import { isConnectorFeatureEnabled } from "@paperclipai/shared";
import { instanceSettingsService } from "../../instance-settings.js";
import { connectionSetupApp } from "../connections/setup-policy.js";
import type { ConnectionHealth } from "./health.js";
export type ConnectionHealthDeadline = { id: string; companyId: string; dueAt: number };
const HEALTH_FRESHNESS_MS = 15 * 60_000;
export function connectionHealthSchedule(db: Db, checkHealth: ConnectionHealth["checkConnectionHealth"]) {
async function load(companyId?: string, connectionId?: string): Promise<ConnectionHealthDeadline[]> {
const rows = await db.select({
id: toolConnections.id,
companyId: toolConnections.companyId,
checkedAt: toolConnections.healthCheckedAt,
applicationKey: toolApplications.applicationKey,
sourceTemplateKey: sql<string | null>`coalesce(${toolConnections.config}->>'sourceTemplateKey', ${toolConnections.transportConfig}->>'sourceTemplateKey')`,
}).from(toolConnections).innerJoin(toolApplications, and(
eq(toolApplications.id, toolConnections.applicationId),
eq(toolApplications.companyId, toolConnections.companyId),
)).where(and(
eq(toolConnections.enabled, true), eq(toolConnections.status, "active"),
ne(toolConnections.transport, "chat_sdk"), ne(toolConnections.transport, "runtime_auth"),
ne(toolApplications.type, "paperclip_plugin"),
companyId === undefined ? undefined : eq(toolConnections.companyId, companyId),
connectionId === undefined ? undefined : eq(toolConnections.id, connectionId),
));
const definitions = rows.map(row => ({ row, app: connectionSetupApp(row.applicationKey, row.sourceTemplateKey) }));
const settings = definitions.some(({ app }) => app?.experimentalFeatureKey)
? await instanceSettingsService(db).getExperimental() : null;
return definitions.filter(({ app }) => !app || !settings || isConnectorFeatureEnabled(app, settings))
.map(({ row }) => ({ id: row.id, companyId: row.companyId,
dueAt: row.checkedAt === null ? 0 : row.checkedAt.getTime() + HEALTH_FRESHNESS_MS }));
}
async function check(id: string): Promise<ConnectionHealthDeadline | null> {
const [before] = await load(undefined, id);
if (!before || before.dueAt > Date.now()) return before ?? null;
let failure: unknown;
try { await checkHealth(id, { actorType: "system", actorId: "tool_health_sweep" }); }
catch (error) { failure = error; }
const [after] = await load(before.companyId, id);
if (failure && after && after.dueAt <= Date.now()) throw failure;
return after ?? null;
}
return { load, check };
}
server/src/services/tool-access/catalog/health.ts
+3 -2
View File
@@ -1,3 +1,4 @@
import { writeToolConnectionHealthChange } from "../../tool-connection-health-notifications.js";
import { assertSupportedConnection } from "../connections/setup-policy.js";
import { connectionSetupUrl, connectionReconnectUrl } from "../identifiers.js";
import { createGitHubGrantContinuity } from "../github-grant-continuity.js";
@@ -46,7 +47,7 @@ export function createConnectionHealth(db: Db, { now, audit, getConnectionRow, d
message: string | null,
) {
const now = new Date();
const [updated] = await db
const [updated] = await writeToolConnectionHealthChange(db, connection.companyId, tx => tx
.update(toolConnections)
.set({
healthStatus: status,
@@ -57,7 +58,7 @@ export function createConnectionHealth(db: Db, { now, audit, getConnectionRow, d
updatedAt: now,
})
.where(eq(toolConnections.id, connection.id))
.returning();
.returning());
if (connection.transport === "local_stdio") {
await db
.update(toolRuntimeSlots)
server/src/services/tool-access/catalog/publication.ts
+3 -2
View File
@@ -1,3 +1,4 @@
import { writeToolConnectionHealthChange } from "../../tool-connection-health-notifications.js";
import { connectorToolPolicy } from "../../connectors/tool-policy.js";
import { connectionSetupUrl, connectionReconnectUrl } from "../identifiers.js";
import { createToolRuntimeSlots } from "../runtime/slots.js";
@@ -268,7 +269,7 @@ export function createToolCatalogPublication(db: Db, { now, audit, getConnection
: refreshOptions.enableAllByDefault
? { ...connection.transportConfig, quarantineNewEntries: false }
: connection.transportConfig;
const [updatedConnection] = await db
const [updatedConnection] = await writeToolConnectionHealthChange(db, connection.companyId, tx => tx
.update(toolConnections)
.set({
config: normalizedConfig,
@@ -286,7 +287,7 @@ export function createToolCatalogPublication(db: Db, { now, audit, getConnection
updatedAt: refreshedAt,
})
.where(eq(toolConnections.id, connection.id))
.returning();
.returning());
if (connection.transport === "local_stdio") {
await ensureRuntimeSlot(updatedConnection);
server/src/services/tool-access/connections/removal.ts
+2
View File
@@ -1,3 +1,4 @@
import { notifyToolConnectionHealth } from "../../tool-connection-health-notifications.js";
import { toConnection } from "./projection.js";
import { actorBinding } from "../actor.js";
import { vercelConnectResourcesFor } from "../vercel-credentials.js";
@@ -536,6 +537,7 @@ async function removeConnection(
.returning({ id: connectionTokenIssuances.id });
const archived = await db.transaction(async (tx) => {
await notifyToolConnectionHealth(tx, connection.companyId);
await tx.select({ id: toolApplications.id }).from(toolApplications)
.where(and(eq(toolApplications.id, connection.applicationId), eq(toolApplications.companyId, connection.companyId)))
.for("update");
server/src/services/tool-access/connections/setup-policy.ts
+6 -2
View File
@@ -14,6 +14,11 @@ export function assertSupportedConnection(connection: typeof toolConnections.$in
}
}
export function connectionSetupApp(applicationKey: unknown, sourceTemplateKey: unknown) {
return applicationKey != null ? getAppDefinitionForApplicationKey(applicationKey)
: typeof sourceTemplateKey === "string" ? getConnectableAppDefinition(sourceTemplateKey) : null;
}
export function createConnectionSetupPolicy(db: Db) {
async function assertExperimentalConnectorSetupEnabled(provider: unknown, existing = false) {
const app = typeof provider === "string" ? getConnectableAppDefinition(provider) : null;
@@ -36,8 +41,7 @@ export function createConnectionSetupPolicy(db: Db) {
resolvedApplicationKey = application?.applicationKey;
}
const marker = connection.config?.sourceTemplateKey ?? connection.transportConfig?.sourceTemplateKey;
const app = resolvedApplicationKey != null ? getAppDefinitionForApplicationKey(resolvedApplicationKey)
: typeof marker === "string" ? getConnectableAppDefinition(marker) : null;
const app = connectionSetupApp(resolvedApplicationKey, marker);
if (app) await assertExperimentalConnectorSetupEnabled(app.slug, connection.status !== "draft");
}
return { assertExperimentalConnectorSetupEnabled, assertConnectorConnectionSetupEnabled };
server/src/services/tool-access/oauth/access-finalization.ts
+33 -27
View File
@@ -1,3 +1,4 @@
import { notifyToolConnectionHealth, writeToolConnectionHealthChange } from "../../tool-connection-health-notifications.js";
import type { ActorInfo } from "../types.js";
import { connectionMethodForConnection } from "../connection-methods.js";
import { emitConnectionUpdated } from "../../connector-telemetry.js";
@@ -112,33 +113,37 @@ async function finalizeOAuthAccess(
) {
throw conflict("This connection already uses a company identity");
}
[connection] = await db
.update(toolConnections)
.set({
credentialPolicy: "per_user",
credentialRefs: connection.credentialRefs.filter(
(ref) => ref.name !== "oauth.access_token",
),
credentialSecretRefs: connection.credentialSecretRefs.filter(
(ref) =>
ref.configPath !== "oauth.access_token" &&
ref.configPath !== "oauth.refresh_token",
),
status: "active",
enabled: true,
updatedAt: new Date(),
})
.where(
and(
eq(toolConnections.id, connection.id),
eq(toolConnections.companyId, companyId),
),
)
.returning();
await syncCredentialBindings(
connection,
personalGrant.credentialSecretRefs,
);
connection = await writeToolConnectionHealthChange(db, companyId, async tx => {
const [updated] = await tx
.update(toolConnections)
.set({
credentialPolicy: "per_user",
credentialRefs: connection.credentialRefs.filter(
(ref) => ref.name !== "oauth.access_token",
),
credentialSecretRefs: connection.credentialSecretRefs.filter(
(ref) =>
ref.configPath !== "oauth.access_token" &&
ref.configPath !== "oauth.refresh_token",
),
status: "active",
enabled: true,
updatedAt: new Date(),
})
.where(
and(
eq(toolConnections.id, connection.id),
eq(toolConnections.companyId, companyId),
),
)
.returning();
await syncCredentialBindings(
updated,
personalGrant.credentialSecretRefs,
tx,
);
return updated;
});
} else if (
connection.credentialPolicy !== "shared" ||
connection.credentialSecretRefs.length === 0
@@ -323,6 +328,7 @@ async function finalizeOAuthAccess(
updatedAt: nowAt,
})
.where(eq(connectionGrants.id, personalGrant.id));
await notifyToolConnectionHealth(tx, companyId);
[connection] = await tx
.update(toolConnections)
.set({
server/src/services/tool-access/oauth/cloud-callback.ts
+2
View File
@@ -1,3 +1,4 @@
import { notifyToolConnectionHealth } from "../../tool-connection-health-notifications.js";
import { managedConnectorProfile } from "./client-settings.js";
import type { ActorInfo } from "../types.js";
import { oauthConfig, normalizeOauthScopes } from "./metadata.js";
@@ -355,6 +356,7 @@ export function createCloudOAuthCallback(db: Db, dependencies: Dependencies) {
scopes: [...profile.scopes],
},
};
await notifyToolConnectionHealth(tx, connection.companyId);
[connection] = await tx
.update(toolConnections)
.set({
server/src/services/tool-access/oauth/connection-refresh.ts
+3 -2
View File
@@ -1,3 +1,4 @@
import { writeToolConnectionHealthChange } from "../../tool-connection-health-notifications.js";
import type { ActorInfo } from "../types.js";
import { createOAuthClientProtocol, storedOAuthTokenEndpointAuthMethod } from "./client.js";
import { oauthConfig, normalizeOauthScopes } from "./metadata.js";
@@ -59,7 +60,7 @@ const oauthRefreshFlights = new Map<string, Promise<unknown>>();
reauthorizationRequiredAt: now().toISOString(),
},
};
const [updated] = await db
const [updated] = await writeToolConnectionHealthChange(db, connection.companyId, tx => tx
.update(toolConnections)
.set({
status: "draft",
@@ -87,7 +88,7 @@ const oauthRefreshFlights = new Map<string, Promise<unknown>>();
)`,
),
)
.returning();
.returning());
if (updated) {
await syncCredentialBindings(updated);
emitConnectionUpdated(updated, connection, "credential_refresh");
server/src/services/tool-access/oauth/direct-callback.ts
+3
View File
@@ -1,3 +1,4 @@
import { notifyToolConnectionHealth } from "../../tool-connection-health-notifications.js";
import { credentialScope } from "./client-settings.js";
import { withoutOAuthRefreshLease } from "./refresh-leases.js";
import type { ActorInfo } from "../types.js";
@@ -331,6 +332,7 @@ export function createDirectOAuthCallback(db: Db, dependencies: Dependencies) {
},
},
};
await notifyToolConnectionHealth(tx, connection.companyId);
const [updatedConnection] = await tx
.update(toolConnections)
.set({
@@ -610,6 +612,7 @@ export function createDirectOAuthCallback(db: Db, dependencies: Dependencies) {
},
},
};
await notifyToolConnectionHealth(tx, connection.companyId);
const [updatedConnection] = await tx
.update(toolConnections)
.set({
server/src/services/tool-access/oauth/grant-refresh.ts
+3 -2
View File
@@ -1,3 +1,4 @@
import { writeToolConnectionHealthChange } from "../../tool-connection-health-notifications.js";
import type { ActorInfo } from "../types.js";
import { createOAuthClientProtocol, storedOAuthTokenEndpointAuthMethod } from "./client.js";
import { oauthConfig, normalizeOauthScopes, resolveGrantedOauthScopes } from "./metadata.js";
@@ -393,7 +394,7 @@ export function createOAuthGrantRefresh(db: Db, { now, store, leases, currentClo
connection.id,
connection.companyId,
);
const [reauthorizationRequired] = await db
const [reauthorizationRequired] = await writeToolConnectionHealthChange(db, latestConnection.companyId, tx => tx
.update(toolConnections)
.set({
status: "draft",
@@ -417,7 +418,7 @@ export function createOAuthGrantRefresh(db: Db, { now, store, leases, currentClo
eq(toolConnections.companyId, latestConnection.companyId),
),
)
.returning();
.returning());
await syncCredentialBindings(reauthorizationRequired);
emitConnectionUpdated(
reauthorizationRequired,
server/src/services/tool-access/oauth/vercel-callback.ts
+3 -2
View File
@@ -1,3 +1,4 @@
import { writeToolConnectionHealthChange } from "../../tool-connection-health-notifications.js";
import { vercelConnectHttpError, vercelCredentialFor, vercelConnectResourcesFor } from "../vercel-credentials.js";
import type { ActorInfo } from "../types.js";
import { connectionMethodForConnection, projectedConnectionHeaders } from "../connection-methods.js";
@@ -183,7 +184,7 @@ export function createVercelOAuthCallback(db: Db, dependencies: Dependencies) {
status: connection.status,
enabled: connection.enabled,
};
[connection] = await db
[connection] = await writeToolConnectionHealthChange(db, connection.companyId, tx => tx
.update(toolConnections)
.set({
status: "active",
@@ -198,7 +199,7 @@ export function createVercelOAuthCallback(db: Db, dependencies: Dependencies) {
eq(toolConnections.companyId, connection.companyId),
),
)
.returning();
.returning());
emitConnectionUpdated(
connection,
preActivationLifecycle,
server/src/services/tool-connection-health-notifications.ts
+12
View File
@@ -0,0 +1,12 @@
import type { Db } from "@paperclipai/db";
import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js";
export const notifyToolConnectionHealth = (transaction: object, companyId: string) =>
notifyDeliveryWork(transaction, DELIVERY_QUEUES.toolConnectionHealth, companyId);
export function writeToolConnectionHealthChange<T>(db: Db, companyId: string, write: (transaction: Db) => Promise<T>): Promise<T> {
return db.transaction(async tx => {
await notifyToolConnectionHealth(tx, companyId);
return write(tx as unknown as Db);
});
}
server/src/services/tool-connection-health-work.ts
+73
View File
@@ -0,0 +1,73 @@
import type { createDeliveryWorkCoordinator } from "./delivery-work-coordinator.js";
import { DELIVERY_QUEUES } from "./delivery-work-notifications.js";
import type { ConnectionHealthDeadline } from "./tool-access/catalog/health-schedule.js";
export function registerToolConnectionHealthWork(
coordinator: ReturnType<typeof createDeliveryWorkCoordinator>,
service: {
load(companyId?: string): Promise<ConnectionHealthDeadline[]>;
check(id: string): Promise<ConnectionHealthDeadline | null>;
},
canRun: () => boolean,
) {
const dirty = new Set<string>(["*"]);
const deadlines = new Map<string, ConnectionHealthDeadline>();
const failures = new Map<string, { count: number; retryAt: number }>();
const retryMs = 30_000;
let earliest = 0;
function request(hints?: ReadonlySet<string>) {
for (const hint of hints ?? ["*"]) dirty.add(hint);
}
function next() {
let result: number | null = dirty.size ? earliest : null;
for (const entry of deadlines.values()) {
const at = Math.max(earliest, entry.dueAt, failures.get(entry.id)?.retryAt ?? 0);
result = result === null ? at : Math.min(result, at);
}
return result;
}
return coordinator.register(DELIVERY_QUEUES.toolConnectionHealth, {
retryMs,
pausePassiveMaintenance: true,
onSettled: request,
run: async signal => {
if (!canRun() || Date.now() < earliest) return;
earliest = Date.now() + retryMs;
const scopes = dirty.has("*") ? ["*"] : [...dirty];
dirty.clear();
const errors: unknown[] = [];
const unavailable = new Set<string>();
for (let index = 0; index < scopes.length; index++) {
const scope = scopes[index];
if (signal.aborted || !canRun()) { for (const remaining of scopes.slice(index)) dirty.add(remaining); return; }
try {
const rows = await service.load(scope === "*" ? undefined : scope);
for (const entry of deadlines.values()) if (scope === "*" || entry.companyId === scope) deadlines.delete(entry.id);
for (const entry of rows) deadlines.set(entry.id, entry);
for (const id of failures.keys()) {
const current = deadlines.get(id);
if (!current || current.dueAt > Date.now()) failures.delete(id);
}
} catch (error) { dirty.add(scope); unavailable.add(scope); errors.push(error); }
}
const due = [...deadlines.values()].filter(entry => !unavailable.has("*") && !unavailable.has(entry.companyId)
&& Math.max(entry.dueAt, failures.get(entry.id)?.retryAt ?? 0) <= Date.now())
.sort((a, b) => a.dueAt - b.dueAt || a.id.localeCompare(b.id)).slice(0, 25);
for (const entry of due) {
if (signal.aborted || !canRun()) break;
try {
const current = await service.check(entry.id);
if (current) deadlines.set(entry.id, current); else deadlines.delete(entry.id);
failures.delete(entry.id);
} catch (error) {
const count = Math.min((failures.get(entry.id)?.count ?? 0) + 1, 6);
failures.set(entry.id, { count, retryAt: Date.now() + Math.min(15 * 60_000, retryMs * 2 ** (count - 1)) });
errors.push(error);
}
}
if (errors.length) throw new AggregateError(errors, "Tool connection health checks could not settle");
},
hasPending: async () => dirty.size > 0,
nextRunAt: next,
});
}
server/src/services/tool-gateway.ts
+3 -2
View File
@@ -1,3 +1,4 @@
import { writeToolConnectionHealthChange } from "./tool-connection-health-notifications.js";
import { writeApprovalChange } from "./approval-work-notifications.js";
import { toolReviewProjectionService } from "./tool-review-projection.js";
import { toolReviewInterruptionService } from "./tool-review-interruption.js";
@@ -3804,7 +3805,7 @@ export function createToolGatewayService(
message: string | null,
) {
const now = new Date();
await db
await writeToolConnectionHealthChange(db, connection.companyId, tx => tx
.update(toolConnections)
.set({
healthStatus: status,
@@ -3814,7 +3815,7 @@ export function createToolGatewayService(
lastError: status === "ok" ? null : message,
updatedAt: now,
})
.where(eq(toolConnections.id, connection.id));
.where(eq(toolConnections.id, connection.id)));
}
function grantRefForCredential(