Merge latest Codex qualification and master fixes

Co-Authored-By: Paperclip <noreply@paperclip.ing>

* codex/claude-runner-default: (25 commits)
  fix: pin ARM qualification helper to workflow source
  perf(ci): cache the Runner's release crate artifacts keyed by their inputs (#15932)
  perf(build): reuse a current Runner build in the server's prepare:runner-vendor step (#15926)
  fix(test): make the vitest home teardown remove read-only runtime-context bundles (#15931)
  fix(test): stop the Sentry timeout assertion matching a random event id (#15929)
  test: qualify Linux ARM with stock sandbox prerequisites
  docs(release): align 2026.1006.0-beta.0 stable notes header with v2026.1012.0 (#16000)
  fix: use Hermes adapter API URL env in prompt (#9425)
  fix(docker): skip the full-tree ownership walk on routine boots (#15992)
  refactor(chat): isolate Telegram, Slack, GitHub and Discord workflows (#15976)
  Preserve repository dependencies when importing GitHub skills (#15731)
  refactor(heartbeat): extract instruction working-copy lifecycle (#15984)
  test: retire redundant suites and consolidate owner coverage (#15978)
  refactor(heartbeat): extract runtime service coordination (#15980)
  refactor(heartbeat): extract execution configuration (#15973)
  refactor(heartbeat): extract run output recording (#15969)
  refactor(chat): extract failed-run retry coordination (#15845)
  refactor(chat): extract publication replacement decisions (#15844)
  refactor(chat): extract identity linking (#15836)
  refactor(heartbeat): extract session preparation (#15966)
  ...

# Conflicts:
#	packages/paperclip-runner/package.json
This commit is contained in:
Dotta committed 2026-10-12 13:45:03 -05:00
commit adf258ede2
210 files changed
+25068 -25979

No files matched your search

File diff suppressed because it is too large. Load diff
@@ -0,0 +1,639 @@
import type { EndpointRow, DbTransaction, DbOrTransaction, RuntimeContext, LifecycleRuntimeFence } from "../runtime-types.js";
import { chatCredentialMutationLease, type CredentialMutationLeaseGuard } from "../../chat-credential-mutation-lease.js";
import {
and,
desc,
eq,
gt,
} from "drizzle-orm";
import {
readChatControlChronology,
} from "../../chat-control-chronology.js";
import type { Db } from "@paperclipai/db";
import {
chatActions,
chatConversations,
chatEndpointLeases,
chatEndpointResources,
chatEndpoints,
issues,
toolConnections,
} from "@paperclipai/db";
import type {
ChatEndpointSetupState,
SafeChatPublicationPayload,
ToolCredentialSecretRef,
} from "@paperclipai/shared";
import {
conflict,
notFound,
} from "../../../errors.js";
import { logger } from "../../../middleware/logger.js";
import {
type ChatSdkCallbackEvent,
type DiscordNativeCommandResponse,
} from "../../chat-sdk-runtime.js";
import {
parseDiscordNativeCommand,
parseDiscordNativeCommandReceipt,
type DiscordNativeCommandTarget,
} from "../../chat-discord-native-commands.js";
import {
logActivity,
} from "../../activity-log.js";
import { projectSafeChatPublication } from "../../chat-publication-projection.js";
import {
readRegisteredDiscordCommandRegistration,
reconcileStoredDiscordCommandRegistration,
} from "../../chat-discord-command-registration-store.js";
import type {
Author,
SlashCommandEvent,
} from "chat";
import type { EndpointRecord } from "../runtime-types.js";
type DiscordCommandsOptions = {
endpointRecord: (endpointId: string) => Promise<EndpointRecord | null>;
runtimeContextForRecord: (record: EndpointRecord) => RuntimeContext;
runtimeGeneration: (setup: ChatEndpointSetupState) => number;
credentialFingerprint: (refs: ToolCredentialSecretRef[]) => string;
resolveCredentialRefs: (endpoint: EndpointRow, refs: ToolCredentialSecretRef[], database?: DbOrTransaction) => Promise<Record<string, string>>;
fetchImpl: typeof fetch;
withCredentialMutationLease: ReturnType<typeof chatCredentialMutationLease>;
runtimeCallbackRecord: (endpointId: string, context: RuntimeContext, statuses: EndpointRow["status"][]) => Promise<EndpointRecord | null>;
ensurePrincipal: (endpoint: EndpointRow, author: Author, raw?: unknown) => Promise<{ principal: { id: string }; userId: string | null }>;
runtimeCallbackEndpoint: (tx: DbTransaction, endpointId: string, context: LifecycleRuntimeFence, statuses: EndpointRow["status"][]) => Promise<EndpointRow | null>;
nonDirectDestinationAllowed: (endpoint: EndpointRow, resource: typeof chatEndpointResources.$inferSelect | null | undefined) => boolean;
lockCurrentPrincipalAuthorization: (tx: DbOrTransaction, endpoint: EndpointRow, principalId: string) => Promise<{ allowed: boolean; userId: string | null }>;
getDiscordGatewayOwnership: (endpointId: string) => { leaseKey: string; token: string } | undefined;
discordGatewayRuntimeIsCurrent: (endpointId: string, context: RuntimeContext) => boolean;
stageAuthorizedTaskControlPublication: (tx: DbOrTransaction, input: { companyId: string; conversationId: string; endpointId: string; idempotencyKey: string; issueId: string; payload: SafeChatPublicationPayload; principalId: string }) => Promise<{ id: string }>;
scheduleMessageProcessing: (task: () => Promise<void>) => void;
processPendingPublications: () => Promise<unknown>;
};
export function createDiscordCommands(db: Db, options: DiscordCommandsOptions) {
const { endpointRecord, runtimeContextForRecord, runtimeGeneration, credentialFingerprint, resolveCredentialRefs, fetchImpl, withCredentialMutationLease, runtimeCallbackRecord, ensurePrincipal, runtimeCallbackEndpoint, nonDirectDestinationAllowed, lockCurrentPrincipalAuthorization, getDiscordGatewayOwnership, discordGatewayRuntimeIsCurrent, stageAuthorizedTaskControlPublication, scheduleMessageProcessing, processPendingPublications } = options;
async function reconcileDiscordCommands(
endpointId: string,
existingLease?: CredentialMutationLeaseGuard,
force = false,
): Promise<void> {
const initial = await endpointRecord(endpointId);
if (!initial || initial.endpoint.provider !== "discord") return;
const reconcile = async (lease: CredentialMutationLeaseGuard) => {
const record = await endpointRecord(endpointId);
if (
!record ||
record.endpoint.provider !== "discord" ||
!["verifying", "active", "attention"].includes(
record.endpoint.status,
) ||
!record.endpoint.botExternalId ||
!record.endpoint.providerAccountId
)
return;
const scope = {
companyId: record.endpoint.companyId,
endpointId,
applicationId: record.endpoint.botExternalId,
guildId: record.endpoint.providerAccountId,
};
const fence = runtimeContextForRecord(record);
const authorize = async (tx: DbTransaction) => {
const endpoint = await tx
.select()
.from(chatEndpoints)
.where(
and(
eq(chatEndpoints.companyId, scope.companyId),
eq(chatEndpoints.id, scope.endpointId),
),
)
.for("no key update")
.then((rows) => rows[0]);
if (
!endpoint ||
endpoint.provider !== "discord" ||
!["verifying", "active", "attention"].includes(endpoint.status) ||
endpoint.botExternalId !== scope.applicationId ||
endpoint.providerAccountId !== scope.guildId ||
runtimeGeneration(endpoint.setup) !== fence.generation
)
throw conflict("Discord command registration authority changed");
const connection = await tx
.select()
.from(toolConnections)
.where(
and(
eq(toolConnections.companyId, scope.companyId),
eq(toolConnections.id, endpoint.connectionId),
),
)
.for("no key update")
.then((rows) => rows[0]);
if (
!connection?.enabled ||
connection.status !== "active" ||
credentialFingerprint(connection.credentialSecretRefs) !==
fence.credentialFingerprint
)
throw conflict("Discord command registration authority changed");
await lease.assertOwned(tx);
};
const setCapability = async (tx: DbTransaction, registered: boolean) => {
const current = await tx
.select()
.from(chatEndpoints)
.where(
and(
eq(chatEndpoints.companyId, scope.companyId),
eq(chatEndpoints.id, endpointId),
),
)
.then((rows) => rows[0]);
if (!current) throw notFound("Chat endpoint not found");
if (
current.capabilities.slashCommands === registered &&
current.capabilities.ephemeralMessages === registered
)
return;
await tx
.update(chatEndpoints)
.set({
capabilities: {
...current.capabilities,
slashCommands: registered,
ephemeralMessages: registered,
},
updatedAt: new Date(),
})
.where(
and(
eq(chatEndpoints.companyId, scope.companyId),
eq(chatEndpoints.id, endpointId),
),
);
await logActivity(tx as unknown as Db, {
companyId: scope.companyId,
actorType: "system",
actorId: "system",
action: registered
? "chat_endpoint.commands_registered"
: "chat_endpoint.commands_unavailable",
entityType: "tool_connection",
entityId: current.connectionId,
details: { endpointId, provider: "discord" },
});
};
await lease.assertOwned();
const credentials = await resolveCredentialRefs(
record.endpoint,
record.credentialSecretRefs,
);
if (
credentials.applicationId !== scope.applicationId ||
credentials.guildId !== scope.guildId
)
throw conflict("Discord command registration identity changed");
const result = await reconcileStoredDiscordCommandRegistration(db, {
scope,
runtimeFence: {
generation: fence.generation,
credentialFingerprint: fence.credentialFingerprint,
},
botToken: credentials.botToken,
fetch: fetchImpl,
authorize,
force,
onState: async (tx, state) =>
setCapability(tx, state.phase === "registered"),
});
if (result.kind === "deferred") return;
await db.transaction(async (tx) => {
await authorize(tx);
const registered =
result.kind === "registered"
? await readRegisteredDiscordCommandRegistration(tx, scope)
: null;
await setCapability(tx, registered !== null);
await lease.assertOwned(tx);
});
};
try {
if (existingLease) await reconcile(existingLease);
else await withCredentialMutationLease(initial.endpoint, reconcile);
} catch {
// Commands are additive. Do not fail an otherwise working mention/thread
// connection or expose provider errors/tokens when registration is busy.
logger.warn(
{ endpointId },
"Discord command registration remains unconfirmed",
);
}
}
async function handleDiscordNativeCommand(
event: ChatSdkCallbackEvent<SlashCommandEvent>,
runtimeContext: RuntimeContext,
): Promise<DiscordNativeCommandResponse> {
const denied: DiscordNativeCommandResponse = { kind: "denied" };
const record = await runtimeCallbackRecord(
event.endpointId,
runtimeContext,
["active"],
);
if (
!record ||
record.endpoint.provider !== "discord" ||
record.endpoint.capabilities.slashCommands !== true ||
event.transport !== "discord_gateway" ||
runtimeContext.discordGatewayOwned !== true
)
return denied;
const scope = {
companyId: record.endpoint.companyId,
endpointId: record.endpoint.id,
applicationId: record.endpoint.botExternalId ?? "",
guildId: record.endpoint.providerAccountId ?? "",
};
const invocation = parseDiscordNativeCommand(event, scope);
if (!invocation) return denied;
const initialRegistration = await readRegisteredDiscordCommandRegistration(
db,
scope,
);
if (
initialRegistration?.receipt.commandId !== invocation.registeredCommandId
)
return denied;
const principal = await ensurePrincipal(
record.endpoint,
event.event.user,
event.event.raw,
);
const authorityChanged = new Error("Discord command authority changed");
let publish = false;
try {
const response = await db.transaction(
async (tx): Promise<DiscordNativeCommandResponse> => {
const endpoint = await runtimeCallbackEndpoint(
tx,
scope.endpointId,
runtimeContext,
["active"],
);
if (
!endpoint ||
endpoint.companyId !== scope.companyId ||
endpoint.provider !== "discord" ||
endpoint.botExternalId !== scope.applicationId ||
endpoint.providerAccountId !== scope.guildId ||
endpoint.capabilities.slashCommands !== true
)
return denied;
const connection = await tx
.select()
.from(toolConnections)
.where(
and(
eq(toolConnections.companyId, scope.companyId),
eq(toolConnections.id, endpoint.connectionId),
),
)
.for("no key update")
.then((rows) => rows[0] ?? null);
if (
!connection ||
!connection.enabled ||
connection.status !== "active" ||
credentialFingerprint(connection.credentialSecretRefs) !==
runtimeContext.credentialFingerprint
)
return denied;
const registration = await readRegisteredDiscordCommandRegistration(
tx,
scope,
true,
);
if (
registration?.receipt.commandId !== invocation.registeredCommandId
)
return denied;
const providerActionId = `discord-native-command:${invocation.interactionId}`;
const prior = await tx
.select()
.from(chatActions)
.where(
and(
eq(chatActions.companyId, scope.companyId),
eq(chatActions.endpointId, scope.endpointId),
eq(chatActions.providerActionId, providerActionId),
),
)
.for("update")
.then((rows) => rows[0] ?? null);
const receipt = prior
? parseDiscordNativeCommandReceipt(prior, invocation, scope)
: null;
if (
prior &&
(!receipt ||
receipt.principalId !== principal.principal.id ||
receipt.runtimeFence.generation !== runtimeContext.generation ||
receipt.runtimeFence.credentialFingerprint !==
runtimeContext.credentialFingerprint)
)
return denied;
// A replay selects only its original target, including a completed
// generation. Never reinterpret an old close as a new task's command.
const conversation = receipt
? receipt.target
? await tx
.select()
.from(chatConversations)
.where(
and(
eq(chatConversations.companyId, scope.companyId),
eq(chatConversations.endpointId, scope.endpointId),
eq(chatConversations.id, receipt.target.conversationId),
),
)
.for("update")
.then((rows) => rows[0] ?? null)
: null
: invocation.sourceKind !== "guild_channel"
? await tx
.select()
.from(chatConversations)
.where(
and(
eq(chatConversations.companyId, scope.companyId),
eq(chatConversations.endpointId, scope.endpointId),
eq(
chatConversations.externalConversationId,
invocation.channelId,
),
eq(
chatConversations.externalThreadId,
invocation.threadId,
),
),
)
.orderBy(desc(chatConversations.sessionGeneration))
.limit(1)
.for("update")
.then((rows) => rows[0] ?? null)
: null;
if (
(receipt?.target && !conversation) ||
(conversation &&
(conversation.externalConversationId !== invocation.channelId ||
conversation.externalThreadId !== invocation.threadId ||
conversation.isDirectMessage !==
(invocation.sourceKind === "direct_message") ||
!["active", "waiting", "completed"].includes(
conversation.state,
) ||
(receipt?.target &&
(conversation.issueId !== receipt.target.issueId ||
conversation.sessionGeneration !==
receipt.target.sessionGeneration))))
)
return denied;
const resource =
invocation.sourceKind === "direct_message"
? null
: await tx
.select()
.from(chatEndpointResources)
.where(
and(
eq(chatEndpointResources.companyId, scope.companyId),
eq(chatEndpointResources.endpointId, scope.endpointId),
eq(
chatEndpointResources.providerResourceId,
invocation.providerResourceId,
),
),
)
.for("update")
.then((rows) => rows[0] ?? null);
if (
invocation.sourceKind === "direct_message"
? !endpoint.allowDirectMessages
: !nonDirectDestinationAllowed(endpoint, resource) ||
(conversation && conversation.resourceId !== resource?.id)
)
return denied;
const authorization = await lockCurrentPrincipalAuthorization(
tx,
endpoint,
principal.principal.id,
);
if (
!authorization.allowed ||
authorization.userId !== principal.userId
)
return denied;
const issue = conversation
? await tx
.select()
.from(issues)
.where(
and(
eq(issues.companyId, scope.companyId),
eq(issues.id, conversation.issueId),
),
)
.for("share")
.then((rows) => rows[0] ?? null)
: null;
if (conversation && !issue) return denied;
const ownership = getDiscordGatewayOwnership(scope.endpointId);
if (
!ownership ||
!discordGatewayRuntimeIsCurrent(scope.endpointId, runtimeContext)
)
return denied;
const lease = await tx
.select()
.from(chatEndpointLeases)
.where(
and(
eq(chatEndpointLeases.companyId, scope.companyId),
eq(chatEndpointLeases.endpointId, scope.endpointId),
eq(chatEndpointLeases.leaseKey, ownership.leaseKey),
eq(chatEndpointLeases.token, ownership.token),
gt(chatEndpointLeases.expiresAt, new Date()),
),
)
.for("update")
.then((rows) => rows[0] ?? null);
if (
!lease ||
!discordGatewayRuntimeIsCurrent(scope.endpointId, runtimeContext)
)
return denied;
if (receipt)
return { kind: "accepted", content: receipt.result.content };
const target: DiscordNativeCommandTarget | null = conversation
? {
conversationId: conversation.id,
issueId: conversation.issueId,
sessionGeneration: conversation.sessionGeneration,
}
: null;
const active =
conversation &&
issue &&
["active", "waiting"].includes(conversation.state) &&
(invocation.sourceKind !== "direct_message" ||
!["done", "cancelled"].includes(issue.status));
let content: string;
let publicationId: string | null = null;
if (invocation.command === "status") {
content =
issue && (invocation.sourceKind !== "direct_message" || active)
? `${issue.identifier}: ${issue.title} — ${issue.status}${conversation?.state === "completed" ? " (conversation closed)" : ""}`.slice(
0,
2000,
)
: invocation.sourceKind === "guild_channel"
? "Open the Discord task thread to view its Paperclip status."
: "No task is active in this conversation.";
} else if (
invocation.command === "new" &&
invocation.sourceKind !== "direct_message"
) {
content =
"Open a new Discord thread by mentioning this agent in a new channel message. This thread stays bound to its current Paperclip task.";
} else if (!active || !conversation || !issue) {
content =
invocation.sourceKind === "direct_message"
? "No task is active. Send a message to start a new Paperclip task."
: "No active task is bound here. Open its Discord thread to manage it.";
} else if (
(await readChatControlChronology(
tx,
record.endpoint,
{
id: conversation.externalThreadId,
channelId: conversation.externalConversationId,
},
{
receivedAt: new Date(),
normalizedEvent: {
message: { providerMessageId: invocation.interactionId },
},
},
)) === "before_or_unproven"
) {
content =
"This command predates an already completed chat control. Send a new command for the current conversation.";
} else {
const publicText =
invocation.command === "new"
? "Send your request to start a new Paperclip task."
: invocation.sourceKind === "direct_message"
? "This chat conversation is closed. Send another message to start a new task."
: "This chat conversation is closed. A later message here will continue the same Paperclip task.";
const publication = await stageAuthorizedTaskControlPublication(
tx,
{
companyId: scope.companyId,
endpointId: scope.endpointId,
conversationId: conversation.id,
issueId: issue.id,
principalId: principal.principal.id,
idempotencyKey: `control:${invocation.command}:discord:${scope.endpointId}:${invocation.interactionId}`,
payload: projectSafeChatPublication({
classification: "external",
source: "task_control",
text: publicText,
}),
},
);
publicationId = publication.id;
content = `${invocation.command === "new" ? "New-task" : "Close"} request recorded. Watch this conversation for confirmation.`;
}
const inserted = await tx
.insert(chatActions)
.values({
companyId: scope.companyId,
endpointId: scope.endpointId,
conversationId: target?.conversationId ?? null,
principalId: principal.principal.id,
kind: "discord_native_command",
providerActionId,
payload: {
version: 1,
invocation,
runtimeFence: {
generation: runtimeContext.generation,
credentialFingerprint: runtimeContext.credentialFingerprint,
},
target,
},
status: "processed",
result: {
kind: "discord_native_command_recorded",
content,
publicationId,
},
})
.onConflictDoNothing()
.returning({ id: chatActions.id });
if (!inserted.length) throw authorityChanged;
await logActivity(tx as unknown as Db, {
companyId: scope.companyId,
actorType: authorization.userId ? "user" : "system",
actorId: authorization.userId ?? `chat:${principal.principal.id}`,
action: "chat.discord_command_recorded",
entityType: "chat_endpoint",
entityId: scope.endpointId,
details: {
command: invocation.command,
interactionId: invocation.interactionId,
conversationId: target?.conversationId ?? null,
sessionGeneration: target?.sessionGeneration ?? null,
publicationId,
},
});
// No network work is performed under these locks. A stop while any DB
// await was in flight rolls back the command and its staged outbox row.
if (
!discordGatewayRuntimeIsCurrent(scope.endpointId, runtimeContext) ||
lease.expiresAt.getTime() <= Date.now()
)
throw authorityChanged;
publish = publicationId !== null;
return { kind: "accepted", content };
},
);
if (publish)
scheduleMessageProcessing(async () => {
await processPendingPublications();
});
return response;
} catch (error) {
if (error === authorityChanged) return denied;
throw error;
}
}
return { reconcileDiscordCommands, handleDiscordNativeCommand };
}
@@ -0,0 +1,30 @@
import { createHash } from "node:crypto";
export function discordLifecycleFilesDigest(raw: unknown): string {
const attachments =
raw && typeof raw === "object" && !Array.isArray(raw)
? (raw as Record<string, unknown>).attachments
: undefined;
// Match the pinned Gateway adapter's flattened, ordered file projection.
// Rotating CDN URLs are transport authority, never source revision identity.
const projection = Array.isArray(attachments)
? attachments.map((attachment) => {
const row =
attachment &&
typeof attachment === "object" &&
!Array.isArray(attachment)
? (attachment as Record<string, unknown>)
: {};
return [
...["id", "filename", "content_type"].map((key) =>
typeof row[key] === "string" ? row[key] : null,
),
typeof row.size === "number" && Number.isFinite(row.size)
? row.size
: null,
];
})
: [];
return createHash("sha256").update(JSON.stringify(projection)).digest("hex");
}
File diff suppressed because it is too large. Load diff
@@ -0,0 +1,7 @@
import type { chatExternalPrincipals } from "@paperclipai/db";
export function githubIdentityFields(
principal: Pick<typeof chatExternalPrincipals.$inferSelect, "externalId" | "handle">,
) {
return { githubUserId: principal.externalId, githubLogin: principal.handle };
}
@@ -0,0 +1,132 @@
import { createHash } from "node:crypto";
import { MAX_INBOUND_TEXT, type LifecycleActor } from "../message-lifecycle.js";
export type GitHubLifecycleEvent = {
actor?: LifecycleActor;
isBotMessage?: true;
eventKind: "message_updated" | "message_deleted";
messageId: string;
providerEventId?: string;
providerMessageSequence: number | null;
providerSentAt: string | null;
revision: string;
text: string;
threadId: string;
};
export async function githubLifecycleEventFromRequest(
request: Request,
): Promise<GitHubLifecycleEvent | null> {
const eventType = request.headers.get("x-github-event");
const deliveryId = request.headers.get("x-github-delivery")?.trim() || null;
if (
eventType !== "issue_comment" &&
eventType !== "pull_request_review_comment"
)
return null;
const payload = (await request.json()) as {
action?: unknown;
comment?: {
id?: unknown;
in_reply_to_id?: unknown;
body?: unknown;
updated_at?: unknown;
user?: { type?: unknown };
};
issue?: { number?: unknown; pull_request?: unknown };
pull_request?: { number?: unknown };
repository?: { name?: unknown; owner?: { login?: unknown } };
sender?: { id?: unknown; login?: unknown; type?: unknown };
};
if (payload.action !== "edited" && payload.action !== "deleted") return null;
const owner = payload.repository?.owner?.login;
const repo = payload.repository?.name;
const messageId = payload.comment?.id;
if (
typeof owner !== "string" ||
typeof repo !== "string" ||
(typeof messageId !== "string" && typeof messageId !== "number")
)
return null;
let threadId: string;
if (eventType === "issue_comment") {
const number = payload.issue?.number;
if (typeof number !== "number") return null;
threadId = payload.issue?.pull_request
? `github:${owner}/${repo}:${number}`
: `github:${owner}/${repo}:issue:${number}`;
} else {
const number = payload.pull_request?.number;
const rootCommentId =
payload.comment?.in_reply_to_id ?? payload.comment?.id;
if (
typeof number !== "number" ||
(typeof rootCommentId !== "string" && typeof rootCommentId !== "number")
)
return null;
threadId = `github:${owner}/${repo}:${number}:rc:${rootCommentId}`;
}
const eventKind =
payload.action === "edited" ? "message_updated" : "message_deleted";
const body =
typeof payload.comment?.body === "string"
? payload.comment.body.slice(0, MAX_INBOUND_TEXT)
: "";
const providerRevision =
typeof payload.comment?.updated_at === "string"
? payload.comment.updated_at
: payload.action;
const parsedProviderSentAt = new Date(providerRevision);
const providerSentAt = Number.isFinite(parsedProviderSentAt.getTime())
? parsedProviderSentAt.toISOString()
: null;
const numericMessageId =
typeof messageId === "number"
? messageId
: typeof messageId === "string" && /^\d+$/.test(messageId)
? Number(messageId)
: null;
const providerMessageSequence =
numericMessageId !== null &&
Number.isSafeInteger(numericMessageId) &&
numericMessageId >= 0
? numericMessageId
: null;
const senderId = payload.sender?.id;
const senderLogin = payload.sender?.login;
const actor =
(typeof senderId === "string" || typeof senderId === "number") &&
typeof senderLogin === "string" &&
senderLogin.length > 0
? {
externalId: String(senderId),
displayName: senderLogin,
handle: senderLogin,
}
: undefined;
return {
...(actor ? { actor } : {}),
...(payload.comment?.user?.type === "Bot" || payload.sender?.type === "Bot"
? { isBotMessage: true as const }
: {}),
eventKind,
messageId: String(messageId),
...(deliveryId ? { providerEventId: `github:delivery:${deliveryId}` } : {}),
providerMessageSequence,
providerSentAt,
// GitHub's updated_at value can have coarser resolution than a quick
// sequence of edits. Include the normalized body so distinct edits at the
// same timestamp remain durable while an exact webhook redelivery still
// deduplicates.
revision:
eventKind === "message_updated"
? `${providerRevision}:${createHash("sha256").update(body).digest("hex")}`
: providerRevision,
text:
eventKind === "message_updated"
? `An external message was edited:\n\n${body}`
: "An external message in this conversation was deleted.",
threadId,
};
}
@@ -0,0 +1,278 @@
import { and, asc, eq, sql } from "drizzle-orm";
import { chatConversations, chatEndpointResources, chatEndpoints, type Db } from "@paperclipai/db";
import { conflict, notFound } from "../../../errors.js";
import type { CredentialMutationLeaseGuard } from "../../chat-credential-mutation-lease.js";
import type { ChatProviderResourceInventoryItem } from "../../chat-provider-inventory.js";
type EndpointRow = typeof chatEndpoints.$inferSelect;
type DbTransaction = Parameters<Parameters<Db["transaction"]>[0]>[0];
export function githubRepositoryInventoryItemFromPayload(
payload: unknown,
): ChatProviderResourceInventoryItem | null {
if (!payload || typeof payload !== "object" || Array.isArray(payload))
return null;
const repository = (payload as { repository?: unknown }).repository;
if (
!repository ||
typeof repository !== "object" ||
Array.isArray(repository)
)
return null;
const value = repository as {
id?: unknown;
full_name?: unknown;
html_url?: unknown;
name?: unknown;
owner?: { id?: unknown; login?: unknown };
private?: unknown;
};
const repositoryId =
typeof value.id === "number" && Number.isSafeInteger(value.id)
? String(value.id)
: typeof value.id === "string" && /^\d+$/.test(value.id)
? value.id
: null;
const owner =
typeof value.owner?.login === "string" ? value.owner.login.trim() : "";
const name = typeof value.name === "string" ? value.name.trim() : "";
const fullName =
typeof value.full_name === "string" && value.full_name.includes("/")
? value.full_name.trim()
: owner && name
? `${owner}/${name}`
: "";
if (!repositoryId || !fullName) return null;
const ownerId =
typeof value.owner?.id === "number" && Number.isSafeInteger(value.owner.id)
? String(value.owner.id)
: typeof value.owner?.id === "string" && /^\d+$/.test(value.owner.id)
? value.owner.id
: undefined;
return {
providerResourceId: fullName.toLowerCase(),
parentProviderResourceId: ownerId,
type: "repository",
label: fullName,
providerUrl:
typeof value.html_url === "string" && value.html_url.length > 0
? value.html_url
: `https://github.com/${fullName}`,
metadata: {
providerRepositoryId: repositoryId,
fullName,
...(owner ? { owner } : {}),
private: value.private === true,
source: "provider_webhook",
},
};
}
function githubRepositoryStableId(
item: ChatProviderResourceInventoryItem,
): string | null {
const value = item.metadata?.providerRepositoryId;
return typeof value === "string" && /^\d+$/.test(value) ? value : null;
}
export async function reconcileGitHubRepositoryCoordinates(tx: DbTransaction, endpoint: EndpointRow, item: ChatProviderResourceInventoryItem) {
const stableGitHubId =
githubRepositoryStableId(item);
if (stableGitHubId) {
await tx.execute(
sql`select pg_advisory_xact_lock(hashtextextended(${`chat-github-repository:${endpoint.id}:${stableGitHubId}`}, 0))`,
);
const stableResource = await tx
.select()
.from(chatEndpointResources)
.where(
and(
eq(chatEndpointResources.companyId, endpoint.companyId),
eq(chatEndpointResources.endpointId, endpoint.id),
eq(chatEndpointResources.type, "repository"),
sql`${chatEndpointResources.metadata}->>'providerRepositoryId' = ${stableGitHubId}`,
),
)
.for("update")
.then((rows) => rows[0] ?? null);
if (
stableResource &&
stableResource.providerResourceId !== item.providerResourceId
) {
const coordinateResource = await tx
.select()
.from(chatEndpointResources)
.where(
and(
eq(chatEndpointResources.companyId, endpoint.companyId),
eq(chatEndpointResources.endpointId, endpoint.id),
eq(chatEndpointResources.type, "repository"),
eq(
chatEndpointResources.providerResourceId,
item.providerResourceId,
),
),
)
.for("update")
.then((rows) => rows[0] ?? null);
if (coordinateResource && coordinateResource.id !== stableResource.id) {
const coordinateConversation = await tx
.select({ id: chatConversations.id })
.from(chatConversations)
.where(
and(
eq(chatConversations.endpointId, endpoint.id),
eq(chatConversations.resourceId, coordinateResource.id),
),
)
.limit(1)
.then((rows) => rows[0] ?? null);
if (coordinateConversation) {
throw conflict(
"GitHub repository identity is already bound to two coordinates",
{ code: "chat_github_repository_identity_conflict" },
);
}
await tx
.delete(chatEndpointResources)
.where(eq(chatEndpointResources.id, coordinateResource.id));
}
const oldRepository = stableResource.providerResourceId;
const oldThreadPrefix = `github:${oldRepository}`;
const newThreadPrefix = `github:${item.providerResourceId}`;
const oldProviderUrl = `https://github.com/${oldRepository}`;
const newProviderUrl = `https://github.com/${item.providerResourceId}`;
await tx
.update(chatEndpointResources)
.set({
providerResourceId: item.providerResourceId,
parentProviderResourceId: item.parentProviderResourceId ?? null,
label: item.label,
providerUrl: item.providerUrl ?? null,
availability: "available",
enabled:
stableResource.enabled || coordinateResource?.enabled === true,
metadata: item.metadata ?? {},
updatedAt: new Date(),
})
.where(eq(chatEndpointResources.id, stableResource.id));
await tx
.update(chatConversations)
.set({
externalConversationId: sql<string>`case
when lower(${chatConversations.externalConversationId}) = ${oldRepository} then ${item.providerResourceId}
when lower(${chatConversations.externalConversationId}) = ${oldThreadPrefix} then ${newThreadPrefix}
else ${chatConversations.externalConversationId}
end`,
externalThreadId: sql<string>`case
when lower(${chatConversations.externalThreadId}) = ${oldThreadPrefix} then ${newThreadPrefix}
when lower(${chatConversations.externalThreadId}) like ${`${oldThreadPrefix}:%`}
then ${newThreadPrefix} || substring(
${chatConversations.externalThreadId}
from char_length(${oldThreadPrefix}) + 1
)
else ${chatConversations.externalThreadId}
end`,
externalLabel: item.label,
providerUrl: sql<string | null>`case
when lower(${chatConversations.providerUrl}) like ${`${oldProviderUrl}/%`}
then ${newProviderUrl} || substring(
${chatConversations.providerUrl}
from char_length(${oldProviderUrl}) + 1
)
else ${chatConversations.providerUrl}
end`,
updatedAt: new Date(),
})
.where(
and(
eq(chatConversations.companyId, endpoint.companyId),
eq(chatConversations.endpointId, endpoint.id),
eq(chatConversations.resourceId, stableResource.id),
),
);
return { id: stableResource.id };
}
}
return null;
}
type GitHubResourceOptions = {
endpointRecord: (endpointId: string) => Promise<{ endpoint: EndpointRow } | null>;
upsertProviderResourceRow: (tx: DbTransaction, endpoint: EndpointRow, item: ChatProviderResourceInventoryItem) => Promise<{ id: string } | null>;
updateResourceSelection: (endpointId: string, updates: Array<{ id: string; enabled: boolean }>, actorUserId?: string | null, options?: { allGitHubRepositoriesEnabled?: boolean }) => Promise<void>;
};
export function createGitHubResources(db: Db, options: GitHubResourceOptions) {
const { endpointRecord, upsertProviderResourceRow, updateResourceSelection } = options;
async function reconcileGitHubWebhookRepository(
endpoint: EndpointRow,
payload: unknown,
credentialLease: CredentialMutationLeaseGuard,
) {
const item = githubRepositoryInventoryItemFromPayload(payload);
if (!item) return;
await db.transaction(async (tx) => {
await credentialLease.assertOwned(tx);
const resource = await upsertProviderResourceRow(tx, endpoint, item);
if (!resource) return;
// A correctly signed repository callback is current provider proof. Keep
// Paperclip's enabled choice, and reopen only conversations previously
// quarantined for provider availability loss.
await tx
.update(chatConversations)
.set({ state: "active", updatedAt: new Date() })
.where(
and(
eq(chatConversations.companyId, endpoint.companyId),
eq(chatConversations.endpointId, endpoint.id),
eq(chatConversations.resourceId, resource.id),
eq(chatConversations.state, "unavailable"),
),
);
await credentialLease.assertOwned(tx);
});
}
async function listGitHubRepositories(
endpointId: string,
query: { limit: number; offset: number; search: string },
) {
const record = await endpointRecord(endpointId);
if (!record || record.endpoint.provider !== "github") throw notFound("GitHub bot not found");
const scope = and(
eq(chatEndpointResources.companyId, record.endpoint.companyId),
eq(chatEndpointResources.endpointId, endpointId),
eq(chatEndpointResources.type, "repository"),
);
// Literal, case-insensitive substring search (%, _ and backslashes are
// ordinary characters). Counts always describe the entire connection.
const search = query.search
? sql`strpos(lower(${chatEndpointResources.label}), lower(${query.search})) > 0`
: undefined;
const [rows, [counts]] = await Promise.all([
db.select().from(chatEndpointResources).where(and(scope, search))
.orderBy(asc(chatEndpointResources.label), asc(chatEndpointResources.id))
.offset(query.offset).limit(query.limit + 1),
db.select({
totalCount: sql<number>`count(*)::integer`,
enabledCount: sql<number>`count(*) filter (where ${chatEndpointResources.enabled})::integer`,
availableCount: sql<number>`count(*) filter (where ${chatEndpointResources.availability} = 'available')::integer`,
}).from(chatEndpointResources).where(scope),
]);
return {
items: rows.slice(0, query.limit),
nextOffset: rows.length > query.limit ? query.offset + query.limit : null,
totalCount: counts!.totalCount,
enabledCount: counts!.enabledCount,
availableCount: counts!.availableCount,
};
}
async function toggleAllGitHubRepositories(endpointId: string, enabled: boolean, actorUserId?: string | null) {
await updateResourceSelection(endpointId, [], actorUserId, { allGitHubRepositoriesEnabled: enabled });
return { success: true as const };
}
return { reconcileGitHubWebhookRepository, listGitHubRepositories, toggleAllGitHubRepositories };
}
File diff suppressed because it is too large. Load diff
@@ -0,0 +1,397 @@
import { githubIdentityFields } from "./github/identity-links.js";
import { createSlackIdentityLinks, type SlackIdentityEffectStager } from "./slack/identity-links.js";
import { createHash, randomBytes } from "node:crypto";
import { and, asc, eq, gt, inArray, ne, sql } from "drizzle-orm";
import type { Db } from "@paperclipai/db";
import { authUsers, chatEndpoints, chatExternalPrincipals, chatIdentityLinks, companies, companyMemberships } from "@paperclipai/db";
import { conflict, forbidden, notFound, unprocessable } from "../../errors.js";
type EndpointRow = typeof chatEndpoints.$inferSelect;
type ConfirmationRuntime = { credentialFingerprint: string; generation: number };
type ChatIdentityLinkOptions = {
endpointRecord: (endpointId: string) => Promise<{ endpoint: EndpointRow } | null>;
getPublicBaseUrl: () => string | null;
confirmationRuntimeFor: (endpoint: EndpointRow) => Promise<ConfirmationRuntime | undefined>;
stageProviderEffect: SlackIdentityEffectStager;
scheduleProviderEffect: (actionId: string) => void;
};
export function createChatIdentityLinks(db: Db, options: ChatIdentityLinkOptions) {
const { endpointRecord, getPublicBaseUrl, confirmationRuntimeFor, stageProviderEffect, scheduleProviderEffect } = options;
const slackIdentityLinks = createSlackIdentityLinks(db, { previewIdentityLink, stageProviderEffect });
async function listPrincipals(endpointId: string) {
const record = await endpointRecord(endpointId);
if (!record) throw notFound("Chat endpoint not found");
const principals = await db
.select()
.from(chatExternalPrincipals)
.where(
and(
eq(chatExternalPrincipals.companyId, record.endpoint.companyId),
eq(chatExternalPrincipals.provider, record.endpoint.provider),
eq(
chatExternalPrincipals.providerAccountId,
record.endpoint.providerAccountId ?? "unknown",
),
),
)
.orderBy(asc(chatExternalPrincipals.displayName));
const links = await db
.select()
.from(chatIdentityLinks)
.where(eq(chatIdentityLinks.endpointId, endpointId));
const connects = record.endpoint.provider === "slack" ? await slackIdentityLinks.listConnects(record.endpoint) : [];
const connectByPrincipal = new Map(connects.map((row) => [row.principalId, new Date(row.lastConnectAt).toISOString()]));
const userIds = links.flatMap((link) =>
link.paperclipUserId ? [link.paperclipUserId] : [],
);
const users = userIds.length
? await db
.select({
id: authUsers.id,
name: authUsers.name,
email: authUsers.email,
})
.from(authUsers)
.where(inArray(authUsers.id, userIds))
: [];
const linkByPrincipal = new Map(
links.map((link) => [link.principalId, link]),
);
const userById = new Map(users.map((user) => [user.id, user]));
return principals.map((principal) => {
const link = linkByPrincipal.get(principal.id);
const user = link?.paperclipUserId
? userById.get(link.paperclipUserId)
: null;
return {
id: link?.id ?? principal.id,
principalId: principal.id,
...(record.endpoint.provider === "github" ? githubIdentityFields(principal) : {}),
externalLabel:
principal.displayName ?? principal.handle ?? principal.externalId,
externalDetail: principal.handle
? `@${principal.handle.replace(/^@/, "")}`
: principal.externalId,
paperclipUserId: link?.paperclipUserId ?? null,
paperclipUserLabel: user?.name ?? user?.email ?? null,
status: link?.status ?? "pending",
lastConnectAt: connectByPrincipal.get(principal.id) ?? null,
};
});
}
async function createLinkIntent(
endpointId: string,
principalId: string,
expiresInSeconds: number,
) {
const record = await endpointRecord(endpointId);
if (!record) throw notFound("Chat endpoint not found");
const principal = await db
.select()
.from(chatExternalPrincipals)
.where(
and(
eq(chatExternalPrincipals.companyId, record.endpoint.companyId),
eq(chatExternalPrincipals.id, principalId),
eq(chatExternalPrincipals.provider, record.endpoint.provider),
eq(
chatExternalPrincipals.providerAccountId,
record.endpoint.providerAccountId ?? "",
),
),
)
.then((rows) => rows[0] ?? null);
if (!principal) throw notFound("External identity not found");
if (principal.isBot || principal.kind !== "user") {
throw unprocessable("Only a human external identity can be linked");
}
const token = randomBytes(32).toString("base64url");
const tokenHash = createHash("sha256").update(token).digest("hex");
const expiresAt = new Date(Date.now() + expiresInSeconds * 1000);
await db.transaction(async (tx) => {
await tx.execute(
sql`select pg_advisory_xact_lock(hashtextextended(${`chat-identity:${record.endpoint.companyId}:${principalId}`}, 0))`,
);
const existingLink = await tx
.select({ status: chatIdentityLinks.status })
.from(chatIdentityLinks)
.where(
and(
eq(chatIdentityLinks.endpointId, endpointId),
eq(chatIdentityLinks.principalId, principalId),
),
)
.for("update")
.then((rows) => rows[0] ?? null);
if (existingLink?.status === "linked") {
throw conflict("This external identity is already linked", {
code: "chat_identity_already_linked",
});
}
await tx
.insert(chatIdentityLinks)
.values({
companyId: record.endpoint.companyId,
endpointId,
principalId,
status: "pending",
confirmationTokenHash: tokenHash,
expiresAt,
})
.onConflictDoUpdate({
target: [chatIdentityLinks.endpointId, chatIdentityLinks.principalId],
set: {
paperclipUserId: null,
status: "pending",
confirmationTokenHash: tokenHash,
expiresAt,
confirmedAt: null,
revokedAt: null,
updatedAt: new Date(),
},
});
});
const path = `/chat-identity/confirm?token=${encodeURIComponent(token)}`;
return {
confirmationUrl: getPublicBaseUrl() ? `${getPublicBaseUrl()}${path}` : path,
expiresAt: expiresAt.toISOString(),
};
}
async function previewIdentityLink(token: string, userId?: string | null) {
const tokenHash = createHash("sha256").update(token).digest("hex");
const row = await db
.select({
link: chatIdentityLinks,
principal: chatExternalPrincipals,
endpoint: chatEndpoints,
companyName: companies.name,
companyPrefix: companies.issuePrefix,
})
.from(chatIdentityLinks)
.innerJoin(
chatExternalPrincipals,
and(
eq(chatExternalPrincipals.companyId, chatIdentityLinks.companyId),
eq(chatExternalPrincipals.id, chatIdentityLinks.principalId),
),
)
.innerJoin(
chatEndpoints,
and(
eq(chatEndpoints.companyId, chatIdentityLinks.companyId),
eq(chatEndpoints.id, chatIdentityLinks.endpointId),
),
)
.innerJoin(companies, eq(companies.id, chatIdentityLinks.companyId))
.where(
and(
eq(chatIdentityLinks.confirmationTokenHash, tokenHash),
eq(chatIdentityLinks.status, "pending"),
),
)
.then((rows) => rows[0] ?? null);
if (!row || !row.link.expiresAt || row.link.expiresAt <= new Date()) {
throw unprocessable("This identity-link request is invalid or expired");
}
if (!["active", "verifying"].includes(row.endpoint.status)) throw unprocessable("This connection is not available");
const selfService = await slackIdentityLinks.isSelfService(row.link, tokenHash);
const membership = userId ? await db.select({ status: companyMemberships.status }).from(companyMemberships).where(and(
eq(companyMemberships.companyId, row.link.companyId), eq(companyMemberships.principalType, "user"), eq(companyMemberships.principalId, userId),
)).then((rows) => rows[0]) : null;
return {
selfService,
canConfirm: membership?.status === "active",
endpointId: row.endpoint.id,
companyId: row.endpoint.companyId,
companyName: row.companyName,
companyPrefix: row.companyPrefix,
provider: row.endpoint.provider,
providerAccountLabel: row.endpoint.providerAccountLabel,
botLabel: row.endpoint.botDisplayName,
externalLabel:
row.principal.displayName ??
row.principal.handle ??
row.principal.externalId,
externalDetail: row.principal.handle
? `@${row.principal.handle.replace(/^@/, "")}`
: row.principal.externalId,
expiresAt: row.link.expiresAt.toISOString(),
};
}
async function confirmIdentityLink(token: string, paperclipUserId: string) {
const tokenHash = createHash("sha256").update(token).digest("hex");
const link = await db
.select()
.from(chatIdentityLinks)
.where(
and(
eq(chatIdentityLinks.confirmationTokenHash, tokenHash),
eq(chatIdentityLinks.status, "pending"),
),
)
.then((rows) => rows[0] ?? null);
if (!link || !link.expiresAt || link.expiresAt <= new Date())
throw unprocessable("This identity-link request is invalid or expired");
const endpointRecordForLink = await endpointRecord(link.endpointId);
if (!endpointRecordForLink || !["active", "verifying"].includes(endpointRecordForLink.endpoint.status)) throw unprocessable("This connection is not available");
const connectReceipt = endpointRecordForLink.endpoint.provider === "slack" ? await slackIdentityLinks.latestConnect(link) : null;
const confirmationRuntime = connectReceipt ? await confirmationRuntimeFor(endpointRecordForLink.endpoint) : null;
let confirmationEffectId: string | null = null;
const confirmed = await db.transaction(async (tx) => {
await tx.execute(
sql`select pg_advisory_xact_lock(hashtextextended(${`chat-identity:${link.companyId}:${link.principalId}`}, 0))`,
);
const currentLink = await tx
.select({
id: chatIdentityLinks.id,
companyId: chatIdentityLinks.companyId,
endpointId: chatIdentityLinks.endpointId,
principalId: chatIdentityLinks.principalId,
expiresAt: chatIdentityLinks.expiresAt,
})
.from(chatIdentityLinks)
.where(
and(
eq(chatIdentityLinks.id, link.id),
eq(chatIdentityLinks.status, "pending"),
eq(chatIdentityLinks.confirmationTokenHash, tokenHash),
),
)
.for("update")
.then((rows) => rows[0] ?? null);
const now = new Date();
if (
!currentLink ||
!currentLink.expiresAt ||
currentLink.expiresAt <= now
) {
return null;
}
const membership = await tx
.select({ status: companyMemberships.status })
.from(companyMemberships)
.where(
and(
eq(companyMemberships.companyId, currentLink.companyId),
eq(companyMemberships.principalType, "user"),
eq(companyMemberships.principalId, paperclipUserId),
),
)
.for("update")
.then((rows) => rows[0] ?? null);
if (membership?.status !== "active") {
throw forbidden(
"The signed-in Paperclip account is not a member of this company",
);
}
const conflictingLink = await tx
.select({
id: chatIdentityLinks.id,
paperclipUserId: chatIdentityLinks.paperclipUserId,
})
.from(chatIdentityLinks)
.where(
and(
eq(chatIdentityLinks.companyId, link.companyId),
eq(chatIdentityLinks.principalId, link.principalId),
eq(chatIdentityLinks.status, "linked"),
ne(chatIdentityLinks.paperclipUserId, paperclipUserId),
),
)
.limit(1)
.then((rows) => rows[0] ?? null);
if (conflictingLink) {
throw conflict(
"This provider identity is linked to a different Paperclip account",
{
code: "chat_identity_link_conflict",
},
);
}
const confirmedLink = await tx
.update(chatIdentityLinks)
.set({
paperclipUserId,
status: "linked",
confirmationTokenHash: null,
confirmedAt: now,
updatedAt: now,
})
.where(
and(
eq(chatIdentityLinks.id, link.id),
eq(chatIdentityLinks.status, "pending"),
eq(chatIdentityLinks.confirmationTokenHash, tokenHash),
gt(chatIdentityLinks.expiresAt, now),
),
)
.returning({ endpointId: chatIdentityLinks.endpointId })
.then((rows) => rows[0] ?? null);
if (confirmedLink && connectReceipt && confirmationRuntime &&
typeof connectReceipt.payload.channelId === "string" && typeof connectReceipt.payload.userId === "string") {
confirmationEffectId = await slackIdentityLinks.stageConfirmation(tx, {
endpoint: endpointRecordForLink.endpoint, link, tokenHash,
channelId: connectReceipt.payload.channelId, userId: connectReceipt.payload.userId,
confirmationRuntime,
});
}
return confirmedLink;
});
if (!confirmed) {
throw conflict("This identity-link request was already used or expired", {
code: "chat_identity_link_consumed",
});
}
if (confirmationEffectId) scheduleProviderEffect(confirmationEffectId);
return { ok: true, endpointId: confirmed.endpointId };
}
async function revokeLink(endpointId: string, principalId: string) {
const link = await db
.select({ companyId: chatIdentityLinks.companyId })
.from(chatIdentityLinks)
.where(
and(
eq(chatIdentityLinks.endpointId, endpointId),
eq(chatIdentityLinks.principalId, principalId),
),
)
.then((rows) => rows[0] ?? null);
if (!link) throw notFound("Identity link not found");
const revoked = await db.transaction(async (tx) => {
await tx.execute(
sql`select pg_advisory_xact_lock(hashtextextended(${`chat-identity:${link.companyId}:${principalId}`}, 0))`,
);
return tx
.update(chatIdentityLinks)
.set({
paperclipUserId: null,
status: "revoked",
confirmationTokenHash: null,
revokedAt: new Date(),
updatedAt: new Date(),
})
.where(
and(
eq(chatIdentityLinks.companyId, link.companyId),
eq(chatIdentityLinks.endpointId, endpointId),
eq(chatIdentityLinks.principalId, principalId),
),
)
.returning({ id: chatIdentityLinks.id })
.then((rows) => rows[0] ?? null);
});
if (!revoked) throw notFound("Identity link not found");
}
return { listPrincipals, createLinkIntent, previewIdentityLink, requestIdentityAccess: slackIdentityLinks.requestIdentityAccess, confirmIdentityLink, revokeLink };
}
@@ -0,0 +1,9 @@
import type { SafeChatPublicationPayload } from "@paperclipai/shared";
export function imessageProgressSupersessionReason(
progress: SafeChatPublicationPayload["progressState"],
): string | null {
return progress && ["queued", "working"].includes(progress)
? "iMessage uses typing instead of progress bubbles"
: null;
}
@@ -0,0 +1,14 @@
import type { chatEndpointResources } from "@paperclipai/db";
export function imessageResourceParticipants(resource: typeof chatEndpointResources.$inferSelect) {
return resource.providerResourceId.startsWith("imessage-photon:") &&
Array.isArray(resource.metadata.participants)
? resource.metadata.participants.flatMap((participant) => {
const address =
participant && typeof participant === "object"
? (participant as Record<string, unknown>).address
: null;
return typeof address === "string" ? [address] : [];
})
: undefined;
}
@@ -0,0 +1,39 @@
import type { ChatProvider } from "@paperclipai/shared";
import type { Author } from "chat";
import { normalizeMicrosoftTeamsExternalPrincipalId } from "../chat-teams-credentials.js";
// Provider message identity and chronology only. Callers retain authentication,
// authorization, durable admission, and delivery ownership.
export const MAX_INBOUND_TEXT = 100_000;
export function stableExternalPrincipalId(
provider: ChatProvider,
author: Author,
raw?: unknown,
): string {
if (provider !== "microsoft-teams" || !raw || typeof raw !== "object")
return author.userId;
const from = (raw as { from?: unknown }).from;
if (!from || typeof from !== "object") return author.userId;
const aadObjectId = (from as { aadObjectId?: unknown }).aadObjectId;
return normalizeMicrosoftTeamsExternalPrincipalId(aadObjectId, author.userId);
}
export type LifecycleActor = {
displayName: string;
externalId: string;
handle: string;
};
export function lifecycleActorFromAuthor(
provider: ChatProvider,
author: Author,
raw?: unknown,
): LifecycleActor {
return {
externalId: stableExternalPrincipalId(provider, author, raw),
displayName: author.fullName,
handle: author.userName,
};
}
@@ -0,0 +1,149 @@
import { and, eq, isNull, sql } from "drizzle-orm";
import { assets, chatConversations, chatEndpoints, chatPublications, chatTeamsFileTransfers, issueAttachments, issueComments, type Db } from "@paperclipai/db";
import { isUuidLike, type ChatProvider } from "@paperclipai/shared";
type DbOrTransaction = Db | Parameters<Parameters<Db["transaction"]>[0]>[0];
export type ContinuingTeamsFileTransfer = {
transferId: string;
publicationId: string;
version: number;
phase: "consent_unknown" | "file_info_unknown";
};
type TeamsFileTransferSourceInput = {
companyId: string;
agentId: string;
issueId: string;
failedRunId: string;
provider: ChatProvider;
publication?: boolean;
retryActionId?: string;
runId?: string;
continuingTransfer: ContinuingTeamsFileTransfer;
};
// A selector proves only this transfer's source. The caller must still verify
// the committed result and complete original source batch in this transaction.
export async function continuingTeamsFilePublication(
tx: DbOrTransaction,
input: TeamsFileTransferSourceInput,
failedChatRetryDenied: () => Error,
): Promise<string> {
const { provider, continuingTransfer } = input;
if (
input.publication !== true ||
input.retryActionId !== undefined ||
input.runId !== undefined ||
provider !== "microsoft-teams" ||
!continuingTransfer ||
!isUuidLike(continuingTransfer.transferId) ||
!isUuidLike(continuingTransfer.publicationId) ||
!Number.isSafeInteger(continuingTransfer.version) ||
continuingTransfer.version < 1 ||
!["consent_unknown", "file_info_unknown"].includes(
continuingTransfer.phase,
)
)
throw failedChatRetryDenied();
// The caller supplies a selector only. Independently prove the exact
// unknown publication is this native run's own selected file, not a
// sibling effect or another actor/generation. All ordinary source and
// committed-result authorization below still runs unchanged.
const [continuing] = await tx
.select({
publicationId: chatPublications.id,
attemptId: chatTeamsFileTransfers.attemptId,
attemptExpiresAt: chatTeamsFileTransfers.attemptExpiresAt,
})
.from(chatTeamsFileTransfers)
.innerJoin(
chatPublications,
and(
eq(chatPublications.id, chatTeamsFileTransfers.publicationId),
eq(chatPublications.companyId, chatTeamsFileTransfers.companyId),
eq(chatPublications.endpointId, chatTeamsFileTransfers.endpointId),
eq(
chatPublications.conversationId,
chatTeamsFileTransfers.conversationId,
),
eq(chatPublications.issueId, chatTeamsFileTransfers.issueId),
eq(chatPublications.commentId, chatTeamsFileTransfers.commentId),
eq(chatPublications.state, "delivery_unknown"),
sql`${chatPublications.payload}->'attachmentIds' = jsonb_build_array(${chatTeamsFileTransfers.attachmentId}::text)`,
),
)
.innerJoin(
chatEndpoints,
and(
eq(chatEndpoints.id, chatTeamsFileTransfers.endpointId),
eq(chatEndpoints.companyId, input.companyId),
eq(chatEndpoints.provider, "microsoft-teams"),
),
)
.innerJoin(
chatConversations,
and(
eq(chatConversations.id, chatTeamsFileTransfers.conversationId),
eq(chatConversations.companyId, input.companyId),
eq(chatConversations.endpointId, chatTeamsFileTransfers.endpointId),
eq(chatConversations.issueId, input.issueId),
eq(
chatConversations.sessionGeneration,
chatTeamsFileTransfers.conversationGeneration,
),
),
)
.innerJoin(
issueComments,
and(
eq(issueComments.id, chatTeamsFileTransfers.commentId),
eq(issueComments.companyId, input.companyId),
eq(issueComments.issueId, input.issueId),
eq(issueComments.createdByRunId, input.failedRunId),
eq(issueComments.authorAgentId, input.agentId),
isNull(issueComments.deletedAt),
),
)
.innerJoin(
issueAttachments,
and(
eq(issueAttachments.id, chatTeamsFileTransfers.attachmentId),
eq(issueAttachments.companyId, input.companyId),
eq(issueAttachments.issueId, input.issueId),
eq(issueAttachments.issueCommentId, issueComments.id),
eq(issueAttachments.originatingRunId, input.failedRunId),
),
)
.innerJoin(
assets,
and(
eq(assets.id, issueAttachments.assetId),
eq(assets.companyId, input.companyId),
eq(assets.sha256, chatTeamsFileTransfers.sha256),
eq(assets.byteSize, chatTeamsFileTransfers.byteSize),
),
)
.where(
and(
eq(chatTeamsFileTransfers.companyId, input.companyId),
eq(chatTeamsFileTransfers.issueId, input.issueId),
eq(chatTeamsFileTransfers.id, continuingTransfer.transferId),
eq(
chatTeamsFileTransfers.publicationId,
continuingTransfer.publicationId,
),
eq(chatTeamsFileTransfers.version, continuingTransfer.version),
eq(chatTeamsFileTransfers.phase, continuingTransfer.phase),
),
)
.for("share", { noWait: true });
if (
!continuing ||
(continuing.attemptId &&
(!continuing.attemptExpiresAt ||
continuing.attemptExpiresAt > new Date()))
)
throw failedChatRetryDenied();
return continuing.publicationId;
}
@@ -0,0 +1,99 @@
import { createHash } from "node:crypto";
import type { ChatEndpointRuntime } from "../../voice/voice-runtime.js";
import { MAX_INBOUND_TEXT, lifecycleActorFromAuthor, type LifecycleActor } from "../message-lifecycle.js";
type MicrosoftTeamsLifecycleEvent = {
actor?: LifecycleActor;
eventKind: "message_updated" | "message_deleted" | "message_restored";
messageId: string;
providerSentAt: string | null;
raw?: unknown;
revision: string;
text: string;
threadId: string;
};
export function microsoftTeamsLifecycleEventFromPayload(
payload: unknown,
endpointRuntime: Pick<ChatEndpointRuntime, "parseMicrosoftTeamsMessage">,
): MicrosoftTeamsLifecycleEvent | null {
if (!payload || typeof payload !== "object" || Array.isArray(payload))
return null;
const activity = payload as {
channelData?: { eventType?: unknown };
timestamp?: unknown;
type?: unknown;
};
const eventType = activity.channelData?.eventType;
const isEdit =
activity.type === "messageUpdate" && eventType === "editMessage";
const isRestore =
activity.type === "messageUpdate" && eventType === "undeleteMessage";
const isDelete =
activity.type === "messageDelete" && eventType === "softDeleteMessage";
if (!isEdit && !isRestore && !isDelete) return null;
const message = endpointRuntime.parseMicrosoftTeamsMessage(payload);
if (!message?.id || !message.threadId) return null;
const body = message.text.slice(0, MAX_INBOUND_TEXT);
const providerSentAt =
typeof activity.timestamp === "string" &&
Number.isFinite(Date.parse(activity.timestamp))
? new Date(activity.timestamp).toISOString()
: null;
const bodyHash = createHash("sha256").update(body).digest("hex");
const revision = providerSentAt ? `${providerSentAt}:${bodyHash}` : bodyHash;
return {
...(!isDelete
? {
actor: lifecycleActorFromAuthor(
"microsoft-teams",
message.author,
message.raw,
),
}
: {}),
eventKind: isDelete
? "message_deleted"
: isRestore
? "message_restored"
: "message_updated",
messageId: message.id,
providerSentAt,
raw: message.raw,
revision,
text: isDelete
? "An external message in this conversation was deleted."
: isRestore
? `An external message was restored:\n\n${body}`
: `An external message was edited:\n\n${body}`,
threadId: message.threadId,
};
}
export function teamsActivitySentAt(raw: unknown): Date | null {
if (!raw || typeof raw !== "object" || Array.isArray(raw)) return null;
const value = (raw as { timestamp?: unknown }).timestamp;
// The SDK may supply a Date, but its missing-timestamp display fallback is
// only on Message.metadata. Never use that fallback as provider chronology.
if (value instanceof Date)
return Number.isFinite(value.getTime()) ? new Date(value.getTime()) : null;
if (
typeof value !== "string" ||
value.length > 64 ||
!/^\d{4}-\d{2}-\d{2}T(?:[01]\d|2[0-3]):[0-5]\d:[0-5]\d(?:\.\d{1,7})?(?:Z|[+-](?:[01]\d|2[0-3]):[0-5]\d)$/.test(
value,
)
)
return null;
// Date.parse otherwise normalizes nonexistent dates such as February 30.
const day = value.slice(0, 10);
const calendar = new Date(`${day}T00:00:00.000Z`);
if (
!Number.isFinite(calendar.getTime()) ||
calendar.toISOString().slice(0, 10) !== day
)
return null;
const sentAt = new Date(value);
return Number.isFinite(sentAt.getTime()) ? sentAt : null;
}
@@ -0,0 +1,3 @@
export function baseTeamsConversationId(value: string): string {
return value.replace(/;messageid=[^;]+/i, "");
}
@@ -0,0 +1,965 @@
import { imessageProgressSupersessionReason } from "./imessage-photon/publications.js";
import { fastResponseRequests } from "@paperclipai/db";
import {
and,
desc,
eq,
gt,
inArray,
isNotNull,
isNull,
like,
ne,
or,
sql,
} from "drizzle-orm";
import { alias } from "drizzle-orm/pg-core";
import type { Db } from "@paperclipai/db";
import {
agentWakeupRequests,
chatEndpoints,
chatMessageLinks,
chatPublications,
heartbeatRuns,
issueComments,
issueThreadInteractions,
issues,
} from "@paperclipai/db";
import type {
SafeChatPublicationPayload,
} from "@paperclipai/shared";
import {
isUuidLike,
} from "@paperclipai/shared";
import {
NativeChatReviewPresentationContentionError,
} from "../native-runtime/native-chat-review-presentation.js";
import { isExternalChatWaitAuthorizationContention } from "../native-runtime/chat-attachment-reuse.js";
import {
inboundWakePublicationKey,
parseInboundWakePublicationKey,
} from "../chat-inbound-wakeup-publications.js";
type EndpointRow = typeof chatEndpoints.$inferSelect;
type DbOrTransaction = Db | Parameters<Parameters<Db["transaction"]>[0]>[0];
type ChatPublicationReplacementOptions = {
endpointRecord: (endpointId: string) => Promise<{ endpoint: EndpointRow } | null>;
authorizeRetainedChatSourcePublication: (database: DbOrTransaction, publication: typeof chatPublications.$inferSelect) => Promise<boolean>;
};
export function createChatPublicationReplacement(db: Db, options: ChatPublicationReplacementOptions) {
const { endpointRecord, authorizeRetainedChatSourcePublication } = options;
function runIdFromMilestonePublication(
publication: typeof chatPublications.$inferSelect,
): string | null {
if (!publication.payload.progressState) return null;
const match =
/^run:([^:]+):(?:queued|working|waiting_for_input|completed|failed):/.exec(
publication.idempotencyKey,
);
const runId = match?.[1] ?? null;
// This identifier is also compared against a UUID column below. Old or
// manually repaired rows must not be allowed to turn the global
// publication sweep into a PostgreSQL cast error.
return isUuidLike(runId) ? runId : null;
}
async function runOwnershipMilestoneSupersessionReason(
publication: typeof chatPublications.$inferSelect,
): Promise<string | null> {
const progress = publication.payload.progressState;
const typingReason = imessageProgressSupersessionReason(progress);
if (typingReason) {
const endpoint = await endpointRecord(publication.endpointId);
if (endpoint?.endpoint.provider === "imessage-photon") return typingReason;
}
if (
!progress ||
!["queued", "working", "waiting_for_input"].includes(progress)
) {
return null;
}
const runId = runIdFromMilestonePublication(publication);
if (!runId) return null;
const run = await db.transaction(async (tx) => {
// Use native finalization's issue -> run lock order. If terminalization
// won, the current status suppresses this obsolete provider update. If
// this short authorization snapshot wins, the progress was still true
// at its durable send boundary and the later final remains authoritative.
const currentIssue = await tx
.select({ id: issues.id })
.from(issues)
.where(
and(
eq(issues.id, publication.issueId),
eq(issues.companyId, publication.companyId),
),
)
.for("update")
.then((rows) => rows[0] ?? null);
if (!currentIssue) return null;
return tx
.select({
status: heartbeatRuns.status,
errorCode: heartbeatRuns.errorCode,
})
.from(heartbeatRuns)
.where(
and(
eq(heartbeatRuns.id, runId),
eq(heartbeatRuns.companyId, publication.companyId),
sql`${heartbeatRuns.contextSnapshot} ->> 'issueId' = ${publication.issueId}`,
),
)
.for("update")
.then((rows) => rows[0] ?? null);
});
const baseProgressKey = `run:${runId}:${progress}:${publication.endpointId}`;
const nativeProgressPrefix = `run:${runId}:working:${publication.endpointId}:native:`;
const isTerminalizableProgress =
(progress === "queued" || progress === "working") &&
(publication.idempotencyKey === baseProgressKey ||
publication.idempotencyKey.startsWith(nativeProgressPrefix));
if (
isTerminalizableProgress &&
run &&
["succeeded", "interrupted", "failed", "cancelled", "timed_out"].includes(
run.status,
)
) {
return "Run reached a terminal state before progress delivery";
}
// Match only the closed ownership-attention projection. Historical progress
// without an extant run retains its existing delivery semantics.
const ownershipBlocked =
run?.status === "running" &&
[
"native_execution_ownership_unverified",
"native_adopted_runner_authentication_timeout",
].includes(run.errorCode ?? "");
if (progress === "waiting_for_input") {
return ownershipBlocked
? null
: "Run ownership attention no longer applies to the bound task";
}
return ownershipBlocked
? "Run progress was superseded by ownership recovery attention"
: null;
}
async function providerProgressLaneConsumed(
publication: typeof chatPublications.$inferSelect,
providerMessageId: string,
): Promise<boolean> {
// Published progress rows retain their old provider ID after an edit.
// Consult that exact message's current outbound link, not the conversation
// tail: an authored answer or failure has consumed the lane, while an
// interleaved task-status update may still be replaced normally.
const [consumed] = await db
.select({ id: chatPublications.id })
.from(chatMessageLinks)
.innerJoin(
chatPublications,
and(
eq(chatPublications.id, chatMessageLinks.publicationId),
eq(chatPublications.companyId, publication.companyId),
eq(chatPublications.endpointId, publication.endpointId),
eq(chatPublications.conversationId, publication.conversationId),
eq(chatPublications.issueId, publication.issueId),
eq(chatPublications.providerMessageId, providerMessageId),
eq(chatPublications.state, "published"),
),
)
.where(
and(
eq(chatMessageLinks.companyId, publication.companyId),
eq(chatMessageLinks.endpointId, publication.endpointId),
eq(chatMessageLinks.conversationId, publication.conversationId),
eq(chatMessageLinks.direction, "outbound"),
eq(chatMessageLinks.providerMessageId, providerMessageId),
or(
and(
isNotNull(chatPublications.commentId),
sql`${chatPublications.payload}->>'progressState' is null`,
),
sql`${chatPublications.payload}->>'progressState' = 'failed'`,
),
),
)
.limit(1);
return Boolean(consumed);
}
async function committedResponseMayReplaceFailure(
publication: typeof chatPublications.$inferSelect,
runId: string,
providerMessageId: string,
): Promise<boolean> {
if (
!publication.commentId ||
publication.payload.progressState !== undefined
)
return false;
return db
.transaction(async (tx) => {
const [run] = await tx
.select()
.from(heartbeatRuns)
.where(
and(
eq(heartbeatRuns.id, runId),
eq(heartbeatRuns.companyId, publication.companyId),
eq(heartbeatRuns.nativeIssueId, publication.issueId),
eq(heartbeatRuns.runtimeMode, "native"),
eq(heartbeatRuns.status, "succeeded"),
isNotNull(heartbeatRuns.finishedAt),
),
)
.for("share", { noWait: true });
const presentation = run?.resultJson?.presentationDecision as
Record<string, unknown> | undefined;
if (
!run?.resultJson?.nativeCommittedChatResponse ||
presentation?.commentId !== publication.commentId
)
return false;
const [comment] = await tx
.select({ id: issueComments.id })
.from(issueComments)
.where(
and(
eq(issueComments.id, publication.commentId!),
eq(issueComments.companyId, publication.companyId),
eq(issueComments.issueId, publication.issueId),
eq(issueComments.createdByRunId, runId),
eq(issueComments.authorAgentId, run.agentId),
isNull(issueComments.deletedAt),
),
)
.for("share", { noWait: true });
if (!comment) return false;
const [priorAnswer] = await tx
.select({ id: chatPublications.id })
.from(chatPublications)
.innerJoin(
issueComments,
and(
eq(issueComments.id, chatPublications.commentId),
eq(issueComments.companyId, publication.companyId),
eq(issueComments.createdByRunId, runId),
),
)
.where(
and(
eq(chatPublications.companyId, publication.companyId),
eq(chatPublications.endpointId, publication.endpointId),
eq(chatPublications.conversationId, publication.conversationId),
eq(chatPublications.issueId, publication.issueId),
ne(chatPublications.id, publication.id),
inArray(chatPublications.state, [
"published",
"streaming",
"delivery_unknown",
]),
sql`${chatPublications.payload}->>'progressState' is null`,
),
)
.limit(1);
if (priorAnswer) return false;
// A historical progress row can retain this ID after a later edit.
// Only its CURRENT outbound link may grant this narrow exception; an
// authored answer or another run's failure must remain consumed.
const [failure] = await tx
.select({ id: chatPublications.id })
.from(chatMessageLinks)
.innerJoin(
chatPublications,
and(
eq(chatPublications.id, chatMessageLinks.publicationId),
eq(chatPublications.companyId, publication.companyId),
eq(chatPublications.endpointId, publication.endpointId),
eq(chatPublications.conversationId, publication.conversationId),
eq(chatPublications.issueId, publication.issueId),
eq(chatPublications.providerMessageId, providerMessageId),
eq(
chatPublications.idempotencyKey,
`run:${runId}:failed:${publication.endpointId}`,
),
eq(chatPublications.state, "published"),
isNull(chatPublications.commentId),
sql`${chatPublications.payload}->>'progressState' = 'failed'`,
),
)
.where(
and(
eq(chatMessageLinks.companyId, publication.companyId),
eq(chatMessageLinks.endpointId, publication.endpointId),
eq(chatMessageLinks.conversationId, publication.conversationId),
eq(chatMessageLinks.providerMessageId, providerMessageId),
eq(chatMessageLinks.direction, "outbound"),
),
);
if (!failure) return false;
// The marker is read from the authoritative run, never the publication
// payload. Recheck accepted result/decision plus the complete current
// source batch, actor, access and epoch before borrowing its failure lane.
// The transport claim repeats those checks at the I/O boundary.
return authorizeRetainedChatSourcePublication(tx, publication);
})
.catch((error: unknown) => {
if (isExternalChatWaitAuthorizationContention(error))
throw new NativeChatReviewPresentationContentionError();
throw error;
});
}
async function runPublicationToReplace(
publication: typeof chatPublications.$inferSelect,
payload: SafeChatPublicationPayload,
): Promise<string | null> {
if (payload.attachmentIds?.length) return null;
const currentRunId =
runIdFromMilestonePublication(publication) ??
(publication.commentId
? await db
.select({ runId: issueComments.createdByRunId })
.from(issueComments)
.where(eq(issueComments.id, publication.commentId))
.then((rows) => rows[0]?.runId ?? null)
: null);
if (!currentRunId) return null;
return db
.select({
id: chatPublications.id,
commentId: chatPublications.commentId,
providerMessageId: chatPublications.providerMessageId,
payload: chatPublications.payload,
})
.from(chatPublications)
.leftJoin(issueComments, eq(issueComments.id, chatPublications.commentId))
.where(
and(
eq(chatPublications.companyId, publication.companyId),
eq(chatPublications.endpointId, publication.endpointId),
eq(chatPublications.conversationId, publication.conversationId),
eq(chatPublications.state, "published"),
isNotNull(chatPublications.providerMessageId),
or(
like(chatPublications.idempotencyKey, `run:${currentRunId}:%`),
eq(issueComments.createdByRunId, currentRunId),
),
),
)
.orderBy(desc(chatPublications.createdAt), desc(chatPublications.id))
.then(async (rows) => {
// Progress updates are one replaceable provider-message lane per run.
// The first durable agent comment may turn that placeholder into the
// terminal response, but later comments from the same run are distinct
// user-visible outputs and must be posted separately. Re-editing the
// placeholder for each comment silently erases the earlier replies.
if (
publication.commentId &&
rows.some(
(row) =>
row.commentId !== null && row.payload.progressState === undefined,
)
)
return null;
const replacement = rows.find(
(row) =>
Boolean(row.providerMessageId) &&
row.payload.progressState !== undefined,
);
if (!replacement?.providerMessageId) return null;
if (
(await providerProgressLaneConsumed(
publication,
replacement.providerMessageId,
)) &&
!(await committedResponseMayReplaceFailure(
publication,
currentRunId,
replacement.providerMessageId,
))
)
return null;
// Replacement identity belongs to the run, not to the provider-visible
// tail. A status/control reply may legitimately interleave while the run
// is active; making the tail the edit candidate would strand this run's
// working placeholder forever. The query is bounded by endpoint,
// conversation (the task generation), and run id, so an interleaved
// control or another run can never donate its provider message here.
return replacement.providerMessageId;
});
}
async function receiptReactionCompletionRunId(
tx: Db,
publication: typeof chatPublications.$inferSelect,
payload: SafeChatPublicationPayload,
): Promise<string | null> {
if (
payload.progressState === "completed" ||
payload.progressState === "failed"
) {
return runIdFromMilestonePublication(publication);
}
if (publication.commentId) {
const runId = await tx
.select({ runId: issueComments.createdByRunId })
.from(issueComments)
.where(
and(
eq(issueComments.id, publication.commentId),
eq(issueComments.companyId, publication.companyId),
),
)
.then((rows) => rows[0]?.runId ?? null);
return isUuidLike(runId) ? runId : null;
}
if (
payload.interactionId &&
publication.idempotencyKey ===
`interaction:${payload.interactionId}:${publication.endpointId}`
) {
const runId = await tx
.select({ runId: issueThreadInteractions.sourceRunId })
.from(issueThreadInteractions)
.where(
and(
eq(issueThreadInteractions.id, payload.interactionId),
eq(issueThreadInteractions.companyId, publication.companyId),
eq(issueThreadInteractions.issueId, publication.issueId),
),
)
.then((rows) => rows[0]?.runId ?? null);
return isUuidLike(runId) ? runId : null;
}
return null;
}
async function inboundWakePublicationToReplace(
publication: typeof chatPublications.$inferSelect,
payload: SafeChatPublicationPayload,
): Promise<string | null> {
if (payload.attachmentIds?.length) return null;
const notice = parseInboundWakePublicationKey(publication.idempotencyKey);
let wakeId: string | null =
notice && notice.state !== "queued" ? notice.wakeId : null;
let runId = runIdFromMilestonePublication(publication);
if (
!notice &&
!runId &&
payload.interactionId &&
publication.idempotencyKey ===
`interaction:${payload.interactionId}:${publication.endpointId}`
) {
const [interaction] = await db
.select({ runId: issueThreadInteractions.sourceRunId })
.from(issueThreadInteractions)
.where(
and(
eq(issueThreadInteractions.id, payload.interactionId),
eq(issueThreadInteractions.companyId, publication.companyId),
eq(issueThreadInteractions.issueId, publication.issueId),
inArray(issueThreadInteractions.kind, [
"ask_user_questions",
"request_confirmation",
]),
),
);
runId = interaction?.runId ?? null;
}
if (!notice && !runId && publication.commentId) {
const [comment] = await db
.select({ runId: issueComments.createdByRunId })
.from(issueComments)
.where(
and(
eq(issueComments.id, publication.commentId),
eq(issueComments.companyId, publication.companyId),
eq(issueComments.issueId, publication.issueId),
),
);
runId = comment?.runId ?? null;
}
let runContext: Record<string, unknown> | null = null;
if (runId) {
if (publication.commentId) {
const [priorAnswer] = await db
.select({ id: chatPublications.id })
.from(chatPublications)
.innerJoin(
issueComments,
eq(issueComments.id, chatPublications.commentId),
)
.where(
and(
eq(chatPublications.companyId, publication.companyId),
eq(chatPublications.endpointId, publication.endpointId),
eq(chatPublications.conversationId, publication.conversationId),
eq(chatPublications.state, "published"),
eq(issueComments.createdByRunId, runId),
sql`${chatPublications.payload}->>'progressState' is null`,
),
)
.limit(1);
if (priorAnswer) return null;
}
const [run] = await db
.select({
wakeId: heartbeatRuns.wakeupRequestId,
context: heartbeatRuns.contextSnapshot,
})
.from(heartbeatRuns)
.innerJoin(
agentWakeupRequests,
and(
eq(agentWakeupRequests.id, heartbeatRuns.wakeupRequestId),
eq(agentWakeupRequests.runId, heartbeatRuns.id),
eq(agentWakeupRequests.companyId, heartbeatRuns.companyId),
eq(agentWakeupRequests.agentId, heartbeatRuns.agentId),
// Failed/cancelled is the admitted run's outcome, not loss of
// its original admission. Its terminal milestone still owns
// this exact lane even if no working update was sent first.
ne(agentWakeupRequests.status, "skipped"),
),
)
.where(
and(
eq(heartbeatRuns.id, runId),
eq(heartbeatRuns.companyId, publication.companyId),
sql`${heartbeatRuns.contextSnapshot}->>'issueId' = ${publication.issueId}`,
),
);
wakeId = run?.wakeId ?? null;
runContext = run?.context ?? null;
}
if (!wakeId) return null;
const [queued] = await db
.select({
commentId: chatPublications.commentId,
providerMessageId: chatPublications.providerMessageId,
})
.from(chatPublications)
.where(
and(
eq(chatPublications.companyId, publication.companyId),
eq(chatPublications.endpointId, publication.endpointId),
eq(chatPublications.conversationId, publication.conversationId),
eq(chatPublications.issueId, publication.issueId),
eq(chatPublications.state, "published"),
isNotNull(chatPublications.providerMessageId),
eq(
chatPublications.idempotencyKey,
inboundWakePublicationKey(
wakeId,
"queued",
publication.endpointId,
publication.conversationId,
),
),
),
);
if (!queued?.commentId || !queued.providerMessageId) return null;
// The run helper may have rejected this same provider ID because its
// queued/working placeholder is already an answer or terminal failure.
// Falling back to the older wake row must not erase that newer output.
if (
await providerProgressLaneConsumed(publication, queued.providerMessageId)
)
return null;
if (notice && notice.state !== "queued")
return queued.commentId === publication.commentId
? queued.providerMessageId
: null;
// A deferred owner can be a Board wake into which the exact external
// comment coalesced. The run must actually contain that source comment;
// wake ID alone cannot authorize a different successor's edit lane.
return runContext &&
(runContext.wakeCommentId === queued.commentId ||
runContext.commentId === queued.commentId ||
(Array.isArray(runContext.wakeCommentIds) &&
runContext.wakeCommentIds.includes(queued.commentId)))
? queued.providerMessageId
: null;
}
async function interactionResolutionPublicationToReplace(
publication: typeof chatPublications.$inferSelect,
payload: SafeChatPublicationPayload,
): Promise<string | null> {
if (
!publication.idempotencyKey.startsWith("interaction-resolution:") ||
!payload.interactionId
)
return null;
return db
.select({ providerMessageId: chatPublications.providerMessageId })
.from(chatPublications)
.where(
and(
eq(chatPublications.companyId, publication.companyId),
eq(chatPublications.endpointId, publication.endpointId),
eq(chatPublications.conversationId, publication.conversationId),
eq(chatPublications.issueId, publication.issueId),
eq(
chatPublications.idempotencyKey,
`interaction:${payload.interactionId}:${publication.endpointId}`,
),
eq(chatPublications.state, "published"),
isNotNull(chatPublications.providerMessageId),
),
)
.then((rows) => rows[0]?.providerMessageId ?? null);
}
async function interactionPromptPublicationToReplace(
publication: typeof chatPublications.$inferSelect,
payload: SafeChatPublicationPayload,
): Promise<string | null> {
if (
!payload.interactionId ||
publication.idempotencyKey !==
`interaction:${payload.interactionId}:${publication.endpointId}`
) {
return null;
}
const interaction = await db
.select({
kind: issueThreadInteractions.kind,
sourceRunId: issueThreadInteractions.sourceRunId,
})
.from(issueThreadInteractions)
.where(
and(
eq(issueThreadInteractions.id, payload.interactionId),
eq(issueThreadInteractions.companyId, publication.companyId),
eq(issueThreadInteractions.issueId, publication.issueId),
inArray(issueThreadInteractions.kind, [
"ask_user_questions",
"request_confirmation",
]),
),
)
.then((rows) => rows[0] ?? null);
if (!interaction?.sourceRunId) return null;
// A provider-visible interaction is this run's response, not an additional
// message beside its progress indicator. Move the durable interaction
// publication onto the exact source run's provider-message lane. This
// retires both the normal working placeholder and the queued placeholder
// when a very fast run asks before the working update is published.
for (const progressState of ["working", "queued"] as const) {
const providerMessageId = await db
.select({ providerMessageId: chatPublications.providerMessageId })
.from(chatPublications)
.where(
and(
eq(chatPublications.companyId, publication.companyId),
eq(chatPublications.endpointId, publication.endpointId),
eq(chatPublications.conversationId, publication.conversationId),
eq(chatPublications.issueId, publication.issueId),
eq(
chatPublications.idempotencyKey,
`run:${interaction.sourceRunId}:${progressState}:${publication.endpointId}`,
),
eq(chatPublications.state, "published"),
isNotNull(chatPublications.providerMessageId),
),
)
.then((rows) => rows[0]?.providerMessageId ?? null);
if (providerMessageId) return providerMessageId;
}
return null;
}
async function progressSupersededByFastResponse(publication: typeof chatPublications.$inferSelect): Promise<boolean> {
if (!["queued", "working"].includes(publication.payload.progressState ?? "")) return false;
const runId = runIdFromMilestonePublication(publication);
const [run] = runId ? await db.select({ context: heartbeatRuns.contextSnapshot }).from(heartbeatRuns).where(and(eq(heartbeatRuns.companyId, publication.companyId), eq(heartbeatRuns.id, runId))) : [];
const sourceCommentId = typeof run?.context?.wakeCommentId === "string" ? run.context.wakeCommentId : typeof run?.context?.commentId === "string" ? run.context.commentId : publication.commentId;
if (!sourceCommentId) return false;
const [receipt] = await db.select({ id: fastResponseRequests.id }).from(fastResponseRequests).innerJoin(chatPublications,
and(eq(chatPublications.companyId, publication.companyId), eq(chatPublications.commentId, fastResponseRequests.commentId), eq(chatPublications.state, "published")))
.where(and(eq(fastResponseRequests.companyId, publication.companyId), eq(fastResponseRequests.sourceCommentId, sourceCommentId), eq(fastResponseRequests.endpointId, publication.endpointId), eq(fastResponseRequests.conversationId, publication.conversationId))).limit(1);
return Boolean(receipt);
}
async function runProgressSupersededByPublishedInteraction(
publication: typeof chatPublications.$inferSelect,
): Promise<boolean> {
if (
!["queued", "working"].includes(publication.payload.progressState ?? "")
) {
return false;
}
const sourceRunId = runIdFromMilestonePublication(publication);
if (!sourceRunId) return false;
const interactionIds = await db
.select({ id: issueThreadInteractions.id })
.from(issueThreadInteractions)
.where(
and(
eq(issueThreadInteractions.companyId, publication.companyId),
eq(issueThreadInteractions.issueId, publication.issueId),
eq(issueThreadInteractions.sourceRunId, sourceRunId),
inArray(issueThreadInteractions.kind, [
"ask_user_questions",
"request_confirmation",
]),
),
)
.then((rows) => rows.map((row) => row.id));
if (interactionIds.length === 0) return false;
return db
.select({ id: chatPublications.id })
.from(chatPublications)
.where(
and(
eq(chatPublications.companyId, publication.companyId),
eq(chatPublications.endpointId, publication.endpointId),
eq(chatPublications.conversationId, publication.conversationId),
eq(chatPublications.issueId, publication.issueId),
eq(chatPublications.state, "published"),
inArray(
chatPublications.idempotencyKey,
interactionIds.map(
(interactionId) =>
`interaction:${interactionId}:${publication.endpointId}`,
),
),
),
)
.limit(1)
.then((rows) => rows.length > 0);
}
async function taskStatusPublicationToReplace(
publication: typeof chatPublications.$inferSelect,
payload: SafeChatPublicationPayload,
): Promise<string | null> {
if (
!publication.idempotencyKey.startsWith("control:status:") ||
payload.attachmentIds?.length
)
return null;
const rows = await db
.select({
commentId: chatPublications.commentId,
commentRunId: issueComments.createdByRunId,
idempotencyKey: chatPublications.idempotencyKey,
payload: chatPublications.payload,
providerMessageId: chatPublications.providerMessageId,
})
.from(chatPublications)
.leftJoin(issueComments, eq(issueComments.id, chatPublications.commentId))
.where(
and(
eq(chatPublications.companyId, publication.companyId),
eq(chatPublications.endpointId, publication.endpointId),
eq(chatPublications.conversationId, publication.conversationId),
eq(chatPublications.state, "published"),
isNotNull(chatPublications.providerMessageId),
),
)
.orderBy(desc(chatPublications.createdAt), desc(chatPublications.id));
const rowRunId = (row: (typeof rows)[number]) => {
const milestoneMatch =
/^run:([^:]+):(?:queued|working|waiting_for_input|completed|failed):/.exec(
row.idempotencyKey,
);
return milestoneMatch?.[1] ?? row.commentRunId ?? null;
};
for (const candidate of rows) {
if (
!candidate.providerMessageId ||
!["queued", "working"].includes(candidate.payload.progressState ?? "")
) {
continue;
}
const runId = rowRunId(candidate);
if (!runId) continue;
const laneClosed = rows.some((row) => {
if (rowRunId(row) !== runId) return false;
return (
["waiting_for_input", "completed", "failed"].includes(
row.payload.progressState ?? "",
) ||
(row.commentId !== null && row.payload.progressState === undefined)
);
});
if (!laneClosed) return candidate.providerMessageId;
}
return null;
}
async function closeProgressToReplace(
publication: typeof chatPublications.$inferSelect,
endpoint: EndpointRow,
): Promise<{ providerMessageId: string; runId: string } | null> {
if (
!["telegram", "slack", "discord", "microsoft-teams"].includes(
endpoint.provider,
) ||
!publication.idempotencyKey.startsWith("control:close:")
)
return null;
const current = alias(chatPublications, "close_current_progress");
const plainText = (table: typeof chatPublications | typeof current) =>
and(
sql`jsonb_typeof(${table.payload}->'text') = 'string'`,
sql`${table.payload}->'card' is null`,
sql`${table.payload}->'interactionId' is null`,
sql`${table.payload}->'transportPart' is null`,
sql`(${table.payload}->'attachmentIds' is null or ${table.payload}->'attachmentIds' = '[]'::jsonb)`,
);
const ownedMilestone = (table: typeof chatPublications | typeof current) =>
or(
sql`${table.idempotencyKey} = 'run:' || ${heartbeatRuns.id}::text || ':queued:' || ${endpoint.id}`,
sql`${table.idempotencyKey} = 'run:' || ${heartbeatRuns.id}::text || ':working:' || ${endpoint.id}`,
sql`${table.idempotencyKey} like 'run:' || ${heartbeatRuns.id}::text || ':working:' || ${endpoint.id} || ':native:%'`,
);
// A historical progress row alone is not edit authority: its exact current
// outbound link must still belong to that run's progress (or an interleaved
// status control). Read under the same credential lease as final delivery.
const lanes = await db
.selectDistinct({
providerMessageId: chatMessageLinks.providerMessageId,
runId: heartbeatRuns.id,
})
.from(chatPublications)
.innerJoin(
heartbeatRuns,
and(
eq(heartbeatRuns.companyId, publication.companyId),
eq(heartbeatRuns.agentId, endpoint.assignedAgentId),
sql`${heartbeatRuns.contextSnapshot}->>'issueId' = ${publication.issueId}`,
sql`split_part(${chatPublications.idempotencyKey}, ':', 2) = ${heartbeatRuns.id}::text`,
),
)
.innerJoin(
chatMessageLinks,
and(
eq(chatMessageLinks.companyId, publication.companyId),
eq(chatMessageLinks.endpointId, publication.endpointId),
eq(chatMessageLinks.conversationId, publication.conversationId),
eq(chatMessageLinks.direction, "outbound"),
eq(
chatMessageLinks.providerMessageId,
chatPublications.providerMessageId,
),
),
)
.innerJoin(
current,
and(
eq(current.id, chatMessageLinks.publicationId),
eq(current.companyId, publication.companyId),
eq(current.endpointId, publication.endpointId),
eq(current.conversationId, publication.conversationId),
eq(current.issueId, publication.issueId),
eq(current.state, "published"),
eq(current.providerMessageId, chatMessageLinks.providerMessageId),
isNull(current.commentId),
plainText(current),
or(
and(
sql`${current.payload}->>'progressState' in ('queued', 'working')`,
ownedMilestone(current),
),
like(current.idempotencyKey, "control:status:%"),
),
),
)
.where(
and(
eq(chatPublications.companyId, publication.companyId),
eq(chatPublications.endpointId, publication.endpointId),
eq(chatPublications.conversationId, publication.conversationId),
eq(chatPublications.issueId, publication.issueId),
eq(chatPublications.state, "published"),
isNull(chatPublications.commentId),
plainText(chatPublications),
sql`${chatPublications.payload}->>'progressState' in ('queued', 'working')`,
ownedMilestone(chatPublications),
),
)
.limit(2);
// One close receipt owns one edit. Multiple independent lanes require a
// separate durable multi-effect protocol, never an arbitrary latest guess.
if (lanes.length !== 1) return null;
const lane = lanes[0]!;
const [possiblyConsumed] = await db
.select({ id: chatPublications.id })
.from(chatPublications)
.leftJoin(
issueComments,
and(
eq(issueComments.id, chatPublications.commentId),
eq(issueComments.companyId, publication.companyId),
eq(issueComments.issueId, publication.issueId),
),
)
.where(
and(
eq(chatPublications.companyId, publication.companyId),
eq(chatPublications.endpointId, publication.endpointId),
eq(chatPublications.conversationId, publication.conversationId),
eq(chatPublications.issueId, publication.issueId),
or(
inArray(chatPublications.state, [
"pending",
"retry",
"streaming",
"delivery_unknown",
"published",
]),
// Stopping retries (including an operator's unknown-delivery
// cancellation) does not prove an attempted edit never arrived.
and(
inArray(chatPublications.state, ["cancelled", "failed"]),
gt(chatPublications.attempts, 0),
),
),
or(
eq(issueComments.createdByRunId, lane.runId),
// Question/confirmation prompts consume this same run's progress
// ID through interactionPromptPublicationToReplace, but have no
// commentId. A stale working link is not proof that their edit
// never reached the provider (including an unconfirmed receipt).
sql`exists (
select 1 from issue_thread_interactions consuming_interaction
where consuming_interaction.company_id = ${publication.companyId}::uuid
and consuming_interaction.issue_id = ${publication.issueId}::uuid
and consuming_interaction.source_run_id = ${lane.runId}::uuid
and consuming_interaction.kind in ('ask_user_questions', 'request_confirmation')
and ${chatPublications.payload}->>'interactionId' = consuming_interaction.id::text
and ${chatPublications.idempotencyKey} = 'interaction:' || consuming_interaction.id::text || ':' || ${publication.endpointId}
)`,
eq(
chatPublications.idempotencyKey,
`run:${lane.runId}:completed:${endpoint.id}`,
),
eq(
chatPublications.idempotencyKey,
`run:${lane.runId}:failed:${endpoint.id}`,
),
),
),
)
.limit(1);
// A final may have reached the provider before its database receipt. Even an
// unchanged working link cannot disprove that ambiguous provider effect.
return possiblyConsumed ? null : lane;
}
return { runIdFromMilestonePublication, runOwnershipMilestoneSupersessionReason, runPublicationToReplace, receiptReactionCompletionRunId, inboundWakePublicationToReplace, interactionResolutionPublicationToReplace, interactionPromptPublicationToReplace, progressSupersededByFastResponse, runProgressSupersededByPublishedInteraction, taskStatusPublicationToReplace, closeProgressToReplace };
}
@@ -0,0 +1,598 @@
import { imessageResourceParticipants } from "./imessage-photon/resources.js";
import { createGitHubResources, reconcileGitHubRepositoryCoordinates } from "./github/resources.js";
import { baseTeamsConversationId } from "./microsoft-teams/resources.js";
import { slackResourceLabelIsFallback } from "./slack/resources.js";
import { telegramResourceLabelIsFallback, prepareTelegramResourceMigration, migrateTelegramResourceConversations } from "./telegram/resources.js";
import { and, asc, eq, inArray, ne, sql } from "drizzle-orm";
import { chatConversations, chatDeliveries, chatEndpointResources, chatEndpoints, toolConnections, type Db } from "@paperclipai/db";
import type { ChatEndpointSetupState } from "@paperclipai/shared";
import { badRequest, conflict, notFound, unprocessable } from "../../errors.js";
import { redactSensitiveText } from "../../redaction.js";
import { notifyChatDeliveryWork, notifyChatEndpointWork } from "../chat-work-notifications.js";
import { logActivity, publishActivity, type ActivityPublication } from "../activity-log.js";
import type { CredentialMutationLeaseGuard, chatCredentialMutationLease } from "../chat-credential-mutation-lease.js";
import type { ChatProviderInventoryResult, ChatProviderResourceInventoryItem } from "../chat-provider-inventory.js";
import type { ChatProviderLifecycleEffect } from "../chat-provider-lifecycle.js";
type EndpointRow = typeof chatEndpoints.$inferSelect;
type DbTransaction = Parameters<Parameters<Db["transaction"]>[0]>[0];
type InternalSetupState = ChatEndpointSetupState & { runtimeGeneration?: number };
type ResourceReconciliationOptions = {
endpointRecord: (endpointId: string) => Promise<{ endpoint: EndpointRow } | null>;
withCredentialMutationLease: ReturnType<typeof chatCredentialMutationLease>;
runtimeGeneration: (setup: ChatEndpointSetupState) => number;
maxErrorText: number;
};
export function createChatResourceReconciliation(db: Db, options: ResourceReconciliationOptions) {
const { endpointRecord, withCredentialMutationLease, runtimeGeneration, maxErrorText: MAX_ERROR_TEXT } = options;
async function upsertProviderResourceRow(
tx: DbTransaction,
endpoint: EndpointRow,
item: ChatProviderResourceInventoryItem,
) {
if (endpoint.provider === "github") {
const renamed = await reconcileGitHubRepositoryCoordinates(tx, endpoint, item);
if (renamed) return renamed;
}
const [resource] = await tx
.insert(chatEndpointResources)
.values({
companyId: endpoint.companyId,
endpointId: endpoint.id,
type: item.type,
providerResourceId: item.providerResourceId,
parentProviderResourceId: item.parentProviderResourceId ?? null,
label: item.label,
providerUrl: item.providerUrl ?? null,
availability: "available",
// Slack inventory contains only channels the bot has joined. Apply the
// invitation default on insert; conflict updates preserve operator choices.
enabled:
endpoint.provider === "slack" &&
item.type === "channel" &&
item.metadata?.creator !== endpoint.botExternalId,
metadata: item.metadata ?? {},
})
.onConflictDoUpdate({
target: [
chatEndpointResources.endpointId,
chatEndpointResources.type,
chatEndpointResources.providerResourceId,
],
set: {
parentProviderResourceId: item.parentProviderResourceId ?? null,
label: item.label,
providerUrl: item.providerUrl ?? null,
availability: "available",
metadata: item.metadata ?? {},
updatedAt: new Date(),
},
})
.returning({ id: chatEndpointResources.id });
return resource ?? null;
}
async function reconcileProviderResourceRows(
endpoint: EndpointRow,
inventory: ChatProviderInventoryResult,
credentialLease: CredentialMutationLeaseGuard,
) {
const resourceType =
endpoint.provider === "github" ? "repository" : "channel";
const discovered = new Set(
inventory.resources.map((resource) => resource.providerResourceId),
);
const absentAvailability =
endpoint.provider === "github" ? "removed" : "unavailable";
await db.transaction(async (tx) => {
await credentialLease.assertOwned(tx);
for (const item of inventory.resources) {
const resource = await upsertProviderResourceRow(tx, endpoint, item);
if (resource) {
// Resource recovery reopens only bindings that Paperclip marked
// unavailable. Completed historical tasks stay completed.
await tx
.update(chatConversations)
.set({ state: "active", updatedAt: new Date() })
.where(
and(
eq(chatConversations.companyId, endpoint.companyId),
eq(chatConversations.endpointId, endpoint.id),
eq(chatConversations.resourceId, resource.id),
eq(chatConversations.state, "unavailable"),
),
);
}
}
const existing = await tx
.select({
id: chatEndpointResources.id,
providerResourceId: chatEndpointResources.providerResourceId,
})
.from(chatEndpointResources)
.where(
and(
eq(chatEndpointResources.companyId, endpoint.companyId),
eq(chatEndpointResources.endpointId, endpoint.id),
eq(chatEndpointResources.type, resourceType),
),
);
for (const resource of existing) {
if (discovered.has(resource.providerResourceId)) continue;
await tx
.update(chatEndpointResources)
.set({ availability: absentAvailability, updatedAt: new Date() })
.where(eq(chatEndpointResources.id, resource.id));
await tx
.update(chatConversations)
.set({ state: "unavailable", updatedAt: new Date() })
.where(
and(
eq(chatConversations.companyId, endpoint.companyId),
eq(chatConversations.endpointId, endpoint.id),
eq(chatConversations.resourceId, resource.id),
inArray(chatConversations.state, ["active", "waiting"]),
),
);
}
await credentialLease.assertOwned(tx);
});
}
async function listResources(endpointId: string) {
const resources = await db
.select()
.from(chatEndpointResources)
.where(
and(
eq(chatEndpointResources.endpointId, endpointId),
ne(chatEndpointResources.type, "direct_message"),
),
)
.orderBy(asc(chatEndpointResources.label));
return resources.map((resource) => ({
...resource,
participants: imessageResourceParticipants(resource),
}));
}
async function replaceResources(
endpointId: string,
updates: Array<{ id: string; enabled: boolean }>,
actorUserId?: string | null,
options?: { initialGitHubImport?: boolean },
) {
await updateResourceSelection(endpointId, updates, actorUserId, options);
return listResources(endpointId);
}
async function updateResourceSelection(
endpointId: string,
updates: Array<{ id: string; enabled: boolean }>,
actorUserId?: string | null,
options?: { initialGitHubImport?: boolean; allGitHubRepositoriesEnabled?: boolean },
) {
const initial = await endpointRecord(endpointId);
if (!initial) throw notFound("Chat endpoint not found");
const bulk = options?.allGitHubRepositoriesEnabled !== undefined;
if (bulk && initial.endpoint.provider !== "github") throw badRequest("Only GitHub repositories support toggle all");
if (updates.length === 0 && initial.endpoint.provider !== "github") return;
if (initial.endpoint.provider === "imessage-photon" && initial.endpoint.botExternalId?.startsWith("photon-project:") && updates.some((entry) => entry.enabled))
throw unprocessable("Photon shared channels support direct messages only; groups cannot be enabled");
await withCredentialMutationLease(
initial.endpoint,
async (credentialLease) => {
const ids = updates.map((entry) => entry.id);
const publications: ActivityPublication[] = [];
await db.transaction(async (tx) => {
await credentialLease.assertOwned(tx);
const [endpoint] = await tx
.select({
companyId: chatEndpoints.companyId,
connectionId: chatEndpoints.connectionId,
provider: chatEndpoints.provider,
setup: chatEndpoints.setup,
})
.from(chatEndpoints)
.where(
and(
eq(chatEndpoints.id, endpointId),
eq(chatEndpoints.companyId, initial.endpoint.companyId),
),
)
.for("no key update");
if (!endpoint) throw notFound("Chat endpoint not found");
if (options?.initialGitHubImport && endpoint.provider === "github" && (endpoint.setup.github?.repositorySelectionSaved || !endpoint.setup.github?.initialRepositoryImportPending)) return;
const rows = await tx
.select({
id: chatEndpointResources.id,
availability: chatEndpointResources.availability,
enabled: chatEndpointResources.enabled,
})
.from(chatEndpointResources)
.where(
and(
eq(chatEndpointResources.companyId, endpoint.companyId),
eq(chatEndpointResources.endpointId, endpointId),
bulk
? and(
eq(chatEndpointResources.type, "repository"),
options!.allGitHubRepositoriesEnabled
? eq(chatEndpointResources.availability, "available")
: undefined,
)
: inArray(chatEndpointResources.id, ids),
),
)
.orderBy(asc(chatEndpointResources.id))
.for("no key update");
if (!bulk && rows.length !== new Set(ids).size)
throw unprocessable("Every resource must belong to this endpoint");
const selectedUpdates = bulk
? rows.map((row) => ({ id: row.id, enabled: options!.allGitHubRepositoriesEnabled! }))
: updates;
const availabilityById = new Map(
rows.map((row) => [row.id, row.availability]),
);
// Validate every submitted grant, including intermediate duplicate
// entries. Netting below describes the audit, not new authority.
const unavailable = selectedUpdates.find(
(entry) =>
entry.enabled && availabilityById.get(entry.id) !== "available",
);
if (unavailable)
throw conflict(
"A destination must still be available from the provider before it can be enabled",
{ code: "chat_resource_unavailable", resourceId: unavailable.id },
);
const finalEnabled = new Map(
selectedUpdates.map((entry) => [entry.id, entry.enabled]),
);
const changes = rows
.filter((row) => row.enabled !== finalEnabled.get(row.id))
.map((row) => ({
resourceId: row.id,
before: { enabled: row.enabled },
after: { enabled: finalEnabled.get(row.id)! },
}));
if (bulk) {
// One atomic database update, independent of pagination/search and
// serialized with refresh, reconnect, and individual row changes.
await tx.update(chatEndpointResources)
.set({ enabled: options!.allGitHubRepositoriesEnabled!, updatedAt: new Date() })
.where(and(
eq(chatEndpointResources.companyId, endpoint.companyId),
eq(chatEndpointResources.endpointId, endpointId),
eq(chatEndpointResources.type, "repository"),
options!.allGitHubRepositoriesEnabled
? eq(chatEndpointResources.availability, "available")
: undefined,
));
} else for (const entry of selectedUpdates)
await tx
.update(chatEndpointResources)
.set({ enabled: entry.enabled, updatedAt: new Date() })
.where(
and(
eq(chatEndpointResources.companyId, endpoint.companyId),
eq(chatEndpointResources.endpointId, endpointId),
eq(chatEndpointResources.id, entry.id),
),
);
if (endpoint.provider === "github")
await tx.update(chatEndpoints).set({ setup: sql`jsonb_set(${chatEndpoints.setup}, '{github}', coalesce(${chatEndpoints.setup}->'github', '{}'::jsonb) || '{"repositorySelectionSaved":true,"initialRepositoryImportPending":false}'::jsonb)`, updatedAt: new Date() }).where(eq(chatEndpoints.id, endpointId));
if (changes.length > 0 || (endpoint.provider === "github" && !endpoint.setup.github?.repositorySelectionSaved))
await logActivity(
tx as unknown as Db,
{
companyId: endpoint.companyId,
actorType: "user",
actorId: actorUserId ?? "board",
action: "chat_endpoint.resources_updated",
entityType: "tool_connection",
entityId: endpoint.connectionId,
details: { endpointId, provider: endpoint.provider, changes },
},
publications,
);
await credentialLease.assertOwned(tx);
});
// A committed reach change is factual even if the outer lease's last
// check subsequently fails. Never publish an uncommitted activity.
for (const publication of publications) publishActivity(publication);
},
);
}
async function hasNewerProcessedProviderLifecycleEffect(
endpoint: EndpointRow,
effect: Extract<ChatProviderLifecycleEffect, { kind: "resource" }>,
): Promise<boolean> {
const sequence = effect.providerOrder?.sequence;
const occurredAt = effect.providerOrder?.occurredAt;
if (!sequence && !occurredAt) return false;
const providerResourceId =
endpoint.provider === "microsoft-teams"
? baseTeamsConversationId(effect.providerResourceId)
: effect.providerResourceId;
const newerOrder = sequence
? sql`(
${chatDeliveries.normalizedEvent}#>>'{lifecycle,providerOrder,sequence}' ~ '^[0-9]+([.][0-9]+)?$'
and (${chatDeliveries.normalizedEvent}#>>'{lifecycle,providerOrder,sequence}')::numeric > cast(${sequence} as numeric)
)`
: sql`(
nullif(${chatDeliveries.normalizedEvent}#>>'{lifecycle,providerOrder,occurredAt}', '')::timestamptz > cast(${occurredAt!} as timestamptz)
)`;
const newer = await db
.select({ id: chatDeliveries.id })
.from(chatDeliveries)
.where(
and(
eq(chatDeliveries.endpointId, endpoint.id),
eq(chatDeliveries.state, "processed"),
inArray(chatDeliveries.eventKind, ["installation", "uninstallation"]),
sql`${chatDeliveries.normalizedEvent}#>>'{lifecycle,kind}' = 'resource'`,
sql`${chatDeliveries.normalizedEvent}#>>'{lifecycle,provider}' = ${effect.provider}`,
sql`${chatDeliveries.normalizedEvent}#>>'{lifecycle,providerResourceId}' = ${providerResourceId}`,
newerOrder,
),
)
.limit(1);
return newer.length > 0;
}
async function applyProviderAvailability(input: {
currentEndpoint: EndpointRow;
effect: ChatProviderLifecycleEffect;
credentialLease: CredentialMutationLeaseGuard;
staleResourceEffect: boolean;
claimed: Pick<typeof chatDeliveries.$inferSelect, "id">;
candidate: Pick<typeof chatDeliveries.$inferSelect, "attempts">;
}): Promise<void> {
const { currentEndpoint, effect, credentialLease, staleResourceEffect, claimed, candidate } = input;
await db.transaction(async (tx) => {
await credentialLease.assertOwned(tx);
const now = new Date();
if (staleResourceEffect) {
// Persisted provider-native ordering wins over request arrival or
// lease acquisition order. The older callback remains auditable,
// but cannot roll a resource or conversation back.
await tx
.update(chatEndpoints)
.set({ lastEventAt: now, updatedAt: now })
.where(eq(chatEndpoints.id, currentEndpoint.id));
await notifyChatDeliveryWork(tx);
await tx
.update(chatDeliveries)
.set({
state: "processed",
processedAt: now,
redactedError: null,
nextAttemptAt: null,
updatedAt: now,
})
.where(eq(chatDeliveries.id, claimed.id));
return;
}
if (effect.kind === "resource") {
const providerResourceId =
currentEndpoint.provider === "microsoft-teams"
? baseTeamsConversationId(effect.providerResourceId)
: effect.providerResourceId;
const previousProviderResourceId =
currentEndpoint.provider === "telegram"
? effect.previousProviderResourceId
: undefined;
const { previousResource, migratedLabel, migratedEnabled } = await prepareTelegramResourceMigration(
tx, currentEndpoint, effect, providerResourceId, previousProviderResourceId, now,
);
const preserveProviderLabel =
(currentEndpoint.provider === "slack" &&
slackResourceLabelIsFallback(
providerResourceId,
migratedLabel,
)) ||
(currentEndpoint.provider === "telegram" &&
telegramResourceLabelIsFallback(
providerResourceId,
migratedLabel,
));
const [resource] = await tx
.insert(chatEndpointResources)
.values({
companyId: currentEndpoint.companyId,
endpointId: currentEndpoint.id,
type: effect.resourceType,
providerResourceId,
parentProviderResourceId:
effect.parentProviderResourceId ?? null,
label: migratedLabel,
providerUrl: effect.providerUrl ?? null,
availability: effect.availability,
enabled:
migratedEnabled ||
(currentEndpoint.provider === "slack" &&
effect.resourceType === "channel" &&
effect.availability === "available"),
metadata: effect.metadata ?? {},
})
.onConflictDoUpdate({
target: [
chatEndpointResources.endpointId,
chatEndpointResources.type,
chatEndpointResources.providerResourceId,
],
set: {
parentProviderResourceId:
effect.parentProviderResourceId ?? null,
...(preserveProviderLabel ? {} : { label: migratedLabel }),
providerUrl: effect.providerUrl ?? null,
availability: effect.availability,
...(previousProviderResourceId
? {
enabled: sql<boolean>`${chatEndpointResources.enabled} or ${migratedEnabled}`,
}
: {}),
metadata: effect.metadata ?? {},
updatedAt: now,
},
})
.returning({ id: chatEndpointResources.id });
if (
resource &&
previousResource &&
previousProviderResourceId !== providerResourceId
) {
await migrateTelegramResourceConversations(tx, currentEndpoint, previousResource, resource, previousProviderResourceId, providerResourceId, migratedLabel, now);
}
if (resource) {
await tx
.update(chatConversations)
.set({
state:
effect.availability === "available"
? "active"
: "unavailable",
updatedAt: now,
})
.where(
and(
eq(chatConversations.companyId, currentEndpoint.companyId),
eq(chatConversations.endpointId, currentEndpoint.id),
eq(chatConversations.resourceId, resource.id),
effect.availability === "available"
? eq(chatConversations.state, "unavailable")
: inArray(chatConversations.state, ["active", "waiting"]),
),
);
}
await tx
.update(chatEndpoints)
.set({ lastEventAt: now, updatedAt: now })
.where(eq(chatEndpoints.id, currentEndpoint.id));
} else {
const reason = redactSensitiveText(effect.reason).slice(
0,
MAX_ERROR_TEXT,
);
if (effect.availability === "available") {
if (
currentEndpoint.status !== "paused" &&
currentEndpoint.status !== "archived"
) {
const status =
currentEndpoint.setup.step === "complete"
? "active"
: currentEndpoint.status === "attention"
? "verifying"
: currentEndpoint.status;
if (status !== currentEndpoint.status) await notifyChatEndpointWork(tx);
await tx
.update(chatEndpoints)
.set({
status,
healthMessage:
status === "active"
? "Connected"
: "Waiting for a test conversation",
lastError: null,
lastEventAt: now,
updatedAt: now,
})
.where(eq(chatEndpoints.id, currentEndpoint.id));
await tx
.update(toolConnections)
.set({
status: "active",
enabled: true,
healthStatus: "healthy",
healthMessage: "Connected",
lastError: null,
healthCheckedAt: now,
updatedAt: now,
})
.where(eq(toolConnections.id, currentEndpoint.connectionId));
}
} else {
await tx
.update(chatEndpoints)
.set({
status: effect.availability,
healthMessage: reason,
lastError: reason,
lastEventAt: now,
setup: {
...currentEndpoint.setup,
runtimeGeneration:
runtimeGeneration(currentEndpoint.setup) + 1,
} as InternalSetupState,
updatedAt: now,
})
.where(eq(chatEndpoints.id, currentEndpoint.id));
await tx
.update(toolConnections)
.set({
status: "disabled",
enabled: false,
healthStatus:
effect.availability === "revoked" ? "failed" : "degraded",
healthMessage: reason,
lastError: reason,
healthCheckedAt: now,
updatedAt: now,
})
.where(eq(toolConnections.id, currentEndpoint.connectionId));
await tx
.update(chatEndpointResources)
.set({ availability: "unavailable", updatedAt: now })
.where(
and(
eq(
chatEndpointResources.companyId,
currentEndpoint.companyId,
),
eq(chatEndpointResources.endpointId, currentEndpoint.id),
),
);
await tx
.update(chatConversations)
.set({ state: "unavailable", updatedAt: now })
.where(
and(
eq(chatConversations.companyId, currentEndpoint.companyId),
eq(chatConversations.endpointId, currentEndpoint.id),
inArray(chatConversations.state, ["active", "waiting"]),
),
);
}
}
await notifyChatDeliveryWork(tx);
await tx
.update(chatDeliveries)
.set({
state: "processed",
processedAt: now,
redactedError: null,
nextAttemptAt: null,
updatedAt: now,
})
.where(
and(
eq(chatDeliveries.id, claimed.id),
eq(chatDeliveries.state, "processing"),
eq(chatDeliveries.attempts, candidate.attempts + 1),
),
);
await credentialLease.assertOwned(tx);
});
}
const { reconcileGitHubWebhookRepository, listGitHubRepositories, toggleAllGitHubRepositories } = createGitHubResources(db, { endpointRecord, upsertProviderResourceRow, updateResourceSelection });
return { reconcileGitHubWebhookRepository, reconcileProviderResourceRows, listResources, listGitHubRepositories, replaceResources, toggleAllGitHubRepositories, hasNewerProcessedProviderLifecycleEffect, applyProviderAvailability };
}
@@ -0,0 +1,38 @@
import type { Db, chatEndpoints } from "@paperclipai/db";
import type { ToolCredentialSecretRef } from "@paperclipai/shared";
import type { ChatEndpointRuntime } from "../voice/voice-runtime.js";
export type EndpointRow = typeof chatEndpoints.$inferSelect;
export type DbTransaction = Parameters<Parameters<Db["transaction"]>[0]>[0];
export type DbOrTransaction = Db | DbTransaction;
export type RuntimeContext = {
credentialFingerprint: string;
discordGatewayEventQueue?: Promise<void>;
discordGatewaySequence?: number;
discordGatewayOwned?: boolean;
/** Cache identity only; durable command admission rechecks ownership. */
discordCommandId?: string;
endpointRuntime?: ChatEndpointRuntime;
generation: number;
localEpoch: number;
version: string;
};
export type LifecycleRuntimeFence = Pick<
RuntimeContext,
"credentialFingerprint" | "generation"
>;
export type InboundRuntimeContext = LifecycleRuntimeFence &
Pick<RuntimeContext, "discordGatewayOwned" | "endpointRuntime">;
export type EndpointRecord = {
endpoint: EndpointRow;
assignedAgentName: string;
connectionName: string;
applicationId: string;
credentialSecretRefs: ToolCredentialSecretRef[];
};
@@ -0,0 +1,113 @@
import { createHash, randomBytes } from "node:crypto";
import { and, desc, eq, gt, or, sql } from "drizzle-orm";
import type { Db } from "@paperclipai/db";
import { authUsers, chatActions, chatEndpoints, chatIdentityLinks, invites, joinRequests } from "@paperclipai/db";
import { conflict, forbidden, unprocessable } from "../../../errors.js";
import { logActivity } from "../../activity-log.js";
type EndpointRow = typeof chatEndpoints.$inferSelect;
type DbOrTransaction = Db | Parameters<Parameters<Db["transaction"]>[0]>[0];
type ConfirmationRuntime = { credentialFingerprint: string; generation: number };
export type SlackIdentityEffectStager = (database: DbOrTransaction, input: {
endpoint: EndpointRow; principalId: string; providerActionId: string;
payload: { version: 1; effect: "ephemeral_message"; authorizationMode: "safe_notice"; threadId: string; userId: string; text: string; settleDelivery: false };
runtimeContext: ConfirmationRuntime;
}) => Promise<{ id: string } | null>;
type SlackIdentityLinkOptions = {
previewIdentityLink: (token: string, userId: string) => Promise<{ selfService: boolean; canConfirm: boolean; companyId: string; endpointId: string }>;
stageProviderEffect: SlackIdentityEffectStager;
};
export function createSlackIdentityLinks(db: Db, { previewIdentityLink, stageProviderEffect }: SlackIdentityLinkOptions) {
function listConnects(endpoint: EndpointRow) {
const endpointId = endpoint.id;
return db
.select({ principalId: chatActions.principalId, lastConnectAt: sql<string>`max(${chatActions.createdAt})::text` })
.from(chatActions)
.where(and(
eq(chatActions.companyId, endpoint.companyId),
eq(chatActions.endpointId, endpointId),
eq(chatActions.kind, "slack_connect"),
eq(chatActions.status, "processed"),
))
.groupBy(chatActions.principalId);
}
async function isSelfService(link: typeof chatIdentityLinks.$inferSelect, tokenHash: string) {
const selfService = Boolean(await db.select({ id: chatActions.id }).from(chatActions).where(and(
eq(chatActions.companyId, link.companyId), eq(chatActions.endpointId, link.endpointId),
eq(chatActions.principalId, link.principalId), eq(chatActions.kind, "slack_connect"),
sql`${chatActions.payload}->>'identityLinkHash' = ${tokenHash}`,
)).limit(1).then((rows) => rows[0]));
return selfService;
}
function latestConnect(link: typeof chatIdentityLinks.$inferSelect) {
return db.select({ payload: chatActions.payload }).from(chatActions).where(and(
eq(chatActions.endpointId, link.endpointId), eq(chatActions.principalId, link.principalId), eq(chatActions.kind, "slack_connect"),
)).orderBy(desc(chatActions.createdAt)).limit(1).then((rows) => rows[0]);
}
async function stageConfirmation(tx: DbOrTransaction, input: {
endpoint: EndpointRow; link: typeof chatIdentityLinks.$inferSelect; tokenHash: string;
channelId: string; userId: string; confirmationRuntime: ConfirmationRuntime;
}) {
const { endpoint, link, tokenHash, channelId, userId, confirmationRuntime } = input;
const effect = await stageProviderEffect(tx, {
endpoint: endpoint, principalId: link.principalId,
providerActionId: `provider_effect:identity_linked:${link.id}:${tokenHash}`,
payload: {
version: 1, effect: "ephemeral_message", authorizationMode: "safe_notice",
threadId: channelId, userId: userId,
text: "Your Slack account is connected to Paperclip. Future messages use your Paperclip permissions.", settleDelivery: false
},
runtimeContext: confirmationRuntime,
});
if (!effect) throw conflict("The Slack connection changed; try confirming again");
return effect.id;
}
async function requestIdentityAccess(token: string, userId: string, requestIp: string) {
const preview = await previewIdentityLink(token, userId);
if (!preview.selfService) throw forbidden("This identity link cannot request access");
if (preview.canConfirm) return { status: "member" as const };
return db.transaction(async (tx) => {
await tx.execute(sql`select pg_advisory_xact_lock(hashtextextended(${`chat-join:${preview.companyId}:${userId}`}, 0))`);
const tokenHash = createHash("sha256").update(token).digest("hex");
const validLink = await tx.select({ id: chatIdentityLinks.id }).from(chatIdentityLinks).where(and(
eq(chatIdentityLinks.companyId, preview.companyId), eq(chatIdentityLinks.endpointId, preview.endpointId),
eq(chatIdentityLinks.confirmationTokenHash, tokenHash), eq(chatIdentityLinks.status, "pending"), gt(chatIdentityLinks.expiresAt, new Date()),
)).for("update").then((rows) => rows[0]);
if (!validLink) throw unprocessable("This identity-link request is invalid or expired");
const user = await tx.select({ email: authUsers.email }).from(authUsers).where(eq(authUsers.id, userId)).then((rows) => rows[0]);
if (!user) throw forbidden("Sign in to request company access");
const existing = await tx.select({ id: joinRequests.id }).from(joinRequests).where(and(
eq(joinRequests.companyId, preview.companyId), eq(joinRequests.requestType, "human"), eq(joinRequests.status, "pending_approval"),
or(eq(joinRequests.requestingUserId, userId), sql`lower(${joinRequests.requestEmailSnapshot}) = ${user.email.toLowerCase()}`),
)).then((rows) => rows[0]);
if (existing) return { status: "pending_approval" as const };
const now = new Date();
const [invite] = await tx.insert(invites).values({
companyId: preview.companyId, tokenHash: createHash("sha256").update(randomBytes(32)).digest("hex"),
inviteType: "company_join", allowedJoinTypes: "human", acceptedAt: now, expiresAt: now,
defaultsPayload: { human: { role: "operator" }, source: "slack_identity_link" },
}).returning({ id: invites.id });
const [request] = await tx.insert(joinRequests).values({
inviteId: invite.id, companyId: preview.companyId, requestType: "human", status: "pending_approval",
requestIp, requestingUserId: userId, requestEmailSnapshot: user.email,
}).returning({ id: joinRequests.id });
await logActivity(tx as unknown as Db, {
companyId: preview.companyId, actorType: "user", actorId: userId, action: "join.requested",
entityType: "join_request", entityId: request.id, details: { requestType: "human", source: "slack_identity_link", endpointId: preview.endpointId },
});
return { status: "pending_approval" as const };
});
}
return { listConnects, isSelfService, latestConnect, stageConfirmation, requestIdentityAccess };
}
@@ -0,0 +1,43 @@
import { createHash } from "node:crypto";
export function slackLifecycleFilesDigest(raw: unknown): string {
const files =
raw && typeof raw === "object" && !Array.isArray(raw)
? (raw as Record<string, unknown>).files
: undefined;
// Match the pinned Slack adapter's content-change projection. Transport
// locators/unfurls are not content identity and must never enter durable keys.
const projection = Array.isArray(files)
? files.map((file) => {
const row =
file && typeof file === "object" && !Array.isArray(file)
? (file as Record<string, unknown>)
: {};
return [
...["id", "name", "mimetype"].map((key) =>
typeof row[key] === "string" ? row[key] : null,
),
...["size", "original_w", "original_h"].map((key) =>
typeof row[key] === "number" && Number.isFinite(row[key])
? row[key]
: null,
),
];
})
: [];
return createHash("sha256").update(JSON.stringify(projection)).digest("hex");
}
export function slackMessageSentAt(raw: unknown, messageId: string): Date | null {
if (!raw || typeof raw !== "object" || Array.isArray(raw)) return null;
const ts = (raw as { ts?: unknown }).ts;
if (
ts !== messageId ||
typeof ts !== "string" ||
!/^[1-9][0-9]{0,12}\.[0-9]{6}$/.test(ts)
)
return null;
const date = new Date(Number(ts) * 1_000);
return Number.isFinite(date.getTime()) ? date : null;
}
@@ -0,0 +1,10 @@
export function slackResourceLabelIsFallback(
providerResourceId: string,
label: string,
): boolean {
const candidate = label.trim();
return (
candidate === providerResourceId ||
candidate === `slack:${providerResourceId}`
);
}
@@ -0,0 +1,858 @@
import type { EndpointRow, DbTransaction, DbOrTransaction, RuntimeContext, LifecycleRuntimeFence } from "../runtime-types.js";
import { dueChatAction, PROVIDER_EFFECT_STALE_MS } from "../../chat-action-work.js";
import { notifyChatActionWork } from "../../chat-work-notifications.js";
import {
and,
asc,
desc,
eq,
inArray,
isNull,
lte,
or,
sql,
} from "drizzle-orm";
import type { Db } from "@paperclipai/db";
import {
agentWakeupRequests,
agents,
chatActions,
chatConversations,
chatEndpointResources,
chatEndpoints,
chatExternalPrincipals,
heartbeatRuns,
issues,
} from "@paperclipai/db";
import type {
SafeChatPublicationPayload,
} from "@paperclipai/shared";
import {
isUuidLike,
} from "@paperclipai/shared";
import { logger } from "../../../middleware/logger.js";
import {
logActivity,
} from "../../activity-log.js";
import { projectSafeChatPublication } from "../../chat-publication-projection.js";
import {
type SlackSessionStop,
} from "../../chat-slack-sessions.js";
import {
enqueueChatRunMilestones,
safeMilestoneText,
} from "../../chat-run-publications.js";
export type SlackSessionStopTarget =
{ id: string; kind: "run" } | { id: string; kind: "wakeup" };
export type SlackSessionStopPayload = {
version: 1;
assignedAgentId: string;
eventTimestamp: string;
issueId: string | null;
occurredAt: string;
sessionGeneration: number | null;
target: SlackSessionStopTarget | null;
threadId: string;
userId: string;
};
export function slackSessionStopPayload(
value: Record<string, unknown>,
): SlackSessionStopPayload | null {
const target = value.target;
const parsedTarget =
target === null
? null
: target &&
typeof target === "object" &&
!Array.isArray(target) &&
((target as { kind?: unknown }).kind === "run" ||
(target as { kind?: unknown }).kind === "wakeup") &&
typeof (target as { id?: unknown }).id === "string" &&
isUuidLike((target as { id: string }).id)
? (target as SlackSessionStopTarget)
: undefined;
if (
value.version !== 1 ||
typeof value.assignedAgentId !== "string" ||
!isUuidLike(value.assignedAgentId) ||
typeof value.eventTimestamp !== "string" ||
!value.eventTimestamp ||
(value.issueId !== null &&
(typeof value.issueId !== "string" || !isUuidLike(value.issueId))) ||
typeof value.occurredAt !== "string" ||
!Number.isFinite(new Date(value.occurredAt).getTime()) ||
(value.sessionGeneration !== null &&
(typeof value.sessionGeneration !== "number" ||
!Number.isSafeInteger(value.sessionGeneration) ||
value.sessionGeneration < 1)) ||
(parsedTarget !== null &&
(value.issueId === null || value.sessionGeneration === null)) ||
parsedTarget === undefined ||
typeof value.threadId !== "string" ||
!value.threadId.startsWith("slack:") ||
typeof value.userId !== "string" ||
!value.userId
) {
return null;
}
return {
version: 1,
assignedAgentId: value.assignedAgentId,
eventTimestamp: value.eventTimestamp,
issueId: value.issueId,
occurredAt: value.occurredAt,
sessionGeneration: value.sessionGeneration,
target: parsedTarget,
threadId: value.threadId,
userId: value.userId,
};
}
type SlackSessionStopOptions = {
runtimeCallbackEndpoint: (tx: DbTransaction, endpointId: string, context: LifecycleRuntimeFence, statuses: EndpointRow["status"][]) => Promise<EndpointRow | null>;
lockCurrentPrincipalAuthorization: (tx: DbOrTransaction, endpoint: EndpointRow, principalId: string) => Promise<{ allowed: boolean; userId: string | null }>;
nonDirectDestinationAllowed: (endpoint: EndpointRow, resource: typeof chatEndpointResources.$inferSelect | null | undefined) => boolean;
stageAuthorizedTaskControlPublication: (tx: DbOrTransaction, input: { companyId: string; conversationId: string; endpointId: string; idempotencyKey: string; issueId: string; payload: SafeChatPublicationPayload; principalId: string }) => Promise<{ id: string }>;
getPublicBaseUrl: () => string | null;
heartbeat: { cancelRun?: (runId: string, reason?: string, options?: { errorCode?: string; eventMessage?: string; eventPayload?: Record<string, unknown>; resultJson?: Record<string, unknown> }) => Promise<unknown> };
redactError: (error: unknown) => string;
};
export function createSlackSessionStop(db: Db, options: SlackSessionStopOptions) {
const { runtimeCallbackEndpoint, lockCurrentPrincipalAuthorization, nonDirectDestinationAllowed, stageAuthorizedTaskControlPublication, getPublicBaseUrl, redactError } = options;
async function stageSlackSessionStop(
endpoint: EndpointRow,
event: SlackSessionStop,
runtimeContext: RuntimeContext,
): Promise<string | null> {
return db.transaction(async (tx) => {
const currentEndpoint = await runtimeCallbackEndpoint(
tx,
endpoint.id,
runtimeContext,
["active"],
);
if (!currentEndpoint || currentEndpoint.provider !== "slack") return null;
const conversation = await tx
.select()
.from(chatConversations)
.where(
and(
eq(chatConversations.companyId, currentEndpoint.companyId),
eq(chatConversations.endpointId, currentEndpoint.id),
eq(chatConversations.externalThreadId, event.threadId),
inArray(chatConversations.state, ["active", "waiting"]),
),
)
.orderBy(desc(chatConversations.sessionGeneration))
.limit(1)
.for("update")
.then((rows) => rows[0] ?? null);
const principal = await tx
.select()
.from(chatExternalPrincipals)
.where(
and(
eq(chatExternalPrincipals.companyId, currentEndpoint.companyId),
eq(chatExternalPrincipals.provider, "slack"),
eq(
chatExternalPrincipals.providerAccountId,
currentEndpoint.providerAccountId ?? "unknown",
),
eq(chatExternalPrincipals.externalId, event.userId),
eq(chatExternalPrincipals.kind, "user"),
eq(chatExternalPrincipals.isBot, false),
),
)
.for("update")
.then((rows) => rows[0] ?? null);
const issue = conversation
? await tx
.select()
.from(issues)
.where(
and(
eq(issues.companyId, currentEndpoint.companyId),
eq(issues.id, conversation.issueId),
eq(issues.assigneeAgentId, currentEndpoint.assignedAgentId),
),
)
.for("update")
.then((rows) => rows[0] ?? null)
: null;
// Slack carries fractional event timestamps. Treat that exact instant as
// the cancellation boundary so a run admitted even milliseconds after
// Stop was clicked can never be mistaken for the stopped session.
const eventCutoff = event.occurredAt;
const futureEvent = event.occurredAt.getTime() > Date.now() + 60_000;
let target: SlackSessionStopTarget | null = null;
if (issue && !futureEvent && issue.executionRunId) {
const run = await tx
.select({
agentId: heartbeatRuns.agentId,
companyId: heartbeatRuns.companyId,
contextSnapshot: heartbeatRuns.contextSnapshot,
createdAt: heartbeatRuns.createdAt,
id: heartbeatRuns.id,
startedAt: heartbeatRuns.startedAt,
})
.from(heartbeatRuns)
.where(
and(
eq(heartbeatRuns.id, issue.executionRunId),
eq(heartbeatRuns.companyId, currentEndpoint.companyId),
eq(heartbeatRuns.agentId, currentEndpoint.assignedAgentId),
inArray(heartbeatRuns.status, [
"queued",
"running",
"scheduled_retry",
]),
lte(heartbeatRuns.createdAt, eventCutoff),
or(
isNull(heartbeatRuns.startedAt),
lte(heartbeatRuns.startedAt, eventCutoff),
),
or(
sql`${heartbeatRuns.contextSnapshot}->>'issueId' = ${issue.id}`,
sql`${heartbeatRuns.contextSnapshot}->>'taskId' = ${issue.id}`,
),
),
)
.for("update")
.then((rows) => rows[0] ?? null);
if (run) target = { id: run.id, kind: "run" };
} else if (issue && !futureEvent && !issue.executionRunId) {
const wakeup = await tx
.select({ id: agentWakeupRequests.id })
.from(agentWakeupRequests)
.where(
and(
eq(agentWakeupRequests.companyId, currentEndpoint.companyId),
eq(agentWakeupRequests.agentId, currentEndpoint.assignedAgentId),
inArray(agentWakeupRequests.status, [
"queued",
"deferred_issue_execution",
]),
isNull(agentWakeupRequests.runId),
lte(agentWakeupRequests.requestedAt, eventCutoff),
or(
sql`${agentWakeupRequests.payload}->>'issueId' = ${issue.id}`,
sql`${agentWakeupRequests.payload}->>'taskId' = ${issue.id}`,
sql`${agentWakeupRequests.payload}->'_paperclipWakeContext'->>'issueId' = ${issue.id}`,
sql`${agentWakeupRequests.payload}->'_paperclipWakeContext'->>'taskId' = ${issue.id}`,
),
),
)
.orderBy(desc(agentWakeupRequests.requestedAt))
.limit(1)
.for("update")
.then((rows) => rows[0] ?? null);
if (wakeup) target = { id: wakeup.id, kind: "wakeup" };
}
const payload: SlackSessionStopPayload = {
version: 1,
assignedAgentId: currentEndpoint.assignedAgentId,
eventTimestamp: event.eventTimestamp,
issueId: issue?.id ?? null,
occurredAt: event.occurredAt.toISOString(),
sessionGeneration: conversation?.sessionGeneration ?? null,
target,
threadId: event.threadId,
userId: event.userId,
};
await notifyChatActionWork(tx, "slack_session_stop");
const [inserted] = await tx
.insert(chatActions)
.values({
companyId: currentEndpoint.companyId,
endpointId: currentEndpoint.id,
conversationId: conversation?.id ?? null,
principalId: principal?.id ?? null,
kind: "slack_session_stop",
providerActionId: `slack_session_stop:${event.providerEventId}`,
payload,
status: "received",
})
.onConflictDoNothing()
.returning({ id: chatActions.id });
return (
inserted?.id ??
(await tx
.select({ id: chatActions.id })
.from(chatActions)
.where(
and(
eq(chatActions.endpointId, currentEndpoint.id),
eq(
chatActions.providerActionId,
`slack_session_stop:${event.providerEventId}`,
),
eq(chatActions.kind, "slack_session_stop"),
),
)
.then((rows) => rows[0]?.id ?? null))
);
});
}
async function processSlackSessionStop(actionId: string): Promise<void> {
const now = new Date();
const staleBefore = new Date(now.getTime() - PROVIDER_EFFECT_STALE_MS);
const action = await db
.select()
.from(chatActions)
.where(
and(
eq(chatActions.id, actionId),
eq(chatActions.kind, "slack_session_stop"),
),
)
.then((rows) => rows[0] ?? null);
if (!action || ["processed", "cancelled"].includes(action.status)) return;
if (action.status === "processing" && action.updatedAt > staleBefore)
return;
if (action.status === "failed") {
if (action.result?.retryable !== true) return;
const retryAt =
typeof action.result.retryAt === "string"
? new Date(action.result.retryAt)
: null;
if (retryAt && retryAt > now) return;
} else if (action.status !== "received" && action.status !== "processing") {
return;
}
const payload = slackSessionStopPayload(action.payload);
const previousAttempts =
typeof action.result?.attempts === "number" &&
Number.isSafeInteger(action.result.attempts)
? Math.max(0, action.result.attempts)
: 0;
const attempts = previousAttempts + 1;
if (!payload) {
await db.transaction(async (tx) => {
await notifyChatActionWork(tx, "slack_session_stop");
return tx
.update(chatActions)
.set({
status: "failed",
result: {
attempts,
code: "slack_session_stop_payload_invalid",
retryable: false,
},
updatedAt: now,
})
.where(eq(chatActions.id, action.id));
});
return;
}
type StopClaim =
| { kind: "cancel_run"; runId: string; userId: string }
| { kind: "settled" }
| { kind: "retry" };
const claim = await db.transaction(async (tx): Promise<StopClaim> => {
const current = await tx
.select()
.from(chatActions)
.where(eq(chatActions.id, action.id))
.for("update")
.then((rows) => rows[0] ?? null);
const currentRetryAt =
typeof current?.result?.retryAt === "string"
? new Date(current.result.retryAt)
: null;
const currentRetryIsDue =
currentRetryAt === null ||
(Number.isFinite(currentRetryAt.getTime()) && currentRetryAt <= now);
const eligible =
current?.status === "received" ||
(current?.status === "processing" &&
current.updatedAt <= staleBefore &&
(current.result?.attempts ?? 0) === previousAttempts) ||
(current?.status === "failed" &&
current.result?.retryable === true &&
currentRetryIsDue &&
(current.result?.attempts ?? 0) === previousAttempts);
if (!current || !eligible) return { kind: "settled" };
const settle = async (
code: string,
actorUserId: string | null,
targetKind: "run" | "wakeup" | "none",
): Promise<StopClaim> => {
await notifyChatActionWork(tx, "slack_session_stop");
await tx
.update(chatActions)
.set({
status:
code === "slack_session_stop_cancelled"
? "processed"
: "cancelled",
result: { attempts, code, retryable: false },
updatedAt: now,
})
.where(eq(chatActions.id, current.id));
await logActivity(tx as unknown as Db, {
companyId: current.companyId,
actorType: actorUserId ? "user" : "system",
actorId: actorUserId ?? "slack-session-stop",
action:
code === "slack_session_stop_cancelled"
? "chat.slack_session_stopped"
: "chat.slack_session_stop_filtered",
entityType: "chat_action",
entityId: current.id,
details: {
endpointId: current.endpointId,
conversationId: current.conversationId,
eventTimestamp: payload.eventTimestamp,
targetKind,
resultCode: code,
},
});
return { kind: "settled" };
};
if (
!payload.target ||
!payload.issueId ||
!payload.sessionGeneration ||
!current.conversationId ||
!current.principalId
) {
return settle("slack_session_stop_no_current_work", null, "none");
}
const endpoint = await tx
.select()
.from(chatEndpoints)
.where(
and(
eq(chatEndpoints.id, current.endpointId),
eq(chatEndpoints.companyId, current.companyId),
eq(chatEndpoints.provider, "slack"),
eq(chatEndpoints.status, "active"),
eq(chatEndpoints.assignedAgentId, payload.assignedAgentId),
),
)
.for("update")
.then((rows) => rows[0] ?? null);
const conversation = endpoint
? await tx
.select()
.from(chatConversations)
.where(
and(
eq(chatConversations.id, current.conversationId),
eq(chatConversations.companyId, current.companyId),
eq(chatConversations.endpointId, current.endpointId),
eq(chatConversations.issueId, payload.issueId),
eq(chatConversations.externalThreadId, payload.threadId),
eq(
chatConversations.sessionGeneration,
payload.sessionGeneration,
),
inArray(chatConversations.state, ["active", "waiting"]),
),
)
.for("update")
.then((rows) => rows[0] ?? null)
: null;
const resource =
endpoint && conversation?.resourceId
? await tx
.select()
.from(chatEndpointResources)
.where(
and(
eq(chatEndpointResources.id, conversation.resourceId),
eq(chatEndpointResources.companyId, current.companyId),
eq(chatEndpointResources.endpointId, current.endpointId),
),
)
.for("update")
.then((rows) => rows[0] ?? null)
: null;
const principal = endpoint
? await tx
.select()
.from(chatExternalPrincipals)
.where(
and(
eq(chatExternalPrincipals.id, current.principalId),
eq(chatExternalPrincipals.companyId, current.companyId),
eq(chatExternalPrincipals.provider, "slack"),
eq(
chatExternalPrincipals.providerAccountId,
endpoint.providerAccountId ?? "unknown",
),
eq(chatExternalPrincipals.externalId, payload.userId),
eq(chatExternalPrincipals.kind, "user"),
eq(chatExternalPrincipals.isBot, false),
),
)
.for("update")
.then((rows) => rows[0] ?? null)
: null;
const authorization =
endpoint && principal
? await lockCurrentPrincipalAuthorization(tx, endpoint, principal.id)
: null;
const destinationAllowed =
endpoint && conversation
? conversation.isDirectMessage
? endpoint.allowDirectMessages
: nonDirectDestinationAllowed(endpoint, resource)
: false;
const issue =
endpoint && conversation
? await tx
.select()
.from(issues)
.where(
and(
eq(issues.id, payload.issueId),
eq(issues.companyId, current.companyId),
eq(issues.assigneeAgentId, payload.assignedAgentId),
),
)
.for("update")
.then((rows) => rows[0] ?? null)
: null;
if (
!endpoint ||
!conversation ||
!principal ||
!authorization?.allowed ||
!authorization.userId ||
!destinationAllowed ||
!issue
) {
return settle(
"slack_session_stop_no_longer_authorized",
null,
payload.target.kind,
);
}
const eventCutoff = new Date(payload.occurredAt);
let runId: string | null = null;
if (payload.target.kind === "run") {
runId = payload.target.id;
} else {
const wakeup = await tx
.select()
.from(agentWakeupRequests)
.where(
and(
eq(agentWakeupRequests.id, payload.target.id),
eq(agentWakeupRequests.companyId, current.companyId),
eq(agentWakeupRequests.agentId, payload.assignedAgentId),
lte(agentWakeupRequests.requestedAt, eventCutoff),
or(
sql`${agentWakeupRequests.payload}->>'issueId' = ${issue.id}`,
sql`${agentWakeupRequests.payload}->>'taskId' = ${issue.id}`,
sql`${agentWakeupRequests.payload}->'_paperclipWakeContext'->>'issueId' = ${issue.id}`,
sql`${agentWakeupRequests.payload}->'_paperclipWakeContext'->>'taskId' = ${issue.id}`,
),
),
)
.for("update")
.then((rows) => rows[0] ?? null);
if (
wakeup &&
!wakeup.runId &&
["queued", "deferred_issue_execution"].includes(wakeup.status) &&
!issue.executionRunId
) {
await tx
.update(agentWakeupRequests)
.set({
status: "cancelled",
finishedAt: now,
error: "Cancelled from the bound Slack agent session",
updatedAt: now,
})
.where(
and(
eq(agentWakeupRequests.id, wakeup.id),
inArray(agentWakeupRequests.status, [
"queued",
"deferred_issue_execution",
]),
isNull(agentWakeupRequests.runId),
),
);
const agentName = await tx
.select({ name: agents.name })
.from(agents)
.where(
and(
eq(agents.companyId, current.companyId),
eq(agents.id, payload.assignedAgentId),
),
)
.then((rows) => rows[0]?.name ?? "Paperclip agent");
await stageAuthorizedTaskControlPublication(tx, {
companyId: current.companyId,
conversationId: conversation.id,
endpointId: current.endpointId,
idempotencyKey: `control:stop:${current.id}`,
issueId: issue.id,
payload: projectSafeChatPublication({
classification: "external",
progressState: "failed",
source: "safe_milestone",
text: safeMilestoneText({
agentName,
errorCode: "slack_session_stopped",
issueId: issue.id,
milestone: "failed",
publicBaseUrl: getPublicBaseUrl(),
}),
}),
principalId: principal.id,
});
return settle(
"slack_session_stop_cancelled",
authorization.userId,
"wakeup",
);
}
runId = wakeup?.runId ?? null;
if (!runId && wakeup?.status === "claimed") {
await notifyChatActionWork(tx, "slack_session_stop");
await tx
.update(chatActions)
.set({
status: "failed",
result: {
attempts,
code: "slack_session_stop_wakeup_in_flight",
retryable: true,
retryAt: new Date(Date.now() + 1_000).toISOString(),
},
updatedAt: now,
})
.where(eq(chatActions.id, current.id));
return { kind: "retry" };
}
}
const run = runId
? await tx
.select()
.from(heartbeatRuns)
.where(
and(
eq(heartbeatRuns.id, runId),
eq(heartbeatRuns.companyId, current.companyId),
eq(heartbeatRuns.agentId, payload.assignedAgentId),
payload.target.kind === "wakeup"
? eq(heartbeatRuns.wakeupRequestId, payload.target.id)
: and(
lte(heartbeatRuns.createdAt, eventCutoff),
or(
isNull(heartbeatRuns.startedAt),
lte(heartbeatRuns.startedAt, eventCutoff),
),
),
or(
sql`${heartbeatRuns.contextSnapshot}->>'issueId' = ${issue.id}`,
sql`${heartbeatRuns.contextSnapshot}->>'taskId' = ${issue.id}`,
),
),
)
.for("update")
.then((rows) => rows[0] ?? null)
: null;
if (!run) {
return settle(
"slack_session_stop_target_superseded",
authorization.userId,
payload.target.kind,
);
}
if (run.status === "cancelled") {
return settle(
"slack_session_stop_cancelled",
authorization.userId,
"run",
);
}
if (
issue.executionRunId !== run.id ||
!["queued", "running", "scheduled_retry"].includes(run.status)
) {
return settle(
"slack_session_stop_target_superseded",
authorization.userId,
"run",
);
}
await notifyChatActionWork(tx, "slack_session_stop");
await tx
.update(chatActions)
.set({
status: "processing",
result: {
attempts,
authorizedUserId: authorization.userId,
runId: run.id,
},
updatedAt: now,
})
.where(eq(chatActions.id, current.id));
return {
kind: "cancel_run",
runId: run.id,
userId: authorization.userId,
};
});
if (claim.kind !== "cancel_run") return;
try {
if (!options.heartbeat.cancelRun) {
throw new Error("Heartbeat cancellation is unavailable");
}
await options.heartbeat.cancelRun(
claim.runId,
"Stopped from the bound Slack agent session",
{
errorCode: "slack_session_stopped",
eventMessage: "run cancelled from Slack",
resultJson: {
cancelledByActorType: "user",
cancelledByUserId: claim.userId,
},
eventPayload: {
endpointId: action.endpointId,
conversationId: action.conversationId,
provider: "slack",
},
},
);
await enqueueChatRunMilestones(db, { publicBaseUrl: getPublicBaseUrl() });
const authoritativeRun = await db
.select({ status: heartbeatRuns.status })
.from(heartbeatRuns)
.where(eq(heartbeatRuns.id, claim.runId))
.then((rows) => rows[0] ?? null);
const cancellationWon = authoritativeRun?.status === "cancelled";
await db.transaction(async (tx) => {
await notifyChatActionWork(tx, "slack_session_stop");
const [settled] = await tx
.update(chatActions)
.set({
status: cancellationWon ? "processed" : "cancelled",
result: {
attempts,
code: cancellationWon
? "slack_session_stop_cancelled"
: "slack_session_stop_target_superseded",
runId: claim.runId,
},
updatedAt: new Date(),
})
.where(
and(
eq(chatActions.id, action.id),
eq(chatActions.status, "processing"),
sql`(${chatActions.result}->>'attempts')::int = ${attempts}`,
sql`${chatActions.result}->>'runId' = ${claim.runId}`,
),
)
.returning({ id: chatActions.id });
if (!settled) return;
await logActivity(tx as unknown as Db, {
companyId: action.companyId,
actorType: "user",
actorId: claim.userId,
action: cancellationWon
? "chat.slack_session_stopped"
: "chat.slack_session_stop_filtered",
entityType: "chat_action",
entityId: action.id,
details: {
endpointId: action.endpointId,
conversationId: action.conversationId,
eventTimestamp: payload.eventTimestamp,
targetKind: "run",
resultCode: cancellationWon
? "slack_session_stop_cancelled"
: "slack_session_stop_target_superseded",
},
});
});
} catch (error) {
const retryable = attempts < 5;
await db.transaction(async (tx) => {
await notifyChatActionWork(tx, "slack_session_stop");
return tx
.update(chatActions)
.set({
status: "failed",
result: {
attempts,
code: "slack_session_stop_cancellation_failed",
retryable,
...(retryable
? {
retryAt: new Date(
Date.now() + Math.min(60_000, 1_000 * 2 ** (attempts - 1)),
).toISOString(),
}
: {}),
},
updatedAt: new Date(),
})
.where(
and(
eq(chatActions.id, action.id),
eq(chatActions.status, "processing"),
sql`(${chatActions.result}->>'attempts')::int = ${attempts}`,
sql`${chatActions.result}->>'runId' = ${claim.runId}`,
),
);
});
logger.warn(
{
actionId: action.id,
endpointId: action.endpointId,
attempts,
error: redactError(error),
},
"Slack session stop cancellation deferred",
);
}
}
async function processPendingSlackSessionStops(
limit = 25,
onlyActionId?: string,
) {
const actions = await db.select({ id: chatActions.id }).from(chatActions)
.where(and(dueChatAction("slack_session_stop"), onlyActionId ? eq(chatActions.id, onlyActionId) : undefined))
.orderBy(asc(chatActions.createdAt)).limit(limit);
for (const action of actions) await processSlackSessionStop(action.id);
return actions.length;
}
return { stageSlackSessionStop, processSlackSessionStop, processPendingSlackSessionStops };
}
File diff suppressed because it is too large. Load diff
@@ -0,0 +1,931 @@
import type { EndpointRow, DbTransaction, LifecycleRuntimeFence } from "../runtime-types.js";
import { PROVIDER_EFFECT_STALE_MS } from "../../chat-action-work.js";
import type { chatCredentialMutationLease, CredentialMutationLeaseGuard } from "../../chat-credential-mutation-lease.js";
import {
createHash,
} from "node:crypto";
import {
and,
asc,
eq,
gt,
lte,
or,
sql,
} from "drizzle-orm";
import type { Db } from "@paperclipai/db";
import {
chatActions,
chatEndpoints,
toolConnections,
} from "@paperclipai/db";
import type {
ChatEndpointSetupState,
ToolCredentialSecretRef,
} from "@paperclipai/shared";
import {
telegramStopSubscriptionPlan,
telegramStopSubscriptionConfirmed,
} from "../../chat-telegram-stop-subscription.js";
import {
notFound,
} from "../../../errors.js";
import { logger } from "../../../middleware/logger.js";
import {
logActivity,
} from "../../activity-log.js";
import { classifyChatPublicationError } from "../../chat-publication-errors.js";
import type { EndpointRecord } from "../runtime-types.js";
export const TELEGRAM_COMMANDS = [
{ command: "task", description: "Start or continue a Paperclip task" },
{ command: "status", description: "Show the active Paperclip task" },
{ command: "new", description: "Start a new task after the current one" },
{ command: "close", description: "Close the active chat conversation" },
] as const;
export type TelegramMaintenancePayload = {
version: 1;
operation: "register_commands" | "remove_endpoint" | "stop_subscription";
runtimeGeneration: number;
credentialFingerprint: string;
webhookUrlSha256?: string;
botUserId?: string;
};
export function telegramMaintenancePayload(
payload: Record<string, unknown>,
): TelegramMaintenancePayload | null {
if (
payload.version !== 1 ||
(payload.operation !== "register_commands" &&
payload.operation !== "remove_endpoint" &&
payload.operation !== "stop_subscription") ||
typeof payload.runtimeGeneration !== "number" ||
!Number.isSafeInteger(payload.runtimeGeneration) ||
typeof payload.credentialFingerprint !== "string" ||
(payload.operation === "stop_subscription" &&
(typeof payload.webhookUrlSha256 !== "string" ||
!/^[a-f0-9]{64}$/.test(payload.webhookUrlSha256) ||
typeof payload.botUserId !== "string" ||
!/^[1-9][0-9]{0,15}$/.test(payload.botUserId)))
) {
return null;
}
return payload as TelegramMaintenancePayload;
}
type TelegramMaintenanceOptions = {
endpointRecord: (endpointId: string) => Promise<EndpointRecord | null>;
runtimeContextForRecord: (record: EndpointRecord) => LifecycleRuntimeFence;
withCredentialMutationLease: ReturnType<typeof chatCredentialMutationLease>;
resolveCredentials: (endpoint: EndpointRow) => Promise<Record<string, string>>;
clearCredentials: (endpoint: EndpointRow, lease: CredentialMutationLeaseGuard) => Promise<void>;
runtimeCallbackEndpoint: (tx: DbTransaction, endpointId: string, context: LifecycleRuntimeFence, statuses: EndpointRow["status"][]) => Promise<EndpointRow | null>;
getWebhookPublicBaseUrl: () => string | null;
fetchImpl: typeof fetch;
runtimeGeneration: (setup: ChatEndpointSetupState) => number;
credentialFingerprint: (refs: ToolCredentialSecretRef[]) => string;
redactError: (error: unknown) => string;
credentialCheckTimeoutMs: number;
};
export function createTelegramMaintenance(db: Db, options: TelegramMaintenanceOptions) {
const { endpointRecord, runtimeContextForRecord, withCredentialMutationLease, resolveCredentials, clearCredentials, runtimeCallbackEndpoint, getWebhookPublicBaseUrl, fetchImpl, runtimeGeneration, credentialFingerprint, redactError, credentialCheckTimeoutMs: PROVIDER_CREDENTIAL_CHECK_TIMEOUT_MS } = options;
let telegramSubscriptionScanAfter: string | null = null;
async function stageTelegramMaintenance(
endpoint: EndpointRow,
operation: TelegramMaintenancePayload["operation"],
) {
const record = await endpointRecord(endpoint.id);
if (!record) throw notFound("Chat endpoint not found");
const context = runtimeContextForRecord(record);
const subscriptionScope =
operation === "stop_subscription"
? telegramStopSubscriptionScope(record.endpoint, context)
: null;
if (operation === "stop_subscription" && !subscriptionScope) return null;
const operationGeneration = subscriptionScope
? `${context.generation}:${context.credentialFingerprint}:${subscriptionScope.botUserId}:${subscriptionScope.webhookUrlSha256}`
: operation === "register_commands"
? `${context.generation}:${record.endpoint.setup.testStartedAt ?? record.endpoint.updatedAt.toISOString()}`
: String(context.generation);
const providerActionId = `telegram_maintenance:${operation}:${operationGeneration}`;
const [inserted] = await db
.insert(chatActions)
.values({
companyId: endpoint.companyId,
endpointId: endpoint.id,
kind: "telegram_maintenance",
providerActionId,
payload: {
version: 1,
operation,
runtimeGeneration: context.generation,
credentialFingerprint: context.credentialFingerprint,
...(subscriptionScope
? {
webhookUrlSha256: subscriptionScope.webhookUrlSha256,
botUserId: subscriptionScope.botUserId,
}
: {}),
} satisfies TelegramMaintenancePayload,
status: "received",
})
.onConflictDoNothing()
.returning();
return (
inserted ??
(await db
.select()
.from(chatActions)
.where(
and(
eq(chatActions.endpointId, endpoint.id),
eq(chatActions.providerActionId, providerActionId),
),
)
.then((rows) => rows[0] ?? null))
);
}
async function telegramMaintenanceRequest(
botToken: string,
method: "deleteWebhook" | "deleteMyCommands" | "setMyCommands",
body: Record<string, unknown>,
): Promise<void> {
const response = await fetchImpl(
`https://api.telegram.org/bot${encodeURIComponent(botToken)}/${method}`,
{
method: "POST",
signal: AbortSignal.timeout(PROVIDER_CREDENTIAL_CHECK_TIMEOUT_MS),
headers: { "content-type": "application/json" },
body: JSON.stringify(body),
},
);
const result = (await response.json()) as {
ok?: boolean;
description?: string;
parameters?: { retry_after?: number };
};
if (response.ok && result.ok) return;
throw Object.assign(
new Error(
`Telegram ${method} failed: ${result.description ?? response.status}`,
),
response.status === 429
? {
adapter: "telegram",
code: "RATE_LIMITED",
name: "AdapterRateLimitError",
retryAfter: result.parameters?.retry_after,
response: { status: response.status, headers: response.headers },
}
: {
adapter: "telegram",
status: response.status,
response: { status: response.status, headers: response.headers },
},
);
}
async function processTelegramMaintenance(
actionId: string,
force = false,
credentialLease?: CredentialMutationLeaseGuard,
): Promise<"processed" | "pending" | "failed"> {
if (!credentialLease) {
const initialAction = await db
.select({ endpointId: chatActions.endpointId })
.from(chatActions)
.where(
and(
eq(chatActions.id, actionId),
eq(chatActions.kind, "telegram_maintenance"),
),
)
.then((rows) => rows[0] ?? null);
if (!initialAction) return "failed";
const initialEndpoint = await endpointRecord(initialAction.endpointId);
if (!initialEndpoint) return "failed";
// Recovery sweeps run independently from endpoint configuration. Hold
// the same durable credential lease as reconnect, pause, and removal so
// a stale bot token cannot mutate Telegram after its final runtime fence
// but before transport.
return withCredentialMutationLease(initialEndpoint.endpoint, (lease) =>
processTelegramMaintenance(actionId, force, lease),
);
}
let action = await db
.select()
.from(chatActions)
.where(
and(
eq(chatActions.id, actionId),
eq(chatActions.kind, "telegram_maintenance"),
),
)
.then((rows) => rows[0] ?? null);
if (!action) return "failed";
if (action.status === "processed") return "processed";
const payload = telegramMaintenancePayload(action.payload);
if (!payload) {
await db
.update(chatActions)
.set({
status: "failed",
result: {
code: "telegram_maintenance_payload_invalid",
retryable: false,
},
updatedAt: new Date(),
})
.where(eq(chatActions.id, action.id));
return "failed";
}
if (action.status === "processing") {
if (action.updatedAt > new Date(Date.now() - PROVIDER_EFFECT_STALE_MS))
return "pending";
await db
.update(chatActions)
.set({
status: "failed",
result: {
...(action.result ?? {}),
retryable: true,
retryAt: new Date().toISOString(),
},
updatedAt: new Date(),
})
.where(
and(
eq(chatActions.id, action.id),
eq(chatActions.status, "processing"),
eq(chatActions.updatedAt, action.updatedAt),
),
);
action = await db
.select()
.from(chatActions)
.where(eq(chatActions.id, action.id))
.then((rows) => rows[0] ?? null);
if (!action) return "failed";
}
if (action.status === "failed") {
if (action.result?.retryable !== true && !force) return "failed";
const retryAt =
typeof action.result?.retryAt === "string"
? new Date(action.result.retryAt)
: null;
if (!force && retryAt && retryAt > new Date()) return "pending";
} else if (action.status !== "received") {
return "failed";
}
const attempt =
(typeof action.result?.attempts === "number"
? action.result.attempts
: 0) + 1;
let providerConfirmed = action.result?.providerConfirmed === true;
const [claimed] = await db.transaction(async (tx) => {
await credentialLease.assertOwned(tx);
const rows = await tx
.update(chatActions)
.set({
status: "processing",
result: { attempts: attempt, providerConfirmed },
updatedAt: new Date(),
})
.where(
and(
eq(chatActions.id, action!.id),
eq(chatActions.status, action!.status),
),
)
.returning();
await credentialLease.assertOwned(tx);
return rows;
});
if (!claimed) return "pending";
try {
const record = await endpointRecord(action.endpointId);
if (!record || record.endpoint.provider !== "telegram") {
throw Object.assign(new Error("Telegram endpoint is unavailable"), {
code: "CHAT_PROVIDER_PRETRANSPORT_REJECTED",
});
}
if (
payload.operation === "remove_endpoint" &&
record.credentialSecretRefs.length === 0
) {
// Recovery may observe the narrow crash window after provider cleanup
// and credential clearing but before the action receipt was committed.
// Empty refs prove removal cleanup crossed its final local boundary;
// settle the idempotent action instead of misclassifying it as a
// credential rotation.
await db.transaction(async (tx) => {
await credentialLease.assertOwned(tx);
await tx
.update(chatActions)
.set({
status: "processed",
result: { attempts: attempt, providerConfirmed: true },
updatedAt: new Date(),
})
.where(
and(
eq(chatActions.id, action!.id),
eq(chatActions.status, "processing"),
sql`(${chatActions.result}->>'attempts')::int = ${attempt}`,
),
);
await credentialLease.assertOwned(tx);
});
return "processed";
}
if (
credentialFingerprint(record.credentialSecretRefs) !==
payload.credentialFingerprint ||
(payload.operation !== "remove_endpoint" &&
runtimeGeneration(record.endpoint.setup) !==
payload.runtimeGeneration)
) {
await db.transaction(async (tx) => {
await credentialLease.assertOwned(tx);
await tx
.update(chatActions)
.set({
status: "cancelled",
result: {
attempts: attempt,
code: "telegram_maintenance_credentials_superseded",
},
updatedAt: new Date(),
})
.where(
and(
eq(chatActions.id, action!.id),
eq(chatActions.status, "processing"),
sql`(${chatActions.result}->>'attempts')::int = ${attempt}`,
),
);
await credentialLease.assertOwned(tx);
});
return "failed";
}
if (!providerConfirmed || payload.operation === "stop_subscription") {
const credentials = await resolveCredentials(record.endpoint);
if (!credentials.botToken) {
throw Object.assign(new Error("Telegram bot token is unavailable"), {
code: "CHAT_PROVIDER_PRETRANSPORT_REJECTED",
});
}
if (payload.operation === "stop_subscription") {
await repairTelegramStopSubscription(
record.endpoint,
payload,
credentialLease,
);
providerConfirmed = true;
} else if (payload.operation === "register_commands") {
await credentialLease.assertOwned();
await telegramMaintenanceRequest(
credentials.botToken,
"setMyCommands",
{ commands: TELEGRAM_COMMANDS },
);
providerConfirmed = true;
} else {
await credentialLease.assertOwned();
await telegramMaintenanceRequest(
credentials.botToken,
"deleteWebhook",
{ drop_pending_updates: false },
);
await credentialLease.assertOwned();
await telegramMaintenanceRequest(
credentials.botToken,
"deleteMyCommands",
{},
);
providerConfirmed = true;
}
await credentialLease.assertOwned();
await db.transaction(async (tx) => {
await credentialLease.assertOwned(tx);
const confirmed = await tx
.update(chatActions)
.set({
result: { attempts: attempt, providerConfirmed: true },
updatedAt: new Date(),
})
.where(
and(
eq(chatActions.id, action.id),
eq(chatActions.status, "processing"),
sql`(${chatActions.result}->>'attempts')::int = ${attempt}`,
),
)
.returning({ id: chatActions.id });
if (!confirmed.length) {
throw new Error(
"Telegram maintenance ownership changed before provider confirmation",
);
}
await credentialLease.assertOwned(tx);
});
}
if (payload.operation === "remove_endpoint") {
await credentialLease.assertOwned();
await clearCredentials(record.endpoint, credentialLease);
}
await db.transaction(async (tx) => {
await credentialLease.assertOwned(tx);
const processed = await tx
.update(chatActions)
.set({
status: "processed",
result: { attempts: attempt, providerConfirmed: true },
updatedAt: new Date(),
})
.where(
and(
eq(chatActions.id, action.id),
eq(chatActions.status, "processing"),
sql`(${chatActions.result}->>'attempts')::int = ${attempt}`,
),
)
.returning({ id: chatActions.id });
if (!processed.length) {
throw new Error(
"Telegram maintenance ownership changed before success settlement",
);
}
await credentialLease.assertOwned(tx);
});
await logActivity(db, {
companyId: record.endpoint.companyId,
actorType: "system",
actorId: "chat-maintenance",
action: `chat_endpoint.telegram_${payload.operation}_completed`,
entityType: "tool_connection",
entityId: record.endpoint.connectionId,
details: { endpointId: record.endpoint.id, attempt },
});
return "processed";
} catch (error) {
const disposition = classifyChatPublicationError(error, attempt);
// Webhook/menu mutations are idempotent provider operations. A lost
// response is safe to retry, and an explicit provider Retry-After must
// remain durable rather than occupying an in-memory timer.
const retryable =
disposition.kind === "retry" || disposition.kind === "delivery_unknown";
const retryAfterMs =
disposition.kind === "retry"
? disposition.retryAfterMs
: Math.min(60_000, 1_000 * 2 ** Math.max(0, attempt - 1));
const retryAt = retryable
? new Date(Date.now() + retryAfterMs).toISOString()
: null;
const diagnostic = redactError(error);
await db
.update(chatActions)
.set({
status: "failed",
result: {
attempts: attempt,
code: `telegram_maintenance_${disposition.kind}`,
providerConfirmed:
providerConfirmed || action.result?.providerConfirmed === true,
redactedError: diagnostic,
retryable,
retryAt,
},
updatedAt: new Date(),
})
.where(
and(
eq(chatActions.id, action.id),
eq(chatActions.status, "processing"),
sql`(${chatActions.result}->>'attempts')::int = ${attempt}`,
),
);
await logActivity(db, {
companyId: action.companyId,
actorType: "system",
actorId: "chat-maintenance",
action: `chat_endpoint.telegram_${payload.operation}_deferred`,
entityType: "tool_connection",
entityId:
(await endpointRecord(action.endpointId))?.endpoint.connectionId ??
action.endpointId,
details: {
endpointId: action.endpointId,
attempt,
disposition: disposition.kind,
retryAt,
},
}).catch((auditError) => {
logger.error(
{ endpointId: action!.endpointId, error: redactError(auditError) },
"could not persist Telegram maintenance deferral audit",
);
});
logger.warn(
{
endpointId: action.endpointId,
operation: payload.operation,
retryAt,
},
"Telegram maintenance was deferred",
);
return retryable ? "pending" : "failed";
}
}
async function processPendingTelegramMaintenance(limit = 25) {
// Keyset scanning bounds each sweep without starving endpoints behind a
// page whose subscriptions were already confirmed. No provider I/O here.
const candidates = await db
.select({ endpoint: chatEndpoints })
.from(chatEndpoints)
.innerJoin(
toolConnections,
and(
eq(toolConnections.id, chatEndpoints.connectionId),
eq(toolConnections.companyId, chatEndpoints.companyId),
),
)
.where(
and(
eq(chatEndpoints.provider, "telegram"),
eq(chatEndpoints.status, "active"),
eq(toolConnections.status, "active"),
eq(toolConnections.enabled, true),
telegramSubscriptionScanAfter
? gt(chatEndpoints.id, telegramSubscriptionScanAfter)
: undefined,
),
)
.orderBy(asc(chatEndpoints.id))
.limit(Math.max(1, Math.min(limit, 100)));
telegramSubscriptionScanAfter =
candidates.length >= Math.max(1, Math.min(limit, 100))
? candidates.at(-1)!.endpoint.id
: null;
for (const { endpoint } of candidates) {
try {
const record = await endpointRecord(endpoint.id);
if (
record &&
!(await hasTelegramStopSubscription(
record.endpoint,
runtimeContextForRecord(record),
))
)
await stageTelegramMaintenance(record.endpoint, "stop_subscription");
} catch {
logger.warn(
{ endpointId: endpoint.id },
"Telegram subscription maintenance could not be staged",
);
}
}
const now = new Date();
const staleBefore = new Date(now.getTime() - PROVIDER_EFFECT_STALE_MS);
const actions = await db
.select({ id: chatActions.id })
.from(chatActions)
.where(
and(
eq(chatActions.kind, "telegram_maintenance"),
sql`(${chatActions.payload}->>'operation' <> 'stop_subscription' or exists (
select 1 from ${chatEndpoints} inner join ${toolConnections}
on ${toolConnections.id} = ${chatEndpoints.connectionId} and ${toolConnections.companyId} = ${chatEndpoints.companyId}
where ${chatEndpoints.id} = ${chatActions.endpointId} and ${chatEndpoints.companyId} = ${chatActions.companyId}
and ${chatEndpoints.provider} = 'telegram' and ${chatEndpoints.status} = 'active'
and ${toolConnections.status} = 'active' and ${toolConnections.enabled} = true
))`,
or(
eq(chatActions.status, "received"),
and(
eq(chatActions.status, "processing"),
lte(chatActions.updatedAt, staleBefore),
),
and(
eq(chatActions.status, "failed"),
sql`coalesce(${chatActions.result}->>'retryable', 'false') = 'true'`,
sql`(${chatActions.result}->>'retryAt' is null or (${chatActions.result}->>'retryAt')::timestamptz <= ${now.toISOString()}::timestamptz)`,
),
),
),
)
.orderBy(asc(chatActions.createdAt))
.limit(limit);
for (const action of actions) await processTelegramMaintenance(action.id);
return actions.length;
}
function telegramStopSubscriptionScope(
endpoint: EndpointRow,
context: LifecycleRuntimeFence,
) {
if (
endpoint.provider !== "telegram" ||
!endpoint.botExternalId ||
!getWebhookPublicBaseUrl()
)
return null;
const webhookUrlSha256 = createHash("sha256")
.update(
`${getWebhookPublicBaseUrl()}/api/chat-webhooks/${endpoint.publicId}/telegram`,
)
.digest("hex");
return {
version: 1,
botUserId: endpoint.botExternalId,
runtimeGeneration: context.generation,
credentialFingerprint: context.credentialFingerprint,
webhookUrlSha256,
};
}
async function hasTelegramStopSubscription(
endpoint: EndpointRow,
context: LifecycleRuntimeFence,
): Promise<boolean> {
const scope = telegramStopSubscriptionScope(endpoint, context);
if (!scope) return false;
const [receipt] = await db
.select()
.from(chatActions)
.where(
and(
eq(chatActions.companyId, endpoint.companyId),
eq(chatActions.endpointId, endpoint.id),
eq(chatActions.kind, "telegram_stop_subscription"),
eq(
chatActions.providerActionId,
`telegram-stop-subscription:${scope.runtimeGeneration}:${scope.credentialFingerprint}:${scope.webhookUrlSha256}`,
),
eq(chatActions.status, "processed"),
),
)
.limit(1);
return (
!!receipt &&
receipt.result?.code === "telegram_stop_subscription_confirmed" &&
Object.keys(receipt.payload).length === Object.keys(scope).length &&
Object.entries(scope).every(
([key, value]) => receipt.payload[key] === value,
)
);
}
async function recordTelegramStopSubscription(
endpoint: EndpointRow,
context: LifecycleRuntimeFence,
lease: CredentialMutationLeaseGuard,
): Promise<void> {
const scope = telegramStopSubscriptionScope(endpoint, context);
if (!scope)
throw new Error("Telegram Stop subscription scope is unavailable");
await db.transaction(async (tx) => {
await lease.assertOwned(tx);
const current = await runtimeCallbackEndpoint(tx, endpoint.id, context, [
"verifying",
"active",
]);
if (
!current ||
current.botExternalId !== scope.botUserId ||
current.publicId !== endpoint.publicId
)
throw new Error("Telegram Stop subscription ownership changed");
const [connection] = await tx
.select()
.from(toolConnections)
.where(
and(
eq(toolConnections.id, current.connectionId),
eq(toolConnections.companyId, current.companyId),
),
)
.for("share");
if (
!connection?.enabled ||
connection.status !== "active" ||
credentialFingerprint(connection.credentialSecretRefs) !==
context.credentialFingerprint
)
throw new Error(
"Telegram Stop subscription credential ownership changed",
);
await tx
.insert(chatActions)
.values({
companyId: endpoint.companyId,
endpointId: endpoint.id,
kind: "telegram_stop_subscription",
providerActionId: `telegram-stop-subscription:${scope.runtimeGeneration}:${scope.credentialFingerprint}:${scope.webhookUrlSha256}`,
payload: scope,
status: "processed",
result: { code: "telegram_stop_subscription_confirmed" },
})
.onConflictDoUpdate({
target: [chatActions.endpointId, chatActions.providerActionId],
set: {
payload: scope,
status: "processed",
result: { code: "telegram_stop_subscription_confirmed" },
updatedAt: new Date(),
},
});
await lease.assertOwned(tx);
});
}
async function repairTelegramStopSubscription(
endpoint: EndpointRow,
payload: TelegramMaintenancePayload,
lease: CredentialMutationLeaseGuard,
): Promise<void> {
const plainObject = (value: unknown): Record<string, unknown> =>
value !== null && typeof value === "object" && !Array.isArray(value)
? (value as Record<string, unknown>)
: {};
const context = {
generation: payload.runtimeGeneration,
credentialFingerprint: payload.credentialFingerprint,
};
const scope = telegramStopSubscriptionScope(endpoint, context);
const reject = () =>
Object.assign(
new Error(
"Telegram subscription cannot be safely upgraded under current ownership",
),
{
code: "CHAT_PROVIDER_PRETRANSPORT_REJECTED",
},
);
if (
!scope ||
scope.webhookUrlSha256 !== payload.webhookUrlSha256 ||
scope.botUserId !== payload.botUserId
)
throw reject();
const assertCurrent = async () =>
db.transaction(async (tx) => {
await lease.assertOwned(tx);
const current = await runtimeCallbackEndpoint(
tx,
endpoint.id,
context,
["active"],
);
if (
!current ||
current.companyId !== endpoint.companyId ||
current.botExternalId !== scope.botUserId ||
current.publicId !== endpoint.publicId
)
throw reject();
const [connection] = await tx
.select()
.from(toolConnections)
.where(
and(
eq(toolConnections.id, current.connectionId),
eq(toolConnections.companyId, current.companyId),
),
)
.for("share");
if (
!connection?.enabled ||
connection.status !== "active" ||
credentialFingerprint(connection.credentialSecretRefs) !==
context.credentialFingerprint
)
throw reject();
await lease.assertOwned(tx);
});
await assertCurrent();
// A recovered providerConfirmed flag is not a subscription capability.
// Only the separately committed exact current receipt can avoid fresh I/O.
if (await hasTelegramStopSubscription(endpoint, context)) return;
const credentials = await resolveCredentials(endpoint);
if (
!credentials.botToken ||
typeof credentials.webhookSecret !== "string" ||
!/^[A-Za-z0-9_-]{1,256}$/.test(credentials.webhookSecret)
)
throw reject();
const request = async (
method: "getMe" | "getWebhookInfo" | "setWebhook",
body: Record<string, unknown> = {},
) => {
await assertCurrent();
let response: Response;
try {
response = await fetchImpl(
`https://api.telegram.org/bot${encodeURIComponent(credentials.botToken!)}/${method}`,
{
method: "POST",
redirect: "error",
signal: AbortSignal.timeout(PROVIDER_CREDENTIAL_CHECK_TIMEOUT_MS),
headers: { "content-type": "application/json" },
body: JSON.stringify(body),
},
);
} catch {
// Network errors can embed a token-bearing URL. Keep maintenance
// diagnostics closed rather than forwarding provider/transport prose.
throw new Error(
"Telegram subscription transport did not confirm completion",
);
}
let value: unknown;
try {
value = await response.json();
} catch {
throw new Error("Telegram subscription returned an invalid response");
}
const result = plainObject(value);
if (!response.ok || result.ok !== true) {
const retryAfter = plainObject(result.parameters).retry_after;
throw Object.assign(
new Error("Telegram subscription request was not confirmed"),
response.status === 429
? {
adapter: "telegram",
code: "RATE_LIMITED",
name: "AdapterRateLimitError",
retryAfter:
typeof retryAfter === "number" && Number.isFinite(retryAfter)
? retryAfter
: undefined,
response: {
status: response.status,
headers: response.headers,
},
}
: {
adapter: "telegram",
status: response.status,
response: {
status: response.status,
headers: response.headers,
},
},
);
}
return result.result;
};
const identity = plainObject(await request("getMe"));
if (
identity.is_bot !== true ||
!Number.isSafeInteger(identity.id) ||
String(identity.id) !== scope.botUserId
)
throw reject();
const url = `${getWebhookPublicBaseUrl()}/api/chat-webhooks/${endpoint.publicId}/telegram`;
const plan = telegramStopSubscriptionPlan(
await request("getWebhookInfo"),
url,
);
if (!plan) throw reject();
// No URL repoint, credential rotation, certificate guess, IP pin, or queue
// dropping. A successful SET proves the stored secret; GET cannot do so.
if (
(await request("setWebhook", {
url,
secret_token: credentials.webhookSecret,
...plan,
drop_pending_updates: false,
})) !== true
)
throw new Error("Telegram subscription mutation was not confirmed");
if (
!telegramStopSubscriptionConfirmed(
await request("getWebhookInfo"),
url,
plan,
)
)
throw new Error("Telegram subscription verification changed");
await assertCurrent();
await recordTelegramStopSubscription(endpoint, context, lease);
}
return { stageTelegramMaintenance, processTelegramMaintenance, processPendingTelegramMaintenance, hasTelegramStopSubscription, recordTelegramStopSubscription };
}
@@ -0,0 +1,223 @@
import { createHash } from "node:crypto";
import type { chatDeliveries } from "@paperclipai/db";
import type { ChatProvider } from "@paperclipai/shared";
import { normalizeTelegramRichMessage } from "../../chat-telegram-rich-intake.js";
import { MAX_INBOUND_TEXT, type LifecycleActor } from "../message-lifecycle.js";
type TelegramLifecycleEvent = {
actor: LifecycleActor;
eventKind: "message_updated";
messageId: string;
providerEventId?: string;
providerMessageSequence: number;
providerSentAt: string;
providerUpdateId: number | null;
revision: string;
text: string;
threadId: string;
};
function telegramLifecycleActor(message: {
from?: unknown;
sender_chat?: unknown;
}): LifecycleActor | null {
const candidate =
message.from &&
typeof message.from === "object" &&
!Array.isArray(message.from)
? (message.from as Record<string, unknown>)
: message.sender_chat &&
typeof message.sender_chat === "object" &&
!Array.isArray(message.sender_chat)
? (message.sender_chat as Record<string, unknown>)
: null;
if (!candidate) return null;
const id = candidate.id;
if (typeof id !== "string" && typeof id !== "number") return null;
const handle =
typeof candidate.username === "string" ? candidate.username : "";
const displayName =
[candidate.first_name, candidate.last_name]
.filter(
(value): value is string => typeof value === "string" && Boolean(value),
)
.join(" ") ||
(typeof candidate.title === "string" ? candidate.title : "") ||
handle ||
String(id);
return { externalId: String(id), displayName, handle };
}
export function telegramLifecycleEventFromPayload(
payload: unknown,
): TelegramLifecycleEvent | null {
if (!payload || typeof payload !== "object" || Array.isArray(payload))
return null;
const update = payload as {
edited_message?: unknown;
update_id?: unknown;
};
const edited = update.edited_message;
if (!edited || typeof edited !== "object" || Array.isArray(edited))
return null;
const message = edited as {
caption?: unknown;
chat?: { id?: unknown };
edit_date?: unknown;
from?: unknown;
message_id?: unknown;
message_thread_id?: unknown;
sender_chat?: unknown;
text?: unknown;
rich_message?: unknown;
};
const chatId = message.chat?.id;
const messageId = message.message_id;
const editDate = message.edit_date;
if (
(typeof chatId !== "string" && typeof chatId !== "number") ||
typeof messageId !== "number" ||
!Number.isSafeInteger(messageId) ||
messageId < 0 ||
typeof editDate !== "number" ||
!Number.isSafeInteger(editDate) ||
editDate <= 0
)
return null;
const topicId = message.message_thread_id;
if (topicId !== undefined && typeof topicId !== "number") return null;
const rich = normalizeTelegramRichMessage(message);
const body =
rich?.text ??
(typeof message.text === "string"
? message.text
: typeof message.caption === "string"
? message.caption
: "");
const chat = String(chatId);
const providerUpdateId =
typeof update.update_id === "number" &&
Number.isSafeInteger(update.update_id) &&
update.update_id >= 0
? update.update_id
: null;
const bodyHash = createHash("sha256").update(body).digest("hex");
const actor = telegramLifecycleActor(message);
if (!actor) return null;
return {
actor,
eventKind: "message_updated",
messageId: `${chat}:${messageId}`,
...(providerUpdateId === null
? {}
: { providerEventId: `telegram:update:${providerUpdateId}` }),
providerMessageSequence: messageId,
providerSentAt: new Date(editDate * 1_000).toISOString(),
providerUpdateId,
// Bot API timestamps have one-second resolution. The update id is the
// authoritative identity when present; retaining a content hash keeps the
// fallback path from collapsing two distinct edits in that same second.
revision: `${editDate}:${bodyHash}${rich ? `:${rich.mediaDigest}` : ""}`,
text: `An external message was edited:\n\n${body.slice(0, MAX_INBOUND_TEXT)}`,
threadId:
topicId === undefined
? `telegram:${chat}`
: `telegram:${chat}:${topicId}`,
};
}
export function isTelegramEditedMessageRaw(raw: unknown): boolean {
return (
!!raw &&
typeof raw === "object" &&
!Array.isArray(raw) &&
typeof (raw as { edit_date?: unknown }).edit_date === "number"
);
}
export function isTelegramMigrationRaw(raw: unknown): boolean {
if (!raw || typeof raw !== "object" || Array.isArray(raw)) return false;
const value = raw as {
migrate_from_chat_id?: unknown;
migrate_to_chat_id?: unknown;
};
return (
value.migrate_from_chat_id !== undefined ||
value.migrate_to_chat_id !== undefined
);
}
export function telegramMessageSequence(raw: unknown): number | null {
if (!raw || typeof raw !== "object" || Array.isArray(raw)) return null;
const value = (raw as { message_id?: unknown }).message_id;
return typeof value === "number" && Number.isSafeInteger(value) && value > 0
? value
: null;
}
export function telegramMessageId(raw: unknown): string | null {
if (!raw || typeof raw !== "object" || Array.isArray(raw)) return null;
const value = raw as {
chat?: { id?: unknown };
message_id?: unknown;
};
const messageId = value.message_id;
const chatId = value.chat?.id;
if (
typeof messageId !== "number" ||
!Number.isSafeInteger(messageId) ||
messageId <= 0
)
return null;
const normalizedChatId =
typeof chatId === "number" && Number.isSafeInteger(chatId)
? String(chatId)
: typeof chatId === "string" && /^-?\d+$/.test(chatId)
? chatId
: null;
return normalizedChatId ? `${normalizedChatId}:${messageId}` : null;
}
export function telegramZeroMessageId(value: unknown): boolean {
// The pinned adapter qualifies native message IDs with the numeric chat ID.
// Keep legacy synthetic/non-numeric IDs compatible; zero is specifically
// the provider's non-ordinary message identity, never an admitted source.
return typeof value === "string" && /^(?:-?\d+:)?0$/.test(value);
}
export function telegramDeliveryHasZeroMessageId(
delivery: Pick<typeof chatDeliveries.$inferSelect, "normalizedEvent" | "providerEventId">,
provider: ChatProvider,
): boolean {
if (provider !== "telegram") return false;
const normalized = delivery.normalizedEvent as {
message?: {
providerMessageId?: unknown;
providerMessageSequence?: unknown;
};
conversation?: { externalThreadId?: unknown };
};
if (
telegramZeroMessageId(normalized.message?.providerMessageId) ||
normalized.message?.providerMessageSequence === 0
)
return true;
const threadId = normalized.conversation?.externalThreadId;
// Check the independently retained event key only inside its exact thread
// namespace, not arbitrary IDs ending in ':0' from synthetic commands.
return (
typeof threadId === "string" &&
delivery.providerEventId.startsWith(`${threadId}:`) &&
telegramZeroMessageId(delivery.providerEventId.slice(threadId.length + 1))
);
}
export function telegramMessageSentAt(raw: unknown): Date | null {
if (!raw || typeof raw !== "object" || Array.isArray(raw)) return null;
const value = (raw as { date?: unknown }).date;
if (typeof value !== "number" || !Number.isSafeInteger(value) || value <= 0)
return null;
const sentAt = new Date(value * 1_000);
return Number.isFinite(sentAt.getTime()) ? sentAt : null;
}
@@ -0,0 +1,104 @@
import { and, eq, sql } from "drizzle-orm";
import { chatConversations, chatEndpointResources, chatEndpoints, type Db } from "@paperclipai/db";
import type { ChatProviderLifecycleEffect } from "../../chat-provider-lifecycle.js";
type DbTransaction = Parameters<Parameters<Db["transaction"]>[0]>[0];
type EndpointRow = typeof chatEndpoints.$inferSelect;
export function telegramResourceLabelIsFallback(
providerResourceId: string,
label: string,
): boolean {
const candidate = label.trim();
return (
candidate === providerResourceId ||
candidate === `telegram:${providerResourceId}`
);
}
export async function prepareTelegramResourceMigration(
tx: DbTransaction, currentEndpoint: EndpointRow,
effect: Extract<ChatProviderLifecycleEffect, { kind: "resource" }>,
providerResourceId: string, previousProviderResourceId: string | undefined, now: Date,
) {
const previousResource = previousProviderResourceId
? await tx
.select()
.from(chatEndpointResources)
.where(
and(
eq(chatEndpointResources.endpointId, currentEndpoint.id),
eq(chatEndpointResources.type, "chat"),
eq(
chatEndpointResources.providerResourceId,
previousProviderResourceId,
),
),
)
.for("update")
.then((rows) => rows[0] ?? null)
: null;
const migratedLabel =
previousResource &&
telegramResourceLabelIsFallback(providerResourceId, effect.label)
? previousResource.label
: effect.label;
const migratedEnabled = previousResource?.enabled === true;
if (
previousResource &&
previousProviderResourceId !== providerResourceId
) {
await tx
.update(chatEndpointResources)
.set({
availability: "unavailable",
enabled: false,
metadata: {
...previousResource.metadata,
source: "chat_migration",
migratedTo: providerResourceId,
},
updatedAt: now,
})
.where(eq(chatEndpointResources.id, previousResource.id));
}
return { previousResource, migratedLabel, migratedEnabled };
}
export async function migrateTelegramResourceConversations(
tx: DbTransaction, currentEndpoint: EndpointRow,
previousResource: typeof chatEndpointResources.$inferSelect,
resource: { id: string }, previousProviderResourceId: string | undefined,
providerResourceId: string, migratedLabel: string, now: Date,
) {
const previousThreadPrefix = `telegram:${previousProviderResourceId}`;
const migratedThreadPrefix = `telegram:${providerResourceId}`;
await tx
.update(chatConversations)
.set({
resourceId: resource.id,
// Telegram SDK threads use the namespaced channel id. Keep
// the migrated row in that canonical shape so the first
// reply from the replacement supergroup finds this same
// task instead of creating a second conversation.
externalConversationId: migratedThreadPrefix,
externalThreadId: sql<string>`case
when ${chatConversations.externalThreadId} = ${previousThreadPrefix} then ${migratedThreadPrefix}
when ${chatConversations.externalThreadId} like ${`${previousThreadPrefix}:%`}
then ${migratedThreadPrefix} || substring(
${chatConversations.externalThreadId}
from char_length(${previousThreadPrefix}) + 1
)
else ${chatConversations.externalThreadId}
end`,
externalLabel: migratedLabel,
state: "active",
updatedAt: now,
})
.where(
and(
eq(chatConversations.companyId, currentEndpoint.companyId),
eq(chatConversations.endpointId, currentEndpoint.id),
eq(chatConversations.resourceId, previousResource.id),
),
);
}
+57 -28
View File
@@ -19,7 +19,7 @@ import {
companySkillTestInputs,
companySkillTestRunTemplates,
companySkillTestRuns,
companySkillVersions,
companySkillVersions, companySkillRepositorySnapshots,
companySkills,
costEvents,
documents,
@@ -30,6 +30,7 @@ import {
issueWorkProducts,
} from "@paperclipai/db";
import { readPaperclipSkillSyncPreference, writePaperclipSkillSyncPreference } from "@paperclipai/adapter-utils/server-utils";
import { REPOSITORY_CONTENT_DIR, repositorySkillProjection } from "./skill-repository-package.js";
import type { PaperclipDesiredSkillEntry, PaperclipSkillEntry } from "@paperclipai/adapter-utils/server-utils";
import type {
AgentDesiredSkillEntry,
@@ -393,6 +394,8 @@ type BundledSkillReleaseManifestEntry = {
};
type CreateVersionOptions = {
repositorySnapshotId?: string;
repositorySkillPath?: string;
fileInventory?: CompanySkillVersionFileInventoryEntry[];
release?: { id: string; name: string; releasedAt: Date };
updateCurrentVersion?: boolean;
@@ -695,7 +698,11 @@ function deriveCanonicalSkillKey(
}
export function classifyInventoryKind(relativePath: string): CompanySkillFileInventoryEntry["kind"] {
const normalized = normalizePortablePath(relativePath).toLowerCase();
let normalized = normalizePortablePath(relativePath).toLowerCase();
// Projection adds a container, not a new meaning for the original file path.
while (normalized.startsWith(`${REPOSITORY_CONTENT_DIR}/`)) {
normalized = normalized.slice(REPOSITORY_CONTENT_DIR.length + 1);
}
if (normalized.endsWith("/skill.md") || normalized === "skill.md") return "skill";
const fileName = path.posix.basename(normalized);
// Font packages commonly keep their binary assets in a dedicated fonts folder.
@@ -2524,7 +2531,7 @@ function pushFinding(
findings.push({ code, severity, message, path: filePath });
}
async function auditInstalledSkillBytes(skill: CompanySkill): Promise<CompanySkillAuditResult> {
async function auditInstalledSkillBytes(skill: CompanySkill, requireEntrypoint = true): Promise<CompanySkillAuditResult> {
const skillDir = normalizeSkillDirectory(skill);
const scannedAt = new Date().toISOString();
const originHash = asString(getSkillMeta(skill).originHash);
@@ -2554,7 +2561,7 @@ async function auditInstalledSkillBytes(skill: CompanySkill): Promise<CompanySki
sha256: sha256Buffer(file.bytes),
})));
if (!actualPaths.includes("SKILL.md")) {
if (requireEntrypoint && !actualPaths.includes("SKILL.md")) {
pushFinding(findings, "missing_skill_md", "error", "Skill inventory does not contain SKILL.md.", "SKILL.md");
}
@@ -2574,7 +2581,7 @@ async function auditInstalledSkillBytes(skill: CompanySkill): Promise<CompanySki
const fileMap = new Map(files.map((file) => [file.path, file]));
const skillFile = fileMap.get("SKILL.md");
if (skillFile) {
if (requireEntrypoint && skillFile) {
const markdown = skillFile.bytes.toString("utf8");
const parsed = parseFrontmatterMarkdown(markdown);
if (!markdown.startsWith("---\n") || !asString(parsed.frontmatter.name)) {
@@ -2582,7 +2589,8 @@ async function auditInstalledSkillBytes(skill: CompanySkill): Promise<CompanySki
}
}
const remoteExecPattern = /\b(?:curl|wget)\b[\s\S]{0,160}\|\s*(?:sh|bash)|\b(?:bash|sh)\s+-c\b|\beval\b|\bpython\s+-c\b|\bnode\s+-e\b/i;
const remoteExecPattern = /\b(?:curl|wget)\b[\s\S]{0,160}\|\s*(?:sh|bash)|\b(?:bash|sh)\s+-c\b|\beval(?:\s*\(|[ \t]+["'$`])|\bpython\s+-c\b|\bnode\s+-e\b/i;
const shellEvalPattern = /(?:^|[;&|(){}!\n]|\b(?:if|then|else|elif|while|until|do|time)\b)[ \t]*(?:(?:command|builtin)[ \t]+)*["']?eval["']?(?:\s|;|$)/i;
const secretExfilPattern = /\b(?:cat|printenv|env|grep)\b[\s\S]{0,160}(?:\.aws\/credentials|\.ssh\/|\.npmrc|id_rsa|OPENAI_API_KEY|ANTHROPIC_API_KEY|API_KEY|TOKEN|SECRET)[\s\S]{0,160}\b(?:curl|wget|nc|netcat|scp)\b/i;
const networkPattern = /\b(?:curl|wget|fetch|httpie|nc|netcat|scp|ssh)\b|https?:\/\//i;
const secretReferencePattern = /\b(?:process\.env|printenv|\$[A-Z][A-Z0-9_]{2,}|API_KEY|TOKEN|SECRET|PASSWORD|\.env)\b/i;
@@ -2603,7 +2611,7 @@ async function auditInstalledSkillBytes(skill: CompanySkill): Promise<CompanySki
if (file.kind === "asset" && contentLooksBinary(file.bytes)) continue;
const text = file.bytes.toString("utf8");
if (remoteExecPattern.test(text)) {
if (remoteExecPattern.test(text) || (file.kind === "script" && shellEvalPattern.test(text))) {
pushFinding(findings, "remote_fetch_exec", "error", "Remote-fetch or dynamic execution pattern is not allowed.", file.path);
}
if (secretExfilPattern.test(text)) {
@@ -2647,7 +2655,7 @@ async function auditInstalledSkillBytes(skill: CompanySkill): Promise<CompanySki
}
/** Audit downloaded packages before they become installed company content. */
export async function auditSkillSnapshot(files: CompanySkillVersionFileInventoryEntry[]) {
export async function auditSkillSnapshot(files: CompanySkillVersionFileInventoryEntry[], requireEntrypoint = true) {
const root = await fs.mkdtemp(path.join(os.tmpdir(), "paperclip-skill-audit-"));
try {
for (const file of files) {
@@ -2656,7 +2664,7 @@ export async function auditSkillSnapshot(files: CompanySkillVersionFileInventory
await fs.writeFile(target, skillFileBytes(file), { mode: 0o600 });
}
const result = await auditInstalledSkillBytes({ sourceType: "local_path", sourceLocator: root,
fileInventory: files, metadata: null } as unknown as CompanySkill);
fileInventory: files, metadata: null } as unknown as CompanySkill, requireEntrypoint);
return result.findings;
} finally { await fs.rm(root, { recursive: true, force: true }); }
}
@@ -3493,7 +3501,12 @@ export function companySkillService(db: Db) {
const version = await getVersion(companyId, skill.id, skill.currentVersionId);
if (!version) throw unprocessable("Installed skill version is unavailable.");
const sourceLocator = await materializeVersionSnapshot(companyId, skill, version);
const audit = await auditInstalledSkillBytes({ ...skill, sourceType: "local_path", sourceLocator });
const audit = await auditInstalledSkillBytes({
...skill,
sourceType: "local_path",
sourceLocator,
fileInventory: await runtimeVersionInventory(companyId, version),
});
await persistAuditMetadata(skill, audit);
return audit;
}
@@ -3558,11 +3571,14 @@ export function companySkillService(db: Db) {
if (!skill) return null;
const usedByAgents = await usage(companyId, skill.key);
const existingForks = await existingForkSummaries(companyId, skill.id, actor);
const currentVersion = await getCurrentVersion(skill);
const repositoryInventory = currentVersion?.repositorySnapshotId
? await runtimeVersionInventory(companyId, currentVersion) : null;
return enrichSkill(
skill,
repositoryInventory ? { ...skill, fileInventory: repositoryInventory.map(({ path, kind }) => ({ path, kind })) } : skill,
usedByAgents.length,
usedByAgents,
await getCurrentVersion(skill),
currentVersion,
await isStarredByActor(companyId, skill.id, actor),
existingForks,
);
@@ -3629,6 +3645,7 @@ export function companySkillService(db: Db) {
const fileInventory = serializeVersionFileInventory(
options.fileInventory ?? await collectVersionFileInventory(companyId, skill),
);
const previousVersion = options.fileInventory === undefined ? await getCurrentVersion(skill) : null;
const persist = async (tx: DbOrTransaction) => {
await tx.execute(sql`
select ${companySkills.id}
@@ -3648,6 +3665,8 @@ export function companySkillService(db: Db) {
.values({
companyId,
companySkillId: skillId,
repositorySnapshotId: options.repositorySnapshotId ?? previousVersion?.repositorySnapshotId ?? null,
repositorySkillPath: options.repositorySkillPath ?? previousVersion?.repositorySkillPath ?? null,
revisionNumber: Number(nextRevision ?? 1),
label: input.label?.trim() || null,
releaseId: options.release?.id ?? null,
@@ -3984,8 +4003,10 @@ export function companySkillService(db: Db) {
const forkDir = path.resolve(managedRoot, forkSlug);
await fs.rm(forkDir, { recursive: true, force: true });
await fs.mkdir(forkDir, { recursive: true });
for (const entry of source.fileInventory) {
const detail = await readFile(companyId, source.id, entry.path);
const sourceVersion = await getCurrentVersion(source);
const repositoryFiles = sourceVersion?.repositorySnapshotId ? await runtimeVersionInventory(companyId, sourceVersion) : null;
for (const entry of repositoryFiles ?? source.fileInventory) {
const detail = repositoryFiles?.find(file => file.path === entry.path) ?? await readFile(companyId, source.id, entry.path);
if (!detail) continue;
const targetPath = path.resolve(forkDir, detail.path);
await fs.mkdir(path.dirname(targetPath), { recursive: true });
@@ -4335,19 +4356,19 @@ export function companySkillService(db: Db) {
async function readLoadedSkillFile(skill: CompanySkill, relativePath: string): Promise<CompanySkillFileDetail> {
const normalizedPath = normalizePortablePath(relativePath || "SKILL.md");
const fileEntry = skill.fileInventory.find((entry) => entry.path === normalizedPath);
if (!fileEntry) {
throw notFound("Skill file not found");
}
const source = deriveSkillSourceInfo(skill);
if (skill.metadata?.skillSourceId && skill.metadata?.snapshotHash && skill.currentVersionId) {
const version = await getCurrentVersion(skill);
const file = version?.fileInventory.find(entry => entry.path === normalizedPath);
// Keep original package-relative reads working, and expose the repository
// files from the same saved snapshot used by runtime delivery.
const file = version?.fileInventory.find(entry => entry.path === normalizedPath)
?? (version?.repositorySnapshotId ? (await runtimeVersionInventory(skill.companyId, version)).find(entry => entry.path === normalizedPath) : undefined);
if (!file) throw notFound("Installed skill snapshot is missing a file.");
return { ...file, skillId: skill.id, language: file.encoding === "base64" ? null : inferLanguageFromPath(normalizedPath),
markdown: file.encoding !== "base64" && isMarkdownPath(normalizedPath), editable: false };
}
const fileEntry = skill.fileInventory.find((entry) => entry.path === normalizedPath);
if (!fileEntry) throw notFound("Skill file not found");
const source = deriveSkillSourceInfo(skill);
let content = "";
if (skill.sourceType === "local_path" || skill.sourceType === "catalog") {
@@ -4474,6 +4495,8 @@ export function companySkillService(db: Db) {
if (input.forkedFromSkillId && !forkSource) {
throw notFound("Fork source skill not found");
}
const forkVersion = forkSource ? await getCurrentVersion(forkSource) : null;
const repositoryFiles = forkVersion?.repositorySnapshotId ? await runtimeVersionInventory(companyId, forkVersion) : null;
const sharingScope = normalizeMutableSharingScope(input.sharingScope) ?? "company";
const managedRoot = resolveManagedSkillsRoot(companyId);
const skillDir = path.resolve(managedRoot, slug);
@@ -4491,7 +4514,7 @@ export function companySkillService(db: Db) {
].join("\n");
const markdown = input.markdown?.trim().length
? input.markdown
: forkSource?.markdown ?? fallbackMarkdown;
: repositoryFiles?.find(file => file.path === "SKILL.md")?.content ?? forkSource?.markdown ?? fallbackMarkdown;
return db.transaction(async (tx) => {
await tx.execute(sql`
@@ -4508,8 +4531,8 @@ export function companySkillService(db: Db) {
let published = false;
try {
if (forkSource) {
for (const entry of forkSource.fileInventory) {
const detail = await readFile(companyId, forkSource.id, entry.path);
for (const entry of repositoryFiles ?? forkSource.fileInventory) {
const detail = repositoryFiles?.find(file => file.path === entry.path) ?? await readFile(companyId, forkSource.id, entry.path);
if (!detail) continue;
const targetPath = path.resolve(stagingDir, detail.path);
await fs.mkdir(path.dirname(targetPath), { recursive: true });
@@ -4601,10 +4624,7 @@ export function companySkillService(db: Db) {
.set({ forkCount: sql`${companySkills.forkCount} + 1`, installCount: sql`${companySkills.installCount} + 1`, updatedAt: new Date() })
.where(and(eq(companySkills.id, forkSource.id), eq(companySkills.companyId, companyId)));
}
const versionInventory = await Promise.all(inventory.map(async (entry) => ({
...entry,
content: await fs.readFile(path.join(skillDir, entry.path), "utf8"),
})));
const versionInventory = await collectVersionFileInventoryFromDirectory(skillDir);
await createVersion(companyId, created.id, { label: "Initial version" }, actor, {
database: tx,
skipInventoryRefresh: true,
@@ -5939,7 +5959,16 @@ export function companySkillService(db: Db) {
return true;
}
async function runtimeVersionInventory(companyId: string, version: CompanySkillVersion) {
if (!version.repositorySnapshotId) return version.fileInventory;
const [snapshot] = await db.select().from(companySkillRepositorySnapshots).where(and(
eq(companySkillRepositorySnapshots.companyId, companyId), eq(companySkillRepositorySnapshots.id, version.repositorySnapshotId)));
if (!snapshot || !version.repositorySkillPath) throw unprocessable("Installed repository snapshot is unavailable.");
return repositorySkillProjection(snapshot.fileInventory, version.repositorySkillPath);
}
async function materializeVersionSnapshot(companyId: string, skill: CompanySkill, version: CompanySkillVersion): Promise<string> {
version = { ...version, fileInventory: await runtimeVersionInventory(companyId, version) };
const runtimeRoot = path.resolve(resolveManagedSkillsRoot(companyId), "__versions__", skill.id);
const skillDir = path.join(runtimeRoot, version.id);
const existing = versionSnapshotFlights.get(skillDir);
+82 -11
View File
@@ -1,4 +1,5 @@
import path from 'node:path';
import { MAX_REPOSITORY_BYTES, MAX_REPOSITORY_FILES, repositoryPackageInspection } from './skill-repository-package.js';
import type { CompanySkillVersionFileInventoryEntry, SkillSourceCandidate, SkillSourceDiscovery, SkillSourcePreviewRequest, SkillSourceFilePreview, SkillSourceScanUpdate, SkillSourceScanProgress } from '@paperclipai/shared';
import { parseFrontmatterMarkdown, parseGitHubSkillRepositoryUrl } from '@paperclipai/shared';
import { notFound, unprocessable } from '../errors.js';
@@ -15,7 +16,7 @@ export interface GitHubRead {
readonly connectionId?: string | null;
}
export type DiscoveredSkill = SkillSourceCandidate & { files: CompanySkillVersionFileInventoryEntry[] };
export type ScannedSkillSource = SkillSourceDiscovery & { skills: DiscoveredSkill[]; defaultBranch: string };
export type ScannedSkillSource = SkillSourceDiscovery & { skills: DiscoveredSkill[]; defaultBranch: string; repositoryFiles?: CompanySkillVersionFileInventoryEntry[] };
const MAX_FILE_BYTES = 1024 * 1024;
const MAX_SCAN_BYTES = 100 * 1024 * 1024;
const MAX_SCAN_PACKAGES = 1_000;
@@ -34,7 +35,7 @@ export interface SkillScanOptions {
retainFiles?: boolean;
}
type SkillScanInput = { repositoryUrl: string; trackingRef?: string; commitSha?: string; onlySkillPath?: string };
type SkillScanInput = { packageMode?: 'skills' | 'repository'; repositoryUrl: string; trackingRef?: string; commitSha?: string; onlySkillPath?: string };
export function scanGitHubSkills(input: SkillScanInput, providerRead: GitHubRead, options: SkillScanOptions = {}): Promise<ScannedSkillSource> {
const scan = () => scanRepository(input, providerRead, options);
@@ -91,7 +92,7 @@ async function scanRepository(input: SkillScanInput, providerRead: GitHubRead, o
siblings.push(root);
rootsByDirectory.set(dir, siblings);
}
const selectedRoots = roots.filter(root => !input.onlySkillPath || root.path === input.onlySkillPath).sort((a, b) => a.path.localeCompare(b.path));
const selectedRoots = roots.filter(root => input.packageMode === 'repository' || !input.onlySkillPath || root.path === input.onlySkillPath).sort((a, b) => a.path.localeCompare(b.path));
if (selectedRoots.length > MAX_SCAN_PACKAGES) throw unprocessable('Repository exceeds the 1,000 skill package scan limit.');
// Assign each file to its nearest package root once, including hidden and
// nested directories. Memoized ancestors avoid a full-tree pass per skill.
@@ -126,8 +127,13 @@ async function scanRepository(input: SkillScanInput, providerRead: GitHubRead, o
}
if (declaredBytes > MAX_SCAN_BYTES) throw unprocessable('Skill packages exceed the 100 MB scan limit.');
}
const warnings = entries.filter(e => e.mode === '120000' || e.type === 'commit').map(e => `Not followed: ${e.path} (${e.mode === '120000' ? 'symlink' : 'submodule'}).`);
const blobs = new Map<string, Buffer>();
const warnings = entries.filter(e => e.mode === '120000' || e.type === 'commit').map(e => `${e.path} — ${e.mode === '120000' ? 'Symlink' : 'Submodule'} couldn’t be included.`);
const repositoryInventory = input.packageMode === 'repository' ? entries.filter(entry => entry.type !== 'tree') : null;
if (repositoryInventory && repositoryInventory.length > MAX_REPOSITORY_FILES) throw unprocessable('Repository package exceeds the 9,990 file limit.');
const readable = repositoryInventory?.filter(entry => entry.type === 'blob' && ['100644', '100755'].includes(entry.mode) && (entry.size ?? 0) <= MAX_FILE_BYTES);
if (readable && readable.reduce((sum, entry) => sum + (entry.size ?? 0), 0) > MAX_REPOSITORY_BYTES) throw unprocessable('Repository package exceeds the 61 MB runtime limit.');
// A single bounded batch supplies both skill manifests and support files.
const blobs = readable && snapshot.readBlobs ? await snapshot.readBlobs(readable, options.signal) : new Map<string, Buffer>();
let totalBytes = 0;
const readBlob = async (entry: TreeEntry) => {
if ((entry.size ?? 0) > MAX_FILE_BYTES) return null;
@@ -153,19 +159,21 @@ async function scanRepository(input: SkillScanInput, providerRead: GitHubRead, o
Object.assign(progress, { currentPath: root.path, checkedFiles: 0, totalFiles: inventory.length });
await report();
const files: CompanySkillVersionFileInventoryEntry[] = [];
const fileWarnings: string[] = [];
let error: string | null = rootsByDirectory.get(dir)!.length > 1 ? 'Multiple SKILL.md entrypoints share this package directory.' : null;
for (const entry of inventory) {
progress.currentPath = entry.path;
await report();
progress.checkedFiles++;
if (!['100644', '100755'].includes(entry.mode) || entry.type !== 'blob') { error = `Unsupported symlink or submodule: ${entry.path}`; continue; }
if (!['100644', '100755'].includes(entry.mode) || entry.type !== 'blob') { fileWarnings.push(`${entry.path} — ${entry.mode === '120000' ? 'Symlink' : 'Submodule'} couldn’t be included.`); continue; }
const bytes = await readBlob(entry);
if (!bytes) { error = `File exceeds the 1 MB limit: ${entry.path}`; continue; }
if (!bytes) { fileWarnings.push(`${entry.path} — Exceeds the 1 MB file limit; not included.`); continue; }
const relative = entry.path === root.path ? 'SKILL.md' : entry.path.slice(prefix.length);
const kind = relative !== 'SKILL.md' && (entry.mode === '100755' || bytes.subarray(0, 2).toString() === '#!') ? 'script' : classifyInventoryKind(relative);
files.push(snapshotFile(relative, kind, bytes, entry.mode === '100755'));
}
const markdown = files.find(f => f.path === 'SKILL.md');
if (!markdown) error = 'SKILL.md couldn’t be included.';
const frontmatter = markdown && markdown.encoding !== 'base64' ? parseFrontmatterMarkdown(markdown.content).frontmatter : {};
if (markdown?.encoding === 'base64') error = 'SKILL.md must contain UTF-8 text.';
const name = typeof frontmatter.name === 'string' && frontmatter.name.trim() ? frontmatter.name.trim() : path.posix.basename(dir) || repo.full_name;
@@ -173,13 +181,77 @@ async function scanRepository(input: SkillScanInput, providerRead: GitHubRead, o
const findings = !error ? await auditSkillSnapshot(files) : [];
error ??= findings.filter(f => f.severity === 'error').map(f => `${f.path ?? root.path}: ${f.message}`).join(' ') || null;
const inspection = { ...inspectSkillPackage(root.path, files, repositoryPaths, frontmatter, findings), commitSha: commit.sha };
inspection.warnings.push(...fileWarnings);
skills.push({ path: root.path, name, description, fileCount: inventory.length, error, warnings: inspection.warnings, inspection, files: options.retainFiles === false ? [] : files });
options.signal?.throwIfAborted();
await options.onProgress?.({ type: 'candidate', candidate: { path: root.path, name, description, fileCount: inventory.length, error } });
progress.checkedSkills++;
await report();
}
return { connectionId: providerRead.connectionId ?? null, repositoryId: String(repo.id), repositoryUrl: `https://github.com/${repo.full_name.toLowerCase()}`, fullName: repo.full_name, trackingRef, commitSha: commit.sha,
let repositoryFiles: CompanySkillVersionFileInventoryEntry[] | undefined;
let repositoryPackage: ReturnType<typeof repositoryPackageInspection> | undefined;
if (input.packageMode === 'repository') {
const inventory = repositoryInventory!;
repositoryFiles = [];
const packageWarnings = [...warnings];
totalBytes = 0;
Object.assign(progress, { currentPath: null, checkedFiles: 0, totalFiles: inventory.length });
for (const entry of inventory) {
options.signal?.throwIfAborted();
if (!['100644', '100755'].includes(entry.mode) || entry.type !== 'blob') {
continue;
}
const bytes = await readBlob(entry);
if (!bytes) { packageWarnings.push(`${entry.path} — Exceeds the 1 MB file limit; not included.`); continue; }
if (totalBytes > MAX_REPOSITORY_BYTES) throw unprocessable('Repository package exceeds the 61 MB runtime limit.');
const kind = entry.mode === '100755' || bytes.subarray(0, 2).toString() === '#!'
? 'script' : classifyInventoryKind(entry.path);
repositoryFiles.push(snapshotFile(entry.path, kind, bytes, entry.mode === '100755'));
progress.currentPath = entry.path;
progress.checkedFiles++;
await report();
}
const findings = await auditSkillSnapshot(repositoryFiles, false);
const unsafe = findings.filter(f => f.severity === 'error' && ['remote_fetch_exec', 'secret_exfiltration'].includes(f.code));
if (unsafe.some(f => !f.path)) throw unprocessable('Repository audit could not identify an unsafe file.');
const omitted = new Map(unsafe.map(f => [f.path!, `${f.path} — ${f.message} File not included.`]));
repositoryFiles = repositoryFiles.filter(file => !omitted.has(file.path));
repositoryPackage = repositoryPackageInspection(repositoryFiles, roots.map(root => root.path),
[...new Set([...packageWarnings, ...omitted.values(), ...findings.filter(f => f.severity === 'error' && !omitted.has(f.path!)).map(f => `${f.path}: ${f.message}`)])], null);
const included = indexSkillPackagePaths(repositoryFiles.map(file => file.path));
const byPath = new Map(repositoryFiles.map(file => [file.path, file]));
for (const skill of skills) {
if (skill.inspection) {
const dir = path.posix.dirname(skill.path);
const repositoryPath = (relative: string) => relative === 'SKILL.md' ? skill.path : path.posix.join(dir, relative);
const omittedWarnings = skill.inspection.files.flatMap(file => {
const warning = omitted.get(repositoryPath(file.path));
return warning ? [warning] : [];
});
if (omittedWarnings.length) {
const supported = skill.inspection.files.flatMap(file => {
const original = byPath.get(repositoryPath(file.path));
return original ? [{ ...original, path: file.path, kind: file.path === 'SKILL.md' ? 'skill' as const : original.kind === 'script' ? 'script' as const : classifyInventoryKind(file.path) }] : [];
});
const markdown = supported.find(file => file.path === 'SKILL.md');
const frontmatter = markdown && markdown.encoding !== 'base64' ? parseFrontmatterMarkdown(markdown.content).frontmatter : {};
const remainingFindings = await auditSkillSnapshot(supported);
skill.error = rootsByDirectory.get(dir)!.length > 1 ? 'Multiple SKILL.md entrypoints share this package directory.'
: remainingFindings.filter(f => f.severity === 'error').map(f => `${f.path}: ${f.message}`).join(' ') || null;
const fileWarnings = skill.inspection.warnings.filter(warning => warning.includes('not included') || warning.includes('couldn’t be included'));
skill.inspection = { ...inspectSkillPackage(skill.path, supported, included, frontmatter, remainingFindings), commitSha: commit.sha };
skill.inspection.warnings.push(...fileWarnings, ...omittedWarnings);
skill.warnings = skill.inspection.warnings;
skill.fileCount = supported.length;
skill.files = options.retainFiles === false ? [] : supported;
}
}
if (skill.inspection) skill.inspection.references = skill.inspection.references.filter(ref => !included.has(ref.resolvedPath));
}
}
return { ...(input.packageMode ? { packageMode: input.packageMode } : {}), ...(repositoryPackage ? { repositoryPackage } : {}),
...(repositoryFiles && options.retainFiles !== false ? { repositoryFiles } : {}),
connectionId: providerRead.connectionId ?? null, repositoryId: String(repo.id), repositoryUrl: `https://github.com/${repo.full_name.toLowerCase()}`, fullName: repo.full_name, trackingRef, commitSha: commit.sha,
defaultBranch: repo.default_branch, candidates: skills.map(({ files: _files, ...candidate }) => candidate), warnings, skills };
} finally { await snapshot.release(); }
}
@@ -190,9 +262,8 @@ export async function previewGitHubSkillFile(input: SkillSourcePreviewRequest, r
if (scan.commitSha.toLowerCase() !== input.commitSha.toLowerCase()) throw unprocessable('The preview commit did not match the requested snapshot.');
const skill = scan.skills.find(candidate => candidate.path === input.skillPath);
if (!skill) throw notFound('Skill package not found at this commit.');
if (skill.error) throw unprocessable(`Preview unavailable: ${skill.error}`);
const file = skill.files.find(file => file.path === input.filePath);
const manifest = skill.inspection?.files.find(file => file.path === input.filePath);
const file = (input.packageMode === 'repository' ? scan.repositoryFiles : skill.files)?.find(file => file.path === input.filePath);
const manifest = (input.packageMode === 'repository' ? scan.repositoryPackage?.files : skill.inspection?.files)?.find(file => file.path === input.filePath);
if (!file || !manifest) throw notFound('File is not included in this skill package.');
const bytes = skillFileBytes(file);
const limit = 64 * 1024;
File diff suppressed because it is too large. Load diff
@@ -0,0 +1,269 @@
import { eq, sql } from "drizzle-orm";
import { heartbeatRuns, type Db, type agents, type projects } from "@paperclipai/db";
import type { AiConnectionBinding, AiRuntimeConnectionBinding } from "@paperclipai/shared";
import { readPaperclipSkillSyncPreference } from "@paperclipai/adapter-utils/server-utils";
import { parseObject } from "../../adapters/utils.js";
import { logger } from "../../middleware/logger.js";
import { AI_CONNECTION_BUSY_RETRY_REASON, extractWakeCommentIds, isNonAssigneeWorkspaceBusyRetry } from "../../modules/run-dispatch/index.js";
import { createAgentIdentityRedactor } from "../agent-identity-redaction.js";
import { agentIdentityService, supportsManagedAgentIdentity } from "../agent-identity.js";
import { readAiConnectionConfigurationFailure, readAiCredentialAccessFailure } from "../ai-connection-configuration-failure.js";
import { AI_AUTH_ENV_KEYS, isAiConnectionBusy, prepareManagedAiRuntime, stripAiAuthBindings } from "../ai-connection-runtime.js";
import { managedAiSessionIdentityCompatible } from "../ai-connection-session.js";
import type { companySkillService } from "../company-skills.js";
import { connectionIntentService } from "../connection-intents.js";
import { prepareConnectionInstructionDelivery } from "../connection-instructions.js";
import { applyConnectorSkills, prepareConnectorSkillDelivery, resolveConnectorAssignments } from "../connector-runtime.js";
import { buildExecutionWorkspaceAdapterConfig } from "../execution-workspace-policy.js";
import { resolveManagedGitHubIdentitySelection } from "../git-credentials.js";
import { resolveAssignedConnectionInstructionsForRun } from "../native-runtime/assigned-mcp-tools.js";
import { recordFailedSkillPreparation, type NativeRunHistoricalSpan } from "../native-runtime/native-run-trace.js";
import { collectRunFailureSecretValues } from "../run-failure-diagnostics.js";
import { skillVersionSelectionMap } from "../runtime-skill-selections.js";
import type { selectHeartbeatEnvironment } from "./environment-selection.js";
import type { createHeartbeatRetries } from "./retries.js";
import type { createHeartbeatLifecycle } from "./run-lifecycle.js";
import {
ConfigurationIncompleteFailure, applyRunScopedMentionedSkillKeys, resolveExecutionRunAdapterConfig,
resolveRunScopedMentionedSkillKeys, type createHeartbeatRunPreparation, type resolveAcceptedPlanWakeRoutingDecision,
} from "./run-preparation.js";
import { buildExecutionWorkspaceConfigSnapshot, stripWorkspaceRuntimeFromExecutionRunConfig } from "./workspaces.js";
import type { HeartbeatWorkspacePreparationInput } from "./workspace-preparation.js";
const ACCEPTED_PLAN_CONVERSION_SKILL_KEY =
"paperclipai/paperclip/paperclip-converting-plans-to-tasks";
type ManagedAiRuntime = Awaited<ReturnType<typeof prepareManagedAiRuntime>>;
type EnvironmentSelection = Awaited<ReturnType<typeof selectHeartbeatEnvironment>>;
export interface HeartbeatExecutionConfigurationInput {
run: typeof heartbeatRuns.$inferSelect;
agent: typeof agents.$inferSelect;
task: {
issueId: string | null;
context: Record<string, unknown>;
responsibleUserId: string | null;
acceptedPlanContinuationWake: boolean;
acceptedPlanWakeRoutingDecision: Awaited<ReturnType<typeof resolveAcceptedPlanWakeRoutingDecision>> | null;
projectContext: Pick<typeof projects.$inferSelect, "id" | "env"> | null;
routineEnvContext: Awaited<ReturnType<ReturnType<typeof createHeartbeatRunPreparation>["getRoutineEnvForExecutionIssue"]>>;
};
config: {
config: Record<string, unknown>;
issueAssigneeOverrides: { adapterConfig: Record<string, unknown> | null; useProjectWorkspace: boolean | null } | null;
requestedAiBinding: AiRuntimeConnectionBinding | undefined;
aiBinding: AiConnectionBinding | undefined;
};
policy: Pick<HeartbeatWorkspacePreparationInput["policy"],
"trustPreset" | "projectExecutionWorkspacePolicy" | "issueExecutionWorkspaceSettings" | "requestedExecutionWorkspaceMode" | "useIsolatedTaskDirectory">;
environment: Pick<EnvironmentSelection,
"selectedEnvironmentId" | "selectedEnvironmentForConfig" | "resolvedInstanceSettings" | "persistedNativeExecutionInput">;
services: {
secretsSvc: Parameters<typeof resolveExecutionRunAdapterConfig>[0]["secretsSvc"];
companySkills: Pick<ReturnType<typeof companySkillService>, "listRuntimeSkillEntries">;
appendRunEvent: ReturnType<typeof createHeartbeatLifecycle>["appendRunEvent"];
finalizeAiConnectionBusyDeferral: ReturnType<typeof createHeartbeatRetries>["finalizeAiConnectionBusyDeferral"];
};
// The executor must observe these before a later preparation step can throw.
onFailureSecretsResolved: (readSecrets: () => string[]) => void;
onManagedAiRuntimeAcquired: (runtime: ManagedAiRuntime) => void;
onIdentityRedactorPrepared: (redactor: ReturnType<typeof createAgentIdentityRedactor>) => void;
}
/** Assembles effective run configuration; the executor retains cleanup ownership. */
export async function prepareHeartbeatExecutionConfiguration(db: Db, input: HeartbeatExecutionConfigurationInput) {
const { run, agent, onFailureSecretsResolved, onManagedAiRuntimeAcquired, onIdentityRedactorPrepared } = input;
const { issueId, context, responsibleUserId, acceptedPlanContinuationWake, acceptedPlanWakeRoutingDecision, projectContext, routineEnvContext } = input.task;
const { config, issueAssigneeOverrides, requestedAiBinding, aiBinding } = input.config;
const { trustPreset, projectExecutionWorkspacePolicy, issueExecutionWorkspaceSettings, requestedExecutionWorkspaceMode, useIsolatedTaskDirectory } = input.policy;
const { selectedEnvironmentId, selectedEnvironmentForConfig, resolvedInstanceSettings, persistedNativeExecutionInput } = input.environment;
const { secretsSvc, companySkills, appendRunEvent, finalizeAiConnectionBusyDeferral } = input.services;
let managedAiRuntime: ManagedAiRuntime | undefined;
const workspaceManagedConfig = buildExecutionWorkspaceAdapterConfig({
agentConfig: config,
projectPolicy: projectExecutionWorkspacePolicy,
issueSettings: issueExecutionWorkspaceSettings,
mode: requestedExecutionWorkspaceMode,
legacyUseProjectWorkspace:
issueAssigneeOverrides?.useProjectWorkspace ?? null,
});
const mergedConfig = {
...workspaceManagedConfig,
...Object.fromEntries(Object.entries(issueAssigneeOverrides?.adapterConfig ?? {}).filter(([key]) => requestedAiBinding?.mode !== "router" || !["provider", "acpxAgent", "model", "modelReasoningEffort", "reasoningEffort", "effort", "variant"].includes(key))),
...(requestedAiBinding?.mode === "router" ? parseObject(parseObject(context.aiRouterSelection).runtimeConfig) : {}),
// The base below is already task-owned. Keep directory transport while
// preserving isolated mode and the mandatory sandbox preflight.
...(useIsolatedTaskDirectory ? { workspaceStrategy: { type: "project_primary" } } : {}),
};
const configSnapshot = buildExecutionWorkspaceConfigSnapshot(
mergedConfig,
selectedEnvironmentId,
);
const executionRunConfig =
stripWorkspaceRuntimeFromExecutionRunConfig(mergedConfig);
const runScopedMentionedSkillKeys =
await resolveRunScopedMentionedSkillKeys({
db,
companyId: agent.companyId,
issueId,
commentIds: extractWakeCommentIds(context),
});
const runScopedSkillKeys =
acceptedPlanContinuationWake &&
!acceptedPlanWakeRoutingDecision?.suppressAcceptedContinuation
? [...runScopedMentionedSkillKeys, ACCEPTED_PLAN_CONVERSION_SKILL_KEY]
: runScopedMentionedSkillKeys;
const githubSelection = await resolveManagedGitHubIdentitySelection(
db,
agent.companyId,
{
agentId: agent.id,
responsibleUserId,
allowStandingDelegation: false,
},
);
const useHostGitHub =
!githubSelection.configured &&
trustPreset.kind === "standard" &&
["local", "ssh"].includes(
selectedEnvironmentForConfig?.driver ?? "local",
);
const { resolvedConfig, configuredTaskEnvironment, secretKeys, secretManifest } =
await resolveExecutionRunAdapterConfig({
managedAiCredentials: Boolean(aiBinding),
managedGitHubCredentials: !useHostGitHub,
companyId: agent.companyId,
agentId: agent.id,
adapterType: agent.adapterType,
issueId,
heartbeatRunId: run.id,
environmentId: selectedEnvironmentForConfig?.id ?? null,
environmentEnv: aiBinding ? stripAiAuthBindings(selectedEnvironmentForConfig?.envVars) : selectedEnvironmentForConfig?.envVars ?? null,
environmentDriver: selectedEnvironmentForConfig?.driver ?? null,
projectId: projectContext?.id ?? null,
routineId: routineEnvContext.routineId,
responsibleUserId,
executionRunConfig: aiBinding ? { ...executionRunConfig, env: stripAiAuthBindings(executionRunConfig.env) } : executionRunConfig,
projectEnv: aiBinding ? stripAiAuthBindings(projectContext?.env) : projectContext?.env ?? null,
routineEnv: aiBinding ? stripAiAuthBindings(routineEnvContext.env) : routineEnvContext.env,
secretsSvc,
trustPreset,
});
onFailureSecretsResolved(() => collectRunFailureSecretValues(resolvedConfig.env, secretKeys));
if (aiBinding) {
try {
managedAiRuntime = await prepareManagedAiRuntime(db, { companyId: agent.companyId, agentId: agent.id, responsibleUserId, adapterType: agent.adapterType, binding: aiBinding, config: resolvedConfig });
onManagedAiRuntimeAcquired(managedAiRuntime);
} catch (error) {
// Only fresh executions can receive a pre-provider wait receipt. A
// persisted native input may already have provider effects to recover.
if (isAiConnectionBusy(error) && !persistedNativeExecutionInput) {
// Use the authority recorded by the locked admission gate, never
// the issue's mutable assignee observed during runtime preparation.
const authorizedNonAssigneeWake =
parseObject(run.runnerProfileJson).aiConnectionNonAssigneeCommentWake === true ||
(run.scheduledRetryReason === AI_CONNECTION_BUSY_RETRY_REASON &&
isNonAssigneeWorkspaceBusyRetry(run.scheduledRetryReason, parseObject(run.contextSnapshot)));
await finalizeAiConnectionBusyDeferral(run, error, !authorizedNonAssigneeWake);
return { prepared: false as const };
}
if (responsibleUserId && issueId) {
await connectionIntentService(db).request({ sub: agent.id, company_id: agent.companyId, run_id: run.id, responsible_user_id: responsibleUserId }, aiBinding.provider, { purpose: "ai" }).catch(() => {
logger.warn({ runId: run.id, agentId: agent.id }, "Could not attach AI connection request; runtime configuration action remains available");
});
}
throw new ConfigurationIncompleteFailure(error instanceof Error ? error.message : "Configure this agent’s AI connection", {
configurationIncomplete: { reason: "ai_connection_unavailable", companyId: agent.companyId, agentId: agent.id, responsibleUserId,
...(!persistedNativeExecutionInput && readAiConnectionConfigurationFailure(error)
? { selectionFailure: readAiConnectionConfigurationFailure(error) } : {}),
...(readAiCredentialAccessFailure(error)
? { credentialAccess: readAiCredentialAccessFailure(error) } : {}),
provider: aiBinding.provider, method: aiBinding.method, actionUrl: `/agents/${agent.id}/runtime`,
fingerprint: `ai:${agent.id}:${responsibleUserId}:${JSON.stringify(aiBinding)}` },
});
}
const savedAiAccount = parseObject(run.contextSnapshot?.aiConnection);
if (persistedNativeExecutionInput && !managedAiSessionIdentityCompatible(
savedAiAccount.sessionIdentity ?? savedAiAccount.identity,
managedAiRuntime.sessionIdentity,
managedAiRuntime.identity,
)) {
throw new ConfigurationIncompleteFailure("The AI account changed while this native run was suspended. Start a new execution.", { configurationIncomplete: { reason: "ai_connection_changed", actionUrl: `/agents/${agent.id}/runtime` } });
}
Object.assign(resolvedConfig, managedAiRuntime.config);
for (const key of AI_AUTH_ENV_KEYS) secretKeys.add(key);
context.aiConnection = { ...managedAiRuntime.attribution, accountName: managedAiRuntime.accountName, identity: managedAiRuntime.identity, sessionIdentity: managedAiRuntime.sessionIdentity };
await db.update(heartbeatRuns).set({ contextSnapshot: sql`coalesce(${heartbeatRuns.contextSnapshot}, '{}'::jsonb) || ${JSON.stringify({ aiConnection: context.aiConnection, ...(context.aiRouterTaskKey ? { aiRouterTaskKey: context.aiRouterTaskKey } : {}), ...(context.aiRouterSelection ? { aiRouterSelection: context.aiRouterSelection } : {}) })}::jsonb` }).where(eq(heartbeatRuns.id, run.id));
}
if (secretManifest.length > 0) {
context.paperclipSecrets = {
manifest: secretManifest,
};
} else {
delete context.paperclipSecrets;
}
const effectiveResolvedConfig = applyRunScopedMentionedSkillKeys(
resolvedConfig,
runScopedSkillKeys,
);
const runtimeSkillPreference = readPaperclipSkillSyncPreference(
effectiveResolvedConfig,
);
const nativeRunnerPreparationSpans: NativeRunHistoricalSpan[] = [];
const skillsPrepareStartedAtMs = Date.now();
const runtimeSkillEntries = await (async () => {
try {
return await companySkills.listRuntimeSkillEntries(agent.companyId, {
versionSelections: skillVersionSelectionMap(
runtimeSkillPreference.desiredSkillEntries,
{
versionPinsEnabled:
resolvedInstanceSettings.experimental.enableBetaSkills ===
true,
},
),
});
} catch (error) {
if (agent.adapterType === "paperclip_runner") {
await recordFailedSkillPreparation({
runId: run.id,
startedAtMs: skillsPrepareStartedAtMs,
onEvent: async (event) => {
await appendRunEvent(run, event);
},
});
}
throw error;
}
})();
nativeRunnerPreparationSpans.push({
name: "skills.prepare",
parentName: "task.prepare",
startedAtMs: skillsPrepareStartedAtMs,
endedAtMs: Date.now(),
});
const connectorAssignments = await resolveConnectorAssignments(db, { companyId: agent.companyId, agentId: agent.id, runId: run.id, issueId: typeof context.issueId === "string" ? context.issueId : undefined });
const connectorSkillConfig = await applyConnectorSkills(effectiveResolvedConfig, runtimeSkillEntries, connectorAssignments);
// Both CLI adapters and native context materialization use the same resolved set.
runtimeSkillEntries.splice(0, runtimeSkillEntries.length, ...connectorSkillConfig.paperclipRuntimeSkills);
const connectorDelivery = await prepareConnectorSkillDelivery(connectorSkillConfig, agent.adapterType);
// Always replace this runtime-only field; caller wake data cannot supply skills.
context.paperclipWake = { ...parseObject(context.paperclipWake), connectorSkillInstructions: connectorDelivery.instructions };
let runtimeConfig = await prepareConnectionInstructionDelivery({
resolve: () => resolveAssignedConnectionInstructionsForRun(db, { companyId: agent.companyId, agentId: agent.id, runId: run.id }),
context, config: connectorDelivery.config, native: agent.adapterType === "paperclip_runner",
});
const agentIdentity = supportsManagedAgentIdentity(agent.adapterType, agent.adapterConfig, run.runtimeMode === "native" ? run.driverKind : undefined)
? await agentIdentityService(db).ensureAgentIdentity(agent.companyId, agent.id)
: undefined;
const identityRedactor = createAgentIdentityRedactor(agentIdentity?.privateKeyPem);
onIdentityRedactorPrepared(identityRedactor);
if (agentIdentity) secretKeys.add("PAPERCLIP_AGENT_PRIVATE_KEY");
return {
prepared: true as const,
mergedConfig, configSnapshot, runScopedMentionedSkillKeys, githubSelection, useHostGitHub,
resolvedConfig, configuredTaskEnvironment, secretKeys, secretManifest,
runtimeSkillEntries, nativeRunnerPreparationSpans, runtimeConfig, agentIdentity,
};
}
@@ -0,0 +1,100 @@
import { beforeEach, describe, expect, it, vi } from "vitest";
import { createHeartbeatInstructionWorkingCopy } from "./instruction-working-copy.js";
import { AGENT_FILES_CONTRACT } from "../agent-file-store.js";
import { AgentDirectoryReuseInvalidatedError } from "../agent-directory-working-copies.js";
const native = vi.hoisted(() => ({ reserve: vi.fn(), claim: vi.fn() }));
vi.mock("../native-runtime/index.js", async importOriginal => ({
...await importOriginal<typeof import("../native-runtime/index.js")>(),
reserveWarmNativeInstructionDirectory: native.reserve,
claimWarmNativeInstructionCopy: native.claim,
}));
type Input = Parameters<typeof createHeartbeatInstructionWorkingCopy>[0];
type Prepare = Parameters<ReturnType<typeof createHeartbeatInstructionWorkingCopy>["prepare"]>[0];
type Copy = NonNullable<Awaited<ReturnType<Input["instructionCopies"]["prepare"]>>>;
function fixture() {
let run = { id: "current-run", companyId: "company", agentId: "agent", runnerProfileJson: null, status: "queued" } as ReturnType<Input["readRun"]>;
const copy = { runId: run.id, companyId: run.companyId, agentId: run.agentId,
localRoot: "/workspace/copy", executionRoot: "/workspace/copy", entryFile: "AGENTS.md",
state: "prepared", receipt: { schema: AGENT_FILES_CONTRACT, warm: true }, errorCode: null, errorMessage: null,
} as unknown as Copy;
const copies = {
prepare: vi.fn<Input["instructionCopies"]["prepare"]>().mockResolvedValue(copy),
collectStopped: vi.fn<Input["instructionCopies"]["collectStopped"]>().mockResolvedValue({ ...copy, state: "saved" }),
checkpointWarm: vi.fn<Input["instructionCopies"]["checkpointWarm"]>().mockResolvedValue({ ...copy, state: "warm_saved", receipt: { ...copy.receipt, checkpointState: "unchanged" } }),
hasChanges: vi.fn().mockResolvedValue(false), canReuseWarm: vi.fn().mockResolvedValue(true),
reportRetirementUnconfirmed: vi.fn().mockResolvedValue(null),
};
const appendRunEvent = vi.fn().mockResolvedValue(undefined);
const context = { paperclipWorkspace: { agentHome: "/canonical" } } as Record<string, unknown>;
const controller = createHeartbeatInstructionWorkingCopy({
db: {} as Input["db"], instructionCopies: copies as unknown as Input["instructionCopies"],
readRun: () => run,
agent: { id: "agent", companyId: "company", adapterType: "codex_local", adapterConfig: { lifecycleMode: "warm" }, runtimeConfig: {} } as unknown as Input["agent"],
context,
session: { taskSession: null, taskSessionForRun: { lastRunId: "previous-run" }, sessionConfigMetadata: {} as Input["session"]["sessionConfigMetadata"] },
workspace: { executionTarget: null, executionWorkspace: { cwd: "/workspace" } as Input["workspace"]["executionWorkspace"], persistedExecutionWorkspace: null },
appendRunEvent,
});
const prepare: Prepare = { runtimeConfig: { retained: true }, hasInstructionFilesystem: true,
nativeRuntimeResolution: { kind: "native", profile: { backend: "codex_app_server" } } as Prepare["nativeRuntimeResolution"] };
const reservation = { reuseRunId: "previous-run", adopt: vi.fn(), release: vi.fn().mockResolvedValue(undefined) };
native.reserve.mockResolvedValue(reservation);
return { controller, copies, copy, context, prepare, reservation, appendRunEvent,
replaceRun: () => { run = { ...run, status: "running" }; return run; } };
}
beforeEach(() => { vi.clearAllMocks(); native.claim.mockResolvedValue(null); });
describe("instruction working-copy extraction boundary", () => {
it("retains a warm reservation for outer cleanup when subsequent preparation throws", async () => {
const f = fixture();
f.copies.prepare.mockRejectedValue(new Error("preparation interrupted"));
await expect(f.controller.prepare(f.prepare)).rejects.toThrow("preparation interrupted");
expect(f.controller.nativeInstructionReservation).toBe(f.reservation);
expect(f.controller.instructionCopy).toBeNull();
await f.controller.nativeInstructionReservation?.release();
expect(f.reservation.release).toHaveBeenCalledOnce();
});
it("publishes transferred copy ownership before fallible preparation bookkeeping", async () => {
const f = fixture();
f.copies.prepare.mockImplementation(async input => {
input.onWarmHandoff?.(f.copy);
throw new Error("bookkeeping interrupted");
});
await expect(f.controller.prepare(f.prepare)).rejects.toThrow("bookkeeping interrupted");
expect(f.controller.instructionCopy).toBe(f.copy);
expect(f.reservation.adopt).toHaveBeenCalledWith(f.copy.executionRoot, f.controller.collectStoppedInstructions);
await f.controller.nativeInstructionReservation?.release();
expect(f.reservation.release).toHaveBeenCalledOnce();
});
it("releases invalidated reuse before fresh preparation and exposes the replacement copy", async () => {
const f = fixture(), order: string[] = [];
f.reservation.release.mockImplementation(async () => { order.push("release"); });
f.copies.prepare.mockImplementationOnce(async () => { order.push("reuse"); throw new AgentDirectoryReuseInvalidatedError("canonical changed"); })
.mockImplementationOnce(async () => { order.push("fresh"); return f.copy; });
const config = await f.controller.prepare(f.prepare);
expect(order).toEqual(["reuse", "release", "fresh"]);
expect(f.copies.prepare.mock.calls.map(([input]) => input.reuseRunId)).toEqual(["previous-run", undefined]);
expect(f.controller.nativeInstructionReservation).toBeNull();
expect(config).toMatchObject({ retained: true, instructionsFilePath: "/workspace/copy/AGENTS.md" });
expect(f.controller.instructionCopy).toBe(f.copy);
});
it("reads the current run row and receipt during stopped collection and warm checkpoints", async () => {
const f = fixture();
await f.controller.prepare(f.prepare);
const running = f.replaceRun();
await f.controller.collectStoppedInstructions();
expect(f.appendRunEvent.mock.calls.at(-1)?.[0]).toBe(running);
expect(f.controller.getInstructionSave()).toMatchObject({ state: "saved" });
expect(await f.controller.nativeInstructionWorkingCopy()?.checkpointWarm?.()).toBe(true);
expect(f.controller.getInstructionSave()).toMatchObject({ state: "unchanged" });
expect(f.appendRunEvent.mock.calls.at(-1)?.[0]).toBe(running);
expect(f.context.paperclipWorkspace).toMatchObject({ agentHome: f.copy.executionRoot, agentHomeForPermissions: "/canonical" });
});
});
@@ -0,0 +1,233 @@
import { createHash } from "node:crypto";
import path from "node:path";
import { and, eq, sql } from "drizzle-orm";
import { heartbeatRuns, type Db } from "@paperclipai/db";
import { parseObject } from "../../adapters/utils.js";
import { logger } from "../../middleware/logger.js";
import { AgentDirectoryReuseInvalidatedError, isAgentDirectoryCopy } from "../agent-directory-working-copies.js";
import { collectStoppedInstructionCopyWithRetries, instructionWorkingCopyGuidance, type agentInstructionWorkingCopyService } from "../agent-instruction-working-copies.js";
import { claimWarmNativeInstructionCopy, nativeSessionWorkspaceScope, reserveWarmNativeInstructionDirectory } from "../native-runtime/index.js";
import { parseNativeExecutionInput } from "../../vendor/paperclip-runner/index.js";
import type { HeartbeatRuntimeSelectionInput } from "./runtime-selection.js";
import type { createHeartbeatLifecycle } from "./run-lifecycle.js";
import type { EffectiveRunSessionConfigMetadata } from "./workspaces.js";
type Copies = ReturnType<typeof agentInstructionWorkingCopyService>;
type Selection = HeartbeatRuntimeSelectionInput;
type AppendRunEvent = ReturnType<typeof createHeartbeatLifecycle>["appendRunEvent"];
function readNonEmptyString(value: unknown): string | null {
return typeof value === "string" && value.trim().length > 0 ? value : null;
}
/** A failed cleanup is deferred once per run; it cannot replace the run result. */
export function createHeartbeatInstructionCopyRelease(
instructionCleanupRun: Selection["run"],
instructionCopies: Pick<Copies, "release">,
appendRunEvent: AppendRunEvent,
) {
let instructionCleanupDeferred = false;
const releaseInstructionCopy = async () => {
// Cleanup is retried from the durable working-copy receipt by the
// recovery sweep. It must not replace the provider result (or prevent
// lease release), and a timeout must not be repeated in outer teardown.
if (instructionCleanupDeferred) return;
try {
await instructionCopies.release(instructionCleanupRun.companyId, instructionCleanupRun.id);
} catch (err) {
instructionCleanupDeferred = true;
logger.warn({ err, runId: instructionCleanupRun.id }, "Agent file cleanup deferred; run outcome preserved");
await appendRunEvent(instructionCleanupRun, {
eventType: "instruction_cleanup",
stream: "system",
level: "warn",
message: "Agent file cleanup was deferred. The run outcome and file-save receipt are unchanged.",
payload: { state: "deferred" },
}).catch((eventError) => {
logger.warn({ err: eventError, runId: instructionCleanupRun.id }, "Failed to record deferred agent file cleanup");
});
}
};
return releaseInstructionCopy;
}
interface InstructionWorkingCopyInput {
db: Db;
instructionCopies: Copies;
// The executor replaces its run row before preparation. Callbacks must see it.
readRun: () => Selection["run"];
agent: Selection["agent"];
context: Selection["task"]["context"];
session: {
taskSession: { lastRunId: string | null } | null;
taskSessionForRun: Selection["session"]["taskSessionForRun"];
sessionConfigMetadata: EffectiveRunSessionConfigMetadata;
};
workspace: Pick<Selection["workspace"], "executionTarget" | "executionWorkspace" | "persistedExecutionWorkspace">;
appendRunEvent: AppendRunEvent;
}
/** Per-run copy, receipt, and warm ownership; the executor orders final releases. */
export function createHeartbeatInstructionWorkingCopy(input: InstructionWorkingCopyInput) {
const { db, instructionCopies, readRun, agent, context, appendRunEvent } = input;
const { taskSession, taskSessionForRun, sessionConfigMetadata } = input.session;
const { executionTarget, executionWorkspace, persistedExecutionWorkspace } = input.workspace;
let releaseWarmInstructionPreparation: (() => Promise<void>) | null = null;
let nativeInstructionReservation: Awaited<ReturnType<typeof reserveWarmNativeInstructionDirectory>> = null;
let instructionCopy: Awaited<ReturnType<typeof instructionCopies.prepare>> = null;
let instructionSave: Record<string, unknown> | null = null;
const instructionPreparationKey = createHash("sha256").update(JSON.stringify({
adapterType: agent.adapterType, adapterConfig: agent.adapterConfig, runtimeConfig: agent.runtimeConfig, sessionConfigMetadata,
workspace: nativeSessionWorkspaceScope({
binding: { runId: readRun().id, executionWorkspaceId: persistedExecutionWorkspace?.id ?? readRun().id },
workspace: executionWorkspace,
}), cwd: executionWorkspace.cwd,
})).digest("hex");
const recordInstructionSave = async (saved: NonNullable<Awaited<ReturnType<typeof instructionCopies.get>>>) => {
const receipt = parseObject(saved.receipt);
const storageWarning = readNonEmptyString(receipt.storageWarning);
const state = saved.errorCode === "AGENT_FILES_CHECKPOINT_UNSTABLE" ? "pending_collection"
: saved.state === "warm_saved" ? readNonEmptyString(receipt.checkpointState) ?? "saved" : saved.state;
instructionSave = { state, entryFile: saved.entryFile,
...(isAgentDirectoryCopy(saved) ? { contract: "agent_files", appliedCandidateHash: saved.candidateHash, checkpointStats: receipt.checkpointStats }
: { revisionId: parseObject(receipt.revision).id ?? null }), storageWarning, errorCode: saved.errorCode, errorMessage: saved.errorMessage };
await appendRunEvent(readRun(), { eventType: "instruction_save", stream: "system",
level: !saved.errorCode && !storageWarning && ["saved", "unchanged", "resolved"].includes(state) ? "info" : "warn",
message: storageWarning ?? (state === "saved" ? "Agent files saved."
: state === "unchanged" ? "Instruction working copy is unchanged."
: saved.errorMessage ?? "Instruction edits were not saved."), payload: instructionSave });
};
const collectStoppedInstructions = async () => {
if (!instructionCopy) return;
const saved = await collectStoppedInstructionCopyWithRetries(() => instructionCopies.collectStopped({
companyId: agent.companyId, runId: readRun().id, target: executionTarget,
}));
if (!saved) return;
if (saved.state !== "superseded") await recordInstructionSave(saved);
};
const nativeInstructionWorkingCopy = () => instructionCopy ? {
...(isAgentDirectoryCopy(instructionCopy) ? { runId: readRun().id, preparationKey: instructionPreparationKey } : {}),
root: instructionCopy.executionRoot,
...(instructionCopy.receipt?.warm === true ? { checkpointWarm: async () => {
const saved = await instructionCopies.checkpointWarm({ companyId: agent.companyId, runId: readRun().id, target: executionTarget });
if (saved) await recordInstructionSave(saved);
return saved?.state === "warm_saved" && saved.errorCode === null;
} } : {}),
hasChanges: () => instructionCopies.hasChanges({ companyId: agent.companyId, runId: readRun().id, target: executionTarget }),
collectStopped: collectStoppedInstructions,
retirementFailed: async () => { await instructionCopies.reportRetirementUnconfirmed(agent.companyId, readRun().id); },
} : undefined;
async function prepare({ runtimeConfig, nativeRuntimeResolution, hasInstructionFilesystem }: {
runtimeConfig: Record<string, unknown>;
nativeRuntimeResolution: Selection["nativeRuntimeResolution"];
hasInstructionFilesystem: boolean;
}) {
if (hasInstructionFilesystem) {
try {
// Missing contract fields on a restored session mean the deployed
// legacy format. New sessions opt into whole-directory persistence.
const priorFileRun = taskSession?.lastRunId
? await db.select({ profile: heartbeatRuns.runnerProfileJson }).from(heartbeatRuns).where(and(
eq(heartbeatRuns.id, taskSession.lastRunId), eq(heartbeatRuns.companyId, agent.companyId), eq(heartbeatRuns.agentId, agent.id))).then(rows => rows[0])
: null;
const savedFileInput = parseObject(parseObject(readRun().runnerProfileJson).nativeExecutionInput);
const priorFileInput = Object.keys(savedFileInput).length ? savedFileInput : parseObject(parseObject(priorFileRun?.profile).nativeExecutionInput);
const priorWorkingCopy = parseObject(parseObject(parseObject(priorFileInput.runtimeContext).instructions).workingCopy);
const warmFiles = nativeRuntimeResolution.kind === "native" && (nativeRuntimeResolution.profile.backend === "codex_app_server" ||
(nativeRuntimeResolution.profile.backend === "acpx_runtime" && parseObject(agent.adapterConfig).acpxAgent === "cursor")) &&
(executionTarget?.kind === "remote" && executionTarget.transport === "sandbox"
? executionTarget.runnerLifecyclePolicy?.mode === "warm"
: parseObject(agent.adapterConfig).lifecycleMode === "warm");
if (warmFiles && taskSessionForRun?.lastRunId) {
nativeInstructionReservation = await reserveWarmNativeInstructionDirectory({ companyId: agent.companyId, agentId: agent.id,
previousRunId: taskSessionForRun.lastRunId, runId: readRun().id, target: executionTarget,
canReuse: () => instructionCopies.canReuseWarm(agent.companyId, agent.id, taskSessionForRun!.lastRunId!),
});
}
if (!warmFiles && nativeRuntimeResolution.kind === "native" && priorWorkingCopy.kind === "agent_files" && taskSession) {
releaseWarmInstructionPreparation = await claimWarmNativeInstructionCopy({
priorExecution: parseNativeExecutionInput(priorFileInput), companyId: agent.companyId, agentId: agent.id,
executionWorkspaceId: persistedExecutionWorkspace?.id ?? readRun().id, workspace: executionWorkspace,
environmentId: executionTarget?.environmentId ?? null, runId: readRun().id, preparationKey: instructionPreparationKey, forceRetirement: !taskSessionForRun,
adopt: async previousRunId => {
instructionCopy = await instructionCopies.adopt({ companyId: agent.companyId, agentId: agent.id,
runId: readRun().id, previousRunId, target: executionTarget, cwd: executionWorkspace.cwd, allowRetirementHandoff: true });
return nativeInstructionWorkingCopy() ?? null;
},
});
// A collection-only handoff was retired. Re-enter normal
// preparation instead of composing a root already collected.
if (!releaseWarmInstructionPreparation) instructionCopy = null;
}
const prepareInstructions = (reuseRunId?: string) => instructionCopies.prepare({
companyId: agent.companyId, agentId: agent.id, runId: readRun().id,
target: executionTarget, cwd: executionWorkspace.cwd,
legacy: Object.keys(priorFileInput).length > 0 && priorWorkingCopy.kind !== "agent_files",
warm: warmFiles, reuseRunId,
onWarmHandoff: copy => {
instructionCopy = copy;
nativeInstructionReservation?.adopt(copy.executionRoot, collectStoppedInstructions);
},
});
try {
instructionCopy ??= await prepareInstructions(nativeInstructionReservation?.reuseRunId);
} catch (error) {
if (!(error instanceof AgentDirectoryReuseInvalidatedError)) throw error;
// prepare has released its canonical lock. Retirement can now
// collect under that same lock before a fresh restore starts.
await nativeInstructionReservation?.release();
nativeInstructionReservation = null;
instructionCopy = await prepareInstructions();
}
} catch (error) {
if ((error as { status?: number }).status !== 403) throw error;
// Missing write identity must not break a background run's read-only
// prompt. It must also never imply that ordinary file edits will save.
await appendRunEvent(readRun(), { eventType: "instruction_save", stream: "system", level: "warn",
message: "Persistent instruction editing is unavailable. Use an authenticated user with instruction edit access and a managed instruction bundle.",
payload: { state: "unavailable", code: "INSTRUCTION_COPY_UNAVAILABLE" } });
const guidance = "No editable agent instruction working copy is registered for this turn. Use authenticated agent file tools for persistent edits; do not edit a private copy named in an earlier turn or claim its changes will persist.";
for (const key of ["paperclipTaskMarkdown", "paperclipTaskMarkdownCompact"]) {
context[key] = [readNonEmptyString(context[key]), guidance].filter(Boolean).join("\n\n");
}
}
if (instructionCopy) {
const storageWarning = readNonEmptyString(instructionCopy.receipt?.storageWarning);
if (storageWarning) {
instructionSave = { state: "prepared", contract: "agent_files", storageWarning };
// This is an advisory on the run, never an agent pause, execution
// failure, or scheduling gate. Keep it visible while work runs.
await db.update(heartbeatRuns).set({ resultJson: sql`coalesce(${heartbeatRuns.resultJson}, '{}'::jsonb) || ${JSON.stringify({ instructionSave })}::jsonb` }).where(eq(heartbeatRuns.id, readRun().id));
await appendRunEvent(readRun(), { eventType: "instruction_save", stream: "system", level: "warn",
message: storageWarning, payload: instructionSave });
}
runtimeConfig = { ...runtimeConfig, instructionsFilePath: path.join(instructionCopy.localRoot, instructionCopy.entryFile) };
if (isAgentDirectoryCopy(instructionCopy)) {
const workspace = parseObject(context.paperclipWorkspace);
context.paperclipWorkspace = { ...workspace, agentHome: instructionCopy.executionRoot,
// Keep the pre-existing permission root stable for ACP session
// identity. The per-run copy is already under the company root.
agentHomeForPermissions: workspace.agentHome,
};
}
const guidance = instructionWorkingCopyGuidance(instructionCopy);
for (const key of ["paperclipTaskMarkdown", "paperclipTaskMarkdownCompact"]) {
context[key] = [readNonEmptyString(context[key]), guidance].filter(Boolean).join("\n\n");
}
}
}
return runtimeConfig;
}
return {
prepare,
collectStoppedInstructions,
nativeInstructionWorkingCopy,
get instructionCopy() { return instructionCopy; },
getInstructionSave: () => instructionSave,
// These getters remain usable after partial acquisition failures.
get nativeInstructionReservation() { return nativeInstructionReservation; },
get releaseWarmInstructionPreparation() { return releaseWarmInstructionPreparation; },
};
}
+209
View File
@@ -0,0 +1,209 @@
import { eq } from "drizzle-orm";
import { heartbeatRuns, type Db } from "@paperclipai/db";
import type { AdapterInvocationMeta, AdapterRuntimeEvent } from "../../adapters/index.js";
import { redactCurrentUserText, type CurrentUserRedactionOptions } from "../../log-redaction.js";
import type { createAgentIdentityRedactor } from "../agent-identity-redaction.js";
import { touchHeartbeatRunRuntimeStatus } from "../heartbeat-run-runtime-status.js";
import { publishLiveEvent } from "../live-events.js";
import type { RunLogHandle, RunLogStore } from "../run-log-store.js";
import { appendExcerpt, compactRunLogChunk } from "./run-log.js";
import { isHeartbeatRunRuntimeStatusActive, publishHeartbeatRunRuntimeProgress, type createHeartbeatLifecycle } from "./run-lifecycle.js";
const MAX_LIVE_LOG_CHUNK_BYTES = 8 * 1024;
export const ACTIVE_RUN_OUTPUT_PROGRESS_FLUSH_INTERVAL_MS = 60 * 1000;
export const ACTIVE_RUN_LOG_RUNTIME_STATUS_REFRESH_INTERVAL_MS = 5 * 1000;
type Run = typeof heartbeatRuns.$inferSelect;
export interface HeartbeatRunOutputInput {
run: Run;
runId: string;
issueId: string | null;
runLogStore: Pick<RunLogStore, "begin" | "append">;
identityRedactor: Pick<ReturnType<typeof createAgentIdentityRedactor>, "chunk" | "redact">;
secretKeys: Set<string>;
appendRunEvent: ReturnType<typeof createHeartbeatLifecycle>["appendRunEvent"];
getCurrentUserRedactionOptions: () => Promise<CurrentUserRedactionOptions>;
}
/** One recorder per execution attempt; the log store itself stays service-scoped. */
export function createHeartbeatRunOutput(db: Db, input: HeartbeatRunOutputInput) {
const { run, runId, issueId, runLogStore, identityRedactor, secretKeys, appendRunEvent, getCurrentUserRedactionOptions } = input;
let handle: RunLogHandle | null = null;
let stdoutExcerpt = "";
let stderrExcerpt = "";
let outputSeq = Number(run.lastOutputSeq ?? 0);
let lastOutputFlushAt: Date | null = run.lastOutputAt ?? null;
let lastLogRuntimeStatusTouchMs = 0;
const outputProgressState: {
pending: {
at: Date;
seq: number;
stream: "stdout" | "stderr";
bytes: number;
} | null;
} = { pending: null };
let persistedLogBytes = Number(run.logBytes ?? 0);
const flushOutputProgress = async (opts?: { force?: boolean }) => {
const pendingOutputProgress = outputProgressState.pending;
if (!pendingOutputProgress) return;
const shouldFlush =
opts?.force === true ||
!lastOutputFlushAt ||
pendingOutputProgress.at.getTime() - lastOutputFlushAt.getTime() >=
ACTIVE_RUN_OUTPUT_PROGRESS_FLUSH_INTERVAL_MS;
if (!shouldFlush) return;
await db
.update(heartbeatRuns)
.set({
lastOutputAt: pendingOutputProgress.at,
lastOutputSeq: pendingOutputProgress.seq,
lastOutputStream: pendingOutputProgress.stream,
lastOutputBytes: pendingOutputProgress.bytes,
updatedAt: new Date(),
})
.where(eq(heartbeatRuns.id, run.id));
lastOutputFlushAt = pendingOutputProgress.at;
outputProgressState.pending = null;
};
// Bind handlers to the run row persisted at execution start. Progress writes
// above use only the immutable run ID from the initial row.
async function begin(currentRun: Run) {
const run = currentRun;
handle = await runLogStore.begin({
companyId: run.companyId,
agentId: run.agentId,
runId,
});
await db
.update(heartbeatRuns)
.set({
logStore: handle.store,
logRef: handle.logRef,
updatedAt: new Date(),
})
.where(eq(heartbeatRuns.id, runId));
const currentUserRedactionOptions =
await getCurrentUserRedactionOptions();
const appendIdentityRedactedLog = async (stream: "stdout" | "stderr", chunk: string) => {
const sanitizedChunk = compactRunLogChunk(
redactCurrentUserText(chunk, currentUserRedactionOptions),
);
if (stream === "stdout")
stdoutExcerpt = appendExcerpt(stdoutExcerpt, sanitizedChunk);
if (stream === "stderr")
stderrExcerpt = appendExcerpt(stderrExcerpt, sanitizedChunk);
const ts = new Date().toISOString();
outputSeq += 1;
const chunkSeq = outputSeq;
let appendedBytes = 0;
if (handle) {
appendedBytes = await runLogStore.append(handle, {
stream,
chunk: sanitizedChunk,
ts,
seq: chunkSeq,
});
persistedLogBytes += appendedBytes;
}
outputProgressState.pending = {
at: new Date(ts),
seq: chunkSeq,
stream,
bytes: persistedLogBytes,
};
await flushOutputProgress();
// Streamed CLI output is real run activity: keep the in-memory
// runtime status ("Working... / X ago") fresh between structured
// events so sandbox runs with mid-run log streaming never show a
// minutes-stale timestamp. Throttled to avoid churning the live
// event stream on every 250ms tail chunk.
const logActivityAt = new Date(ts);
if (
isHeartbeatRunRuntimeStatusActive(run.status) &&
logActivityAt.getTime() - lastLogRuntimeStatusTouchMs >=
ACTIVE_RUN_LOG_RUNTIME_STATUS_REFRESH_INTERVAL_MS
) {
lastLogRuntimeStatusTouchMs = logActivityAt.getTime();
const touchedStatus = touchHeartbeatRunRuntimeStatus({
companyId: run.companyId,
issueId,
agentId: run.agentId,
runId: run.id,
at: logActivityAt,
});
if (touchedStatus)
publishHeartbeatRunRuntimeProgress(touchedStatus);
}
const payloadChunk =
sanitizedChunk.length > MAX_LIVE_LOG_CHUNK_BYTES
? sanitizedChunk.slice(
sanitizedChunk.length - MAX_LIVE_LOG_CHUNK_BYTES,
)
: sanitizedChunk;
publishLiveEvent({
companyId: run.companyId,
type: "heartbeat.run.log",
payload: {
runId: run.id,
agentId: run.agentId,
issueId,
ts,
seq: chunkSeq,
stream,
chunk: payloadChunk,
truncated: payloadChunk.length !== sanitizedChunk.length,
},
});
};
const onLog = (stream: "stdout" | "stderr", chunk: string) =>
appendIdentityRedactedLog(stream, identityRedactor.chunk(stream, chunk));
const onAdapterMeta = async (meta: AdapterInvocationMeta) => {
meta = identityRedactor.redact(meta);
if (meta.env && secretKeys.size > 0) {
for (const key of secretKeys) {
if (key in meta.env) meta.env[key] = "***REDACTED***";
}
}
await appendRunEvent(currentRun, {
eventType: "adapter.invoke",
stream: "system",
level: "info",
message: "adapter invocation",
payload: meta as unknown as Record<string, unknown>,
});
};
const onAdapterEvent = async (event: AdapterRuntimeEvent) => {
event = identityRedactor.redact(event);
const eventType = event.eventType.trim();
if (!eventType) return;
await appendRunEvent(currentRun, {
eventType: eventType.slice(0, 120),
stream: event.stream,
level: event.level,
color: event.color,
message: event.message,
payload: event.payload,
});
};
return { currentUserRedactionOptions, appendIdentityRedactedLog, onLog, onAdapterMeta, onAdapterEvent };
}
return {
begin,
get handle() { return handle; },
outputProgressState,
flushOutputProgress,
// Finalization awaits I/O and can receive more output before persisting excerpts.
get stdoutExcerpt() { return stdoutExcerpt; },
get stderrExcerpt() { return stderrExcerpt; },
};
}
@@ -0,0 +1,212 @@
import { eq } from "drizzle-orm";
import { heartbeatRuns, type Db } from "@paperclipai/db";
import type { AdapterExecutionResult } from "@paperclipai/adapter-utils";
import { parseObject } from "../../adapters/utils.js";
import { assertLowTrustRuntimeServicesAllowed } from "../low-trust-runtime-containment.js";
import {
buildWorkspaceReadyComment, buildWorkspaceReadyMetadata, buildWorkspaceReadyPresentation,
ensureRuntimeServicesForRun, persistAdapterManagedRuntimeServices,
type RealizedExecutionWorkspace, type RuntimeServiceRef,
} from "../workspace-runtime.js";
function readNonEmptyString(value: unknown): string | null {
return typeof value === "string" && value.trim().length > 0 ? value : null;
}
type WorkspaceReadyCommentWriter = {
addComment: (
issueId: string,
body: string,
actor: { agentId?: string; userId?: string; runId?: string | null },
options?: {
presentation?: ReturnType<typeof buildWorkspaceReadyPresentation>;
metadata?: ReturnType<typeof buildWorkspaceReadyMetadata>;
},
) => Promise<unknown>;
};
export function postWorkspaceReadyComment(input: {
issuesSvc: WorkspaceReadyCommentWriter;
issueId: string;
agentId: string;
runId: string;
workspace: RealizedExecutionWorkspace;
runtimeServices: RuntimeServiceRef[];
}) {
const workspaceReadyInput = {
workspace: input.workspace,
runtimeServices: input.runtimeServices,
};
return input.issuesSvc.addComment(
input.issueId,
buildWorkspaceReadyComment(workspaceReadyInput),
{ agentId: input.agentId, runId: input.runId },
{
presentation: buildWorkspaceReadyPresentation(workspaceReadyInput),
metadata: buildWorkspaceReadyMetadata(workspaceReadyInput),
},
);
}
type RuntimeInput = Parameters<typeof ensureRuntimeServicesForRun>[0];
interface RuntimeServiceContext {
run: { id: string };
agent: RuntimeInput["agent"] & { id: string; adapterType: string };
issueId: string | null;
issueRef: (NonNullable<RuntimeInput["issue"]> & { executionWorkspaceId: string | null }) | null;
executionWorkspace: RealizedExecutionWorkspace;
context: Record<string, unknown>;
issuesSvc: WorkspaceReadyCommentWriter;
onLog: NonNullable<RuntimeInput["onLog"]>;
}
/** Service processes and leases stay in workspace-runtime; the executor releases them. */
export function prepareHeartbeatRuntimeServiceIntents(input: {
hostExecutionWorkspaceConfig: Record<string, unknown>;
context: Record<string, unknown>;
trustPreset: Parameters<typeof assertLowTrustRuntimeServicesAllowed>[0]["resolution"];
}) {
const { hostExecutionWorkspaceConfig, context, trustPreset } = input;
const runtimeServiceIntents = (() => {
const runtimeConfig = parseObject(
hostExecutionWorkspaceConfig.workspaceRuntime,
);
return Array.isArray(runtimeConfig.services)
? runtimeConfig.services.filter(
(value): value is Record<string, unknown> =>
typeof value === "object" && value !== null,
)
: [];
})();
assertLowTrustRuntimeServicesAllowed({
resolution: trustPreset,
runtimeServiceCount: runtimeServiceIntents.length,
});
if (runtimeServiceIntents.length > 0) {
context.paperclipRuntimeServiceIntents = runtimeServiceIntents;
} else {
delete context.paperclipRuntimeServiceIntents;
}
}
export async function startHeartbeatRuntimeServices(db: Db, input: RuntimeServiceContext & {
persistedExecutionWorkspace: { id: string } | null;
hostExecutionWorkspaceConfig: Record<string, unknown>;
adapterEnv: RuntimeInput["adapterEnv"];
workspaceOperationRecorder: RuntimeInput["recorder"];
}) {
const { run, agent, issueId, issueRef, executionWorkspace, context, issuesSvc, onLog,
persistedExecutionWorkspace, hostExecutionWorkspaceConfig, adapterEnv, workspaceOperationRecorder } = input;
const runtimeServices = await ensureRuntimeServicesForRun({
db,
runId: run.id,
agent: {
id: agent.id,
name: agent.name,
companyId: agent.companyId,
},
issue: issueRef,
workspace: executionWorkspace,
executionWorkspaceId:
persistedExecutionWorkspace?.id ??
issueRef?.executionWorkspaceId ??
null,
config: hostExecutionWorkspaceConfig,
adapterEnv,
onLog,
recorder: workspaceOperationRecorder,
});
if (runtimeServices.length > 0) {
context.paperclipRuntimeServices = runtimeServices;
context.paperclipRuntimePrimaryUrl =
runtimeServices.find((service) => readNonEmptyString(service.url))
?.url ?? null;
await db
.update(heartbeatRuns)
.set({
contextSnapshot: context,
updatedAt: new Date(),
})
.where(eq(heartbeatRuns.id, run.id));
}
if (
issueId &&
(executionWorkspace.created ||
runtimeServices.some((service) => !service.reused))
) {
try {
await postWorkspaceReadyComment({
issuesSvc,
issueId,
agentId: agent.id,
runId: run.id,
workspace: executionWorkspace,
runtimeServices,
});
} catch (err) {
await onLog(
"stderr",
`[paperclip] Failed to post workspace-ready comment: ${err instanceof Error ? err.message : String(err)}\n`,
);
}
}
return runtimeServices;
}
export async function recordHeartbeatAdapterRuntimeServices(db: Db, input: RuntimeServiceContext & {
adapterResult: Pick<AdapterExecutionResult, "runtimeServices">;
runtimeServices: RuntimeServiceRef[];
}) {
const { run, agent, issueId, issueRef, executionWorkspace, context, issuesSvc, onLog,
adapterResult, runtimeServices } = input;
const adapterManagedRuntimeServices = adapterResult.runtimeServices
? await persistAdapterManagedRuntimeServices({
db,
adapterType: agent.adapterType,
runId: run.id,
agent: {
id: agent.id,
name: agent.name,
companyId: agent.companyId,
},
issue: issueRef,
workspace: executionWorkspace,
reports: adapterResult.runtimeServices,
})
: [];
if (adapterManagedRuntimeServices.length > 0) {
const combinedRuntimeServices = [
...runtimeServices,
...adapterManagedRuntimeServices,
];
context.paperclipRuntimeServices = combinedRuntimeServices;
context.paperclipRuntimePrimaryUrl =
combinedRuntimeServices.find((service) =>
readNonEmptyString(service.url),
)?.url ?? null;
await db
.update(heartbeatRuns)
.set({
contextSnapshot: context,
updatedAt: new Date(),
})
.where(eq(heartbeatRuns.id, run.id));
if (issueId) {
try {
await postWorkspaceReadyComment({
issuesSvc,
issueId,
agentId: agent.id,
runId: run.id,
workspace: executionWorkspace,
runtimeServices: adapterManagedRuntimeServices,
});
} catch (err) {
await onLog(
"stderr",
`[paperclip] Failed to post adapter-managed runtime comment: ${err instanceof Error ? err.message : String(err)}\n`,
);
}
}
}
}
@@ -0,0 +1,385 @@
import { and, desc, eq } from "drizzle-orm";
import { agentConfigRevisions, type Db, type agents, type agentTaskSessions, type heartbeatRuns, type issues, type projects } from "@paperclipai/db";
import type { AiConnectionBinding, AiRuntimeConnectionBinding } from "@paperclipai/shared";
import { getServerAdapter } from "../../adapters/index.js";
import type { prepareManagedAiRuntime } from "../ai-connection-runtime.js";
import { aiConnectionSessionCompatibilityInputs, managedAiSessionIdentityCompatible } from "../ai-connection-session.js";
import { createNativeSessionHandoffLoader } from "../native-runtime/native-session-handoff.js";
import {
describeSessionResetReason, shouldResetTaskSessionForWake, isCanonicalSessionIdForAdapter,
normalizeResumeParamsForAdapter, normalizeSessionParams, requiresCanonicalSessionIds,
truncateDisplayId, type createHeartbeatRunState, type getAdapterSessionCodec,
} from "./run-state.js";
import {
buildEffectiveRunSessionConfigMetadata, readConfigFingerprintFromSessionParams,
readConfiguredModelFromAdapterConfig, resolveTaskSessionConfigFreshness,
stripPaperclipSessionMetadataFromSessionParams, type EffectiveRunSessionConfigMetadata,
} from "./workspaces.js";
import type { HeartbeatWorkspacePreparationInput, prepareHeartbeatWorkspace } from "./workspace-preparation.js";
import type { prepareHeartbeatTaskContext } from "./task-context.js";
type RunState = ReturnType<typeof createHeartbeatRunState>;
type WorkspaceInput = HeartbeatWorkspacePreparationInput;
type PreparedWorkspace = Awaited<ReturnType<typeof prepareHeartbeatWorkspace>>;
type TaskContext = Awaited<ReturnType<typeof prepareHeartbeatTaskContext>>;
type SessionMetadataInput = Parameters<typeof buildEffectiveRunSessionConfigMetadata>[0];
type ManagedAiRuntime = Awaited<ReturnType<typeof prepareManagedAiRuntime>>;
type TaskSession = typeof agentTaskSessions.$inferSelect | null;
type SessionCodec = ReturnType<typeof getAdapterSessionCodec>;
export interface HeartbeatSessionPreparationInput {
run: Pick<typeof heartbeatRuns.$inferSelect, "createdAt">;
agent: typeof agents.$inferSelect;
task: {
context: Record<string, unknown>;
issueRef: Pick<typeof issues.$inferSelect, "id"> | null;
issueContext: Pick<typeof issues.$inferSelect, "updatedAt"> | null;
projectContext: Pick<typeof projects.$inferSelect, "updatedAt" | "env"> | null;
wakeCommentId: string | null;
acceptedPlanContinuationWake: boolean;
acceptedPlanWakeRoutingDecision: unknown;
};
session: {
taskSession: TaskSession;
taskSessionDecodedParams: Record<string, unknown> | null;
explicitResumeSessionParams: Record<string, unknown> | null;
explicitResumeSessionDisplayId: string | null;
sessionCodec: SessionCodec;
};
config: {
runtimeConfig: Record<string, unknown>;
agentIdentity: { keyId: string } | undefined;
issueAssigneeOverrides: unknown;
originalAiIssueOverrides: unknown;
managedAiRuntime: ManagedAiRuntime | undefined;
aiBinding: AiConnectionBinding | undefined;
requestedAiBinding: AiRuntimeConnectionBinding | undefined;
secretManifest: SessionMetadataInput["secretManifest"];
runtimeSkillEntries: SessionMetadataInput["runtimeSkills"];
routineEnvContext: { env: unknown };
};
workspace: Pick<WorkspaceInput["policy"], "requestedExecutionWorkspaceMode" | "effectiveExecutionWorkspaceMode" | "projectExecutionWorkspacePolicy" | "issueExecutionWorkspaceSettings" | "executionPolicy"> & {
requestedReusableExecutionWorkspaceConfig: NonNullable<WorkspaceInput["reuse"]["reusableExistingExecutionWorkspace"]>["config"] | null;
reusableExistingExecutionWorkspace: WorkspaceInput["reuse"]["reusableExistingExecutionWorkspace"];
};
environment: Pick<WorkspaceInput["environment"], "environmentResolution" | "selectedEnvironmentId" | "selectedEnvironmentForConfig">;
getLatestAgentConfigRevision: RunState["getLatestAgentConfigRevision"];
}
function readNonEmptyString(value: unknown): string | null {
return typeof value === "string" && value.trim().length > 0 ? value : null;
}
/** Selects a resumable session before workspace preparation; acquires no runtime resources. */
export async function prepareHeartbeatSession(db: Db, input: HeartbeatSessionPreparationInput) {
const { run, agent, getLatestAgentConfigRevision } = input;
const { context, issueRef, issueContext, projectContext, wakeCommentId, acceptedPlanContinuationWake, acceptedPlanWakeRoutingDecision } = input.task;
const { taskSession, taskSessionDecodedParams, explicitResumeSessionParams, explicitResumeSessionDisplayId, sessionCodec } = input.session;
const { runtimeConfig, agentIdentity, issueAssigneeOverrides, originalAiIssueOverrides, managedAiRuntime, aiBinding, requestedAiBinding, secretManifest, runtimeSkillEntries, routineEnvContext } = input.config;
const { requestedExecutionWorkspaceMode, effectiveExecutionWorkspaceMode, projectExecutionWorkspacePolicy, issueExecutionWorkspaceSettings, executionPolicy, requestedReusableExecutionWorkspaceConfig, reusableExistingExecutionWorkspace } = input.workspace;
const { environmentResolution, selectedEnvironmentId, selectedEnvironmentForConfig } = input.environment;
const latestAgentConfigRevision = await getLatestAgentConfigRevision(
agent.companyId,
agent.id,
);
const sessionConfigMetadataInput = {
agentIdentityKeyId: agentIdentity?.keyId,
adapterType: agent.adapterType,
effectiveAdapterConfig: runtimeConfig,
managedAiHome: managedAiRuntime?.home,
agentRuntimeConfig: agent.runtimeConfig,
issueOverrides: issueAssigneeOverrides,
workspaceConfig: {
requestedMode: requestedExecutionWorkspaceMode,
effectiveMode: effectiveExecutionWorkspaceMode,
issueConfigRevisionAt:
issueContext?.updatedAt instanceof Date
? issueContext.updatedAt.toISOString()
: (issueContext?.updatedAt ?? null),
projectConfigRevisionAt:
projectContext?.updatedAt instanceof Date
? projectContext.updatedAt.toISOString()
: (projectContext?.updatedAt ?? null),
projectPolicy: projectExecutionWorkspacePolicy,
issueSettings: issueExecutionWorkspaceSettings,
reusableExecutionWorkspaceConfig:
requestedReusableExecutionWorkspaceConfig,
existingExecutionWorkspace: reusableExistingExecutionWorkspace
? {
id: reusableExistingExecutionWorkspace.id,
mode: reusableExistingExecutionWorkspace.mode,
strategyType: reusableExistingExecutionWorkspace.strategyType,
projectWorkspaceId:
reusableExistingExecutionWorkspace.projectWorkspaceId,
repoUrl: reusableExistingExecutionWorkspace.repoUrl,
baseRef: reusableExistingExecutionWorkspace.baseRef,
branchName: reusableExistingExecutionWorkspace.branchName,
config: reusableExistingExecutionWorkspace.config,
}
: null,
},
environment: {
selectionSource: environmentResolution.source,
selectedEnvironmentId,
selectedEnvironment: selectedEnvironmentForConfig
? {
id: selectedEnvironmentForConfig.id,
driver: selectedEnvironmentForConfig.driver,
config: selectedEnvironmentForConfig.config,
configRevisionAt:
selectedEnvironmentForConfig.updatedAt instanceof Date
? selectedEnvironmentForConfig.updatedAt.toISOString()
: (selectedEnvironmentForConfig.updatedAt ?? null),
}
: null,
executionPolicy,
},
environmentEnv: selectedEnvironmentForConfig?.envVars ?? null,
projectEnv: projectContext?.env ?? null,
routineEnv: routineEnvContext.env,
secretManifest,
runtimeSkills: runtimeSkillEntries,
agentConfigRevision: latestAgentConfigRevision
? {
id: latestAgentConfigRevision.id,
changedKeys: latestAgentConfigRevision.changedKeys,
configRevisionAt:
latestAgentConfigRevision.createdAt.toISOString(),
}
: null,
};
const sessionConfigMetadata = await buildEffectiveRunSessionConfigMetadata(sessionConfigMetadataInput);
let compatibleConfigMetadata: EffectiveRunSessionConfigMetadata[] = [];
if (managedAiRuntime && aiBinding && taskSession &&
readConfigFingerprintFromSessionParams(taskSession.sessionParamsJson)?.fingerprint !== sessionConfigMetadata.fingerprint) {
const revisions = requestedAiBinding?.mode === "router"
? await db.select().from(agentConfigRevisions).where(and(eq(agentConfigRevisions.companyId, agent.companyId), eq(agentConfigRevisions.agentId, agent.id))).orderBy(desc(agentConfigRevisions.createdAt), desc(agentConfigRevisions.id)).limit(21)
: [];
const candidates = aiConnectionSessionCompatibilityInputs({
effectiveAdapterConfig: runtimeConfig, agentRuntimeConfig: agent.runtimeConfig,
agentConfigRevision: sessionConfigMetadataInput.agentConfigRevision,
issueOverrides: issueAssigneeOverrides, originalIssueOverrides: originalAiIssueOverrides,
binding: aiBinding, router: requestedAiBinding?.mode === "router",
storedIdentity: taskSession.sessionParamsJson?.paperclipAiCredentialIdentity,
sessionIdentity: managedAiRuntime.sessionIdentity, credentialIdentity: managedAiRuntime.identity, revisions,
});
compatibleConfigMetadata = await Promise.all(candidates.map(candidate => buildEffectiveRunSessionConfigMetadata({ ...sessionConfigMetadataInput, ...candidate })));
}
const configuredModel =
readConfiguredModelFromAdapterConfig(runtimeConfig);
if (context.refreshTools === true && agent.adapterType !== "paperclip_runner") {
const capability = getServerAdapter(agent.adapterType).supportsToolRefreshOnResume;
const canRefresh = typeof capability === "function" ? capability(runtimeConfig) : capability === true;
if (!canRefresh) context.forceFreshSession = true;
}
const wakeSessionResetReason = describeSessionResetReason(context);
const sessionConfigFreshness = resolveTaskSessionConfigFreshness({
hasTaskSession: taskSession != null,
configuredModel,
taskSessionParams:
taskSession?.sessionParamsJson ?? taskSessionDecodedParams,
configMetadata: sessionConfigMetadata,
compatibleConfigMetadata,
wakeResetReason: wakeSessionResetReason,
preserveLegacySessionWithoutConfigMetadata:
acceptedPlanContinuationWake && !acceptedPlanWakeRoutingDecision && !agentIdentity,
});
const resetTaskSession =
shouldResetTaskSessionForWake(context) || sessionConfigFreshness.reset;
const sessionResetReason =
sessionConfigFreshness.reasons.join("; ") || null;
const taskSessionForRun = resetTaskSession ? null : taskSession;
const getFreshSessionHandoff = issueRef ? createNativeSessionHandoffLoader({
db, companyId: agent.companyId, issueId: issueRef.id, agentId: agent.id, before: run.createdAt,
throughCommentId: readNonEmptyString(context.conversationReplayThroughCommentId) ?? (context.interactionKind ? null : wakeCommentId),
}) : undefined;
const previousSessionParams =
explicitResumeSessionParams ??
(isCanonicalSessionIdForAdapter(
agent.adapterType,
explicitResumeSessionDisplayId,
)
? { sessionId: explicitResumeSessionDisplayId }
: null) ??
normalizeResumeParamsForAdapter(
agent.adapterType,
stripPaperclipSessionMetadataFromSessionParams(
sessionCodec.deserialize(
taskSessionForRun?.sessionParamsJson ?? null,
),
),
);
// Legacy plugins can consume the existing context field on a known-fresh
// dispatch. Built-ins also load lazily if their resume attempt fails.
if (agent.adapterType !== "paperclip_runner" && !previousSessionParams && getFreshSessionHandoff) {
const handoff = await getFreshSessionHandoff();
if (handoff) context.paperclipFreshSessionHandoffMarkdown = handoff;
}
return { sessionConfigMetadata, configuredModel, sessionConfigFreshness, resetTaskSession, sessionResetReason, taskSessionForRun, getFreshSessionHandoff, previousSessionParams };
}
export interface HeartbeatAdapterSessionInput {
agent: typeof agents.$inferSelect;
task: {
context: Record<string, unknown>;
taskKey: string | null;
issueId: string | null;
continuationSummary: TaskContext["continuationSummary"];
executionContinuation: TaskContext["executionContinuation"];
};
session: {
runtime: { sessionId: string | null };
taskSession: TaskSession;
sessionCodec: SessionCodec;
explicitResumeSessionDisplayId: string | null;
runtimeSessionParams: Record<string, unknown> | null;
managedAiRuntime: ManagedAiRuntime | undefined;
prepared: Awaited<ReturnType<typeof prepareHeartbeatSession>>;
};
workspace: Pick<PreparedWorkspace, "latestWorkspaceConfigMetadata" | "workspaceConfigFreshness" | "reusedExecutionWorkspace" | "resolvedWorkspaceReusePolicy" | "persistedExecutionWorkspace"> & {
runtimeWorkspaceWarnings: string[];
requestedShouldReuseExisting: boolean;
workspaceReuseRequest: WorkspaceInput["reuse"]["workspaceReuseRequest"];
};
evaluateSessionCompaction: RunState["evaluateSessionCompaction"];
}
/** Resolves adapter identity after workspace preparation, retaining in-place context and delta updates. */
export async function resolveHeartbeatAdapterSession(input: HeartbeatAdapterSessionInput) {
const { agent, evaluateSessionCompaction } = input;
const { context, taskKey, issueId, continuationSummary, executionContinuation } = input.task;
const { runtime, taskSession, sessionCodec, explicitResumeSessionDisplayId, runtimeSessionParams, managedAiRuntime, prepared } = input.session;
const { sessionConfigMetadata, sessionConfigFreshness, resetTaskSession, taskSessionForRun, previousSessionParams } = prepared;
const { latestWorkspaceConfigMetadata, workspaceConfigFreshness, reusedExecutionWorkspace, resolvedWorkspaceReusePolicy, persistedExecutionWorkspace, runtimeWorkspaceWarnings, requestedShouldReuseExisting, workspaceReuseRequest } = input.workspace;
const runtimeSessionFallback =
taskKey || resetTaskSession
? null
: isCanonicalSessionIdForAdapter(agent.adapterType, runtime.sessionId)
? runtime.sessionId
: null;
const runtimeSessionDisplayId = truncateDisplayId(
explicitResumeSessionDisplayId ??
taskSessionForRun?.sessionDisplayId ??
(sessionCodec.getDisplayId
? sessionCodec.getDisplayId(runtimeSessionParams)
: null) ??
readNonEmptyString(runtimeSessionParams?.sessionId) ??
runtimeSessionFallback,
);
let previousSessionDisplayId = requiresCanonicalSessionIds(
agent.adapterType,
)
? truncateDisplayId(
readNonEmptyString(previousSessionParams?.sessionId) ??
(isCanonicalSessionIdForAdapter(
agent.adapterType,
runtimeSessionDisplayId,
)
? runtimeSessionDisplayId
: null) ??
runtimeSessionFallback,
)
: runtimeSessionDisplayId;
let runtimeSessionIdForAdapter =
readNonEmptyString(runtimeSessionParams?.sessionId) ??
runtimeSessionFallback;
let runtimeSessionParamsForAdapter = normalizeSessionParams(
stripPaperclipSessionMetadataFromSessionParams(runtimeSessionParams),
);
const sessionCompaction = await evaluateSessionCompaction({
agent,
sessionId: previousSessionDisplayId ?? runtimeSessionIdForAdapter,
issueId,
continuationSummaryBody: continuationSummary?.body ?? null,
});
if (sessionCompaction.rotate) {
context.paperclipSessionHandoffMarkdown =
sessionCompaction.handoffMarkdown;
context.paperclipSessionRotationReason = sessionCompaction.reason;
context.paperclipPreviousSessionId =
previousSessionDisplayId ?? runtimeSessionIdForAdapter;
runtimeSessionIdForAdapter = null;
runtimeSessionParamsForAdapter = null;
previousSessionDisplayId = null;
if (sessionCompaction.reason) {
runtimeWorkspaceWarnings.push(
`Starting a fresh session because ${sessionCompaction.reason}.`,
);
}
} else {
delete context.paperclipSessionHandoffMarkdown;
delete context.paperclipSessionRotationReason;
delete context.paperclipPreviousSessionId;
}
const taskSessionCredentialCompatible = !managedAiRuntime || managedAiSessionIdentityCompatible(
taskSession?.sessionParamsJson?.paperclipAiCredentialIdentity,
managedAiRuntime.sessionIdentity,
managedAiRuntime.identity,
);
if (managedAiRuntime) {
sessionConfigMetadata.aiCredentialIdentity = managedAiRuntime.sessionIdentity;
if (!taskSessionCredentialCompatible) {
runtimeSessionIdForAdapter = null;
runtimeSessionParamsForAdapter = null;
previousSessionDisplayId = null;
delete executionContinuation?.resumeDelta;
}
}
const runtimeForAdapter = {
sessionId: runtimeSessionIdForAdapter,
sessionParams: runtimeSessionParamsForAdapter,
sessionDisplayId: previousSessionDisplayId,
taskKey,
};
// A delta is safe only when the selected provider session is exactly the
// task session whose last dispatch supplied the baseline history.
if (
executionContinuation?.resumeDelta &&
(!taskSessionForRun ||
!taskSession?.sessionDisplayId ||
runtimeForAdapter.sessionDisplayId !== taskSession.sessionDisplayId ||
resetTaskSession ||
context.forceFreshSession === true)
) {
delete executionContinuation.resumeDelta;
}
const configFreshnessResultMetadata = {
version: sessionConfigMetadata.version,
session: {
fingerprintVersion: sessionConfigMetadata.version,
categories: sessionConfigMetadata.categories,
reset: resetTaskSession,
resetReasons: sessionConfigFreshness.reasons,
changedCategories: sessionConfigFreshness.changedCategories,
taskSessionAvailable: taskSession != null,
taskSessionReused: taskSessionForRun != null,
storedFingerprintPresent: Boolean(
sessionConfigFreshness.storedFingerprint,
),
nextFingerprint: sessionConfigFreshness.nextFingerprint,
},
workspace: {
fingerprintVersion: latestWorkspaceConfigMetadata.version,
categories: latestWorkspaceConfigMetadata.categories,
action: workspaceConfigFreshness.action,
changedCategories: workspaceConfigFreshness.changedCategories,
reasons: workspaceConfigFreshness.reasons,
reuseRequested: requestedShouldReuseExisting,
workspaceReused: Boolean(reusedExecutionWorkspace),
configSnapshotRefreshed:
resolvedWorkspaceReusePolicy.shouldRefreshWorkspaceConfigSnapshot,
storedFingerprintPresent:
workspaceConfigFreshness.storedFingerprintPresent,
storedFingerprint: workspaceConfigFreshness.storedFingerprint,
inferredFingerprint: workspaceConfigFreshness.inferredFingerprint,
nextFingerprint: workspaceConfigFreshness.nextFingerprint,
previousWorkspaceId:
workspaceReuseRequest.requestedExecutionWorkspaceId,
activeWorkspaceId: persistedExecutionWorkspace?.id ?? null,
},
};
return { previousSessionDisplayId, runtimeSessionParamsForAdapter, taskSessionCredentialCompatible, runtimeForAdapter, configFreshnessResultMetadata, sessionCompaction };
}
@@ -0,0 +1,376 @@
import { createHash } from "node:crypto";
import type { Db, agents, agentTaskSessions, heartbeatRuns, issueComments } from "@paperclipai/db";
import { FAST_RESPONSE_AGENT_GUIDANCE, fastResponseHistoryBody } from "@paperclipai/shared";
import type { PaperclipTurnContext } from "@paperclipai/adapter-utils/server-utils";
import { parseObject } from "../../adapters/utils.js";
import { isConversation } from "../agent-conversations.js";
import { chatCompletionInstruction } from "../chat-completion-delivery.js";
import { getConversationConfirmationContext } from "../conversation-confirmation-context.js";
import { buildExecutionContinuation } from "../execution-continuation.js";
import { getIssueContinuationSummaryDocument } from "../issue-continuation-summary.js";
import type { issueService } from "../issues.js";
import { createRunSecretRedactionRegistry } from "../run-secret-redaction.js";
import { redactQuarantinedBodyForHigherTrust, sanitizeQuarantinedCommentForHigherTrust } from "../source-trust.js";
import { getTaskPlanContext } from "../task-plan-context.js";
import type { TrustPresetResolution } from "../trust-preset-resolver.js";
import { liveVoiceExecutionGuidance } from "../voice/voice-execution-guidance.js";
import { buildPaperclipWakePayload, PAPERCLIP_WAKE_PAYLOAD_KEY, type createHeartbeatRunPreparation } from "./run-preparation.js";
import { buildPaperclipTaskMarkdown } from "./task-markdown.js";
type RunPreparation = ReturnType<typeof createHeartbeatRunPreparation>;
type IssueContext = NonNullable<Awaited<ReturnType<RunPreparation["getIssueExecutionContext"]>>>;
export interface HeartbeatTaskContextInput {
run: Pick<typeof heartbeatRuns.$inferSelect, "id">;
agent: Pick<typeof agents.$inferSelect, "id" | "companyId" | "adapterType">;
task: {
issueRef: Pick<IssueContext,
"id" | "identifier" | "title" | "status" | "priority" | "workMode" |
"conversationAgentId" | "description" | "projectId"> | null;
issueContext: IssueContext | null;
wakeCommentContext: Pick<typeof issueComments.$inferSelect,
"id" | "body" | "sourceTrust" | "presentation" | "metadata"> | null;
context: Record<string, unknown>;
taskSession: Pick<typeof agentTaskSessions.$inferSelect, "lastRunId"> | null;
};
policy: {
trustPreset: Pick<TrustPresetResolution, "kind">;
experimentalInstanceSettings: { enableSimplifiedEnglishInteractions?: boolean };
};
services: {
issuesSvc: Pick<ReturnType<typeof issueService>, "getAncestors">;
getPinnedSkillTestContext: RunPreparation["getPinnedSkillTestContext"];
};
}
function readNonEmptyString(value: unknown): string | null {
return typeof value === "string" && value.trim().length > 0 ? value : null;
}
/** Rebuilds task prompt fields on the run's context; acquires no execution resources. */
export async function prepareHeartbeatTaskContext(db: Db, input: HeartbeatTaskContextInput) {
const { run, agent } = input;
const { issueRef, issueContext, wakeCommentContext, context, taskSession } = input.task;
const { trustPreset, experimentalInstanceSettings } = input.policy;
const { issuesSvc, getPinnedSkillTestContext } = input.services;
const continuationSummary = issueRef && !isConversation(issueContext)
? await getIssueContinuationSummaryDocument(db, issueRef.id)
: null;
const exposeLowTrustRaw = trustPreset.kind === "low_trust_review";
const safeContinuationSummary =
continuationSummary && !exposeLowTrustRaw
? redactQuarantinedBodyForHigherTrust(continuationSummary)
: continuationSummary;
const safeWakeCommentContext =
wakeCommentContext && !exposeLowTrustRaw
? sanitizeQuarantinedCommentForHigherTrust(wakeCommentContext)
: wakeCommentContext;
const issueAncestors = issueRef
? await issuesSvc.getAncestors(issueRef.id)
: [];
if (continuationSummary) {
context.paperclipContinuationSummary = {
key: safeContinuationSummary!.key,
title: safeContinuationSummary!.title,
body: safeContinuationSummary!.body,
sourceTrust: safeContinuationSummary!.sourceTrust ?? null,
updatedAt: safeContinuationSummary!.updatedAt.toISOString(),
};
} else {
delete context.paperclipContinuationSummary;
}
const pinnedSkillTestContext =
issueRef?.workMode === "skill_test"
? await getPinnedSkillTestContext(agent.companyId, issueRef.id)
: null;
if (pinnedSkillTestContext) {
context.paperclipSkillTest = {
...pinnedSkillTestContext,
directive:
"Use this pinned file inventory as the exact skill revision under test, regardless of synced runtime skills.",
};
} else {
delete context.paperclipSkillTest;
}
const executionContinuation =
issueRef && !isConversation(issueContext) && issueContext?.assigneeAgentId === agent.id
? await buildExecutionContinuation({
db,
companyId: agent.companyId,
issueId: issueRef.id,
agentId: agent.id,
runId: run.id,
context,
previousContextRunId: taskSession?.lastRunId,
summary: safeContinuationSummary?.body ?? null,
exposeLowTrustRaw,
})
: null;
context.executionContinuation = executionContinuation;
const paperclipWakePayload = await buildPaperclipWakePayload({
db,
companyId: agent.companyId,
agentId: agent.id,
runId: run.id,
contextSnapshot: context,
continuationSummary,
issueSummary: issueRef
? {
id: issueRef.id,
identifier: issueRef.identifier,
title: issueRef.title,
description: issueContext?.description ?? null,
status: issueRef.status,
priority: issueRef.priority,
workMode: issueRef.workMode,
projectId: issueRef.projectId,
executionPolicy: issueContext?.executionPolicy ?? null,
}
: null,
exposeLowTrustRaw,
simplifiedEnglishInteractions:
experimentalInstanceSettings.enableSimplifiedEnglishInteractions ===
true,
});
if (paperclipWakePayload) {
context[PAPERCLIP_WAKE_PAYLOAD_KEY] = paperclipWakePayload;
} else {
delete context[PAPERCLIP_WAKE_PAYLOAD_KEY];
}
const safeWakeComments = (paperclipWakePayload?.comments ?? []).flatMap(
(comment) =>
typeof comment.id === "string" && typeof comment.body === "string"
? [
{
id: comment.id,
body: fastResponseHistoryBody({ body: comment.body, origin: typeof comment.origin === "string" ? comment.origin : null }),
attachments: Array.isArray(comment.attachments)
? comment.attachments.flatMap((attachment) => {
const descriptor = parseObject(attachment);
const id = readNonEmptyString(descriptor.id);
const filename = readNonEmptyString(
descriptor.filename,
);
const contentType = readNonEmptyString(
descriptor.contentType,
);
const contentPath = readNonEmptyString(
descriptor.contentPath,
);
const byteSize = descriptor.byteSize;
return id &&
filename &&
contentType &&
contentPath &&
typeof byteSize === "number"
? [
{
id,
filename,
contentType,
byteSize,
contentPath,
},
]
: [];
})
: [],
},
]
: [],
);
// Always replace caller-supplied context with the immutable, company-scoped
// conversation snapshot. It belongs only to the endpoint's assigned agent.
const liveVoiceGuidance = issueContext?.chatAssignedAgentId === agent.id && issueContext.chatProvider === "speko"
? await liveVoiceExecutionGuidance(db, { companyId: agent.companyId, issueId: issueContext.id, agentId: agent.id })
: null;
context.paperclipTaskCommunicationGuidance =
issueContext?.chatAssignedAgentId === agent.id
? issueContext.chatCommunicationGuidance
: null;
const taskMarkdownInput = {
conversationConfirmations: issueRef && isConversation(issueContext)
? await getConversationConfirmationContext({ db, companyId: agent.companyId, issueId: issueRef.id, agentId: agent.id })
: null,
issue: issueRef
? {
id: issueRef.id,
identifier: issueRef.identifier,
title: issueRef.title,
titleNeedsGeneration: issueContext?.titleNeedsGeneration,
workMode: issueRef.workMode,
conversationAgentId: issueContext?.conversationAgentId,
description: issueRef.description,
}
: null,
ancestors: issueAncestors,
wakeComment: safeWakeCommentContext,
wakeComments: safeWakeComments,
attachmentOmissions: paperclipWakePayload?.attachmentOmissions,
externalChatProvider: paperclipWakePayload?.externalChatProvider,
slackCommand: issueContext?.chatAssignedAgentId === agent.id
? issueContext.chatSlackCommand
: null,
nativeRunner: agent.adapterType === "paperclip_runner",
interaction: {
kind: readNonEmptyString(context.interactionKind),
status: readNonEmptyString(context.interactionStatus),
},
planReview: paperclipWakePayload?.planReviewContext?.interaction
? {
status: paperclipWakePayload.planReviewContext.interaction.status,
reason: paperclipWakePayload.planReviewContext.interaction.result?.reason,
}
: null,
acceptedPlanContinuation:
readNonEmptyString(context.workspaceRefreshReason) ===
"accepted_plan_confirmation" &&
Object.keys(parseObject(context.acceptedPlanWakeRouting)).length ===
0,
acceptedPlan: (() => {
const accepted = parseObject(
parseObject(context.planReviewInteraction).acceptedTargetRevision,
);
const revisionId = readNonEmptyString(accepted.revisionId);
if (!revisionId) return null;
return {
documentId: readNonEmptyString(accepted.documentId),
revisionId,
revisionNumber:
typeof accepted.revisionNumber === "number"
? accepted.revisionNumber
: null,
};
})(),
};
const taskPlan = issueRef && !isConversation(issueContext)
? await getTaskPlanContext({
db,
companyId: agent.companyId,
issueId: issueRef.id,
approvedRevisionId: taskMarkdownInput.acceptedPlan?.revisionId,
exposeLowTrustRaw,
})
: null;
// Unlike the frozen initial communication preferences, live-call context
// must also reach resumed turns. Rebuild it from server records each wake.
const liveVoiceTaskPrefix = liveVoiceGuidance ? `${liveVoiceGuidance}\n\n` : "";
let taskMarkdown = liveVoiceTaskPrefix + buildPaperclipTaskMarkdown({ ...taskMarkdownInput, taskPlan }) + chatCompletionInstruction(context);
let taskMarkdownAssignment = liveVoiceTaskPrefix + buildPaperclipTaskMarkdown({
...taskMarkdownInput,
taskPlan,
includeWakeComments: false,
}) + chatCompletionInstruction(context);
const taskMarkdownCompact = liveVoiceTaskPrefix + buildPaperclipTaskMarkdown({
...taskMarkdownInput,
taskPlan,
includeDescription: false,
}) + chatCompletionInstruction(context);
const taskMarkdownAssignmentCompact = liveVoiceTaskPrefix + buildPaperclipTaskMarkdown({
...taskMarkdownInput,
taskPlan,
includeDescription: false,
includeWakeComments: false,
}) + chatCompletionInstruction(context);
if (issueRef) {
context.paperclipIssue = {
id: issueRef.id,
identifier: issueRef.identifier,
title: issueRef.title,
description: isConversation(issueContext) ? null : issueRef.description,
workMode: issueRef.workMode,
};
} else {
delete context.paperclipIssue;
}
if (wakeCommentContext) {
context.paperclipWakeComment = safeWakeCommentContext;
} else {
delete context.paperclipWakeComment;
}
if (taskMarkdown) {
context.paperclipTaskMarkdown = taskMarkdown;
} else {
delete context.paperclipTaskMarkdown;
}
if (taskMarkdownAssignment) {
context.paperclipTaskMarkdownAssignment = taskMarkdownAssignment;
} else {
delete context.paperclipTaskMarkdownAssignment;
}
if (taskMarkdownCompact && taskMarkdownCompact !== taskMarkdown) {
context.paperclipTaskMarkdownCompact = taskMarkdownCompact;
} else {
delete context.paperclipTaskMarkdownCompact;
}
if (taskMarkdownAssignmentCompact && taskMarkdownAssignmentCompact !== taskMarkdownAssignment) {
context.paperclipTaskMarkdownAssignmentCompact = taskMarkdownAssignmentCompact;
} else {
delete context.paperclipTaskMarkdownAssignmentCompact;
}
context.paperclipTaskCommunicationGuidance = [context.paperclipTaskCommunicationGuidance, FAST_RESPONSE_AGENT_GUIDANCE].filter(Boolean).join("\n\n");
if (issueRef) {
const redactedWakeContext = await createRunSecretRedactionRegistry(
db,
).redactForIssue(agent.companyId, issueRef.id, {
paperclipIssue: context.paperclipIssue,
paperclipWakeComment: context.paperclipWakeComment,
paperclipTaskCommunicationGuidance: context.paperclipTaskCommunicationGuidance,
paperclipTaskMarkdown: context.paperclipTaskMarkdown,
paperclipTaskMarkdownCompact: context.paperclipTaskMarkdownCompact,
paperclipTaskMarkdownAssignment: context.paperclipTaskMarkdownAssignment,
paperclipTaskMarkdownAssignmentCompact: context.paperclipTaskMarkdownAssignmentCompact,
});
context.paperclipIssue = redactedWakeContext.paperclipIssue;
context.paperclipTaskCommunicationGuidance = redactedWakeContext.paperclipTaskCommunicationGuidance;
if (redactedWakeContext.paperclipWakeComment) {
context.paperclipWakeComment =
redactedWakeContext.paperclipWakeComment;
}
if (redactedWakeContext.paperclipTaskMarkdown) {
context.paperclipTaskMarkdown =
redactedWakeContext.paperclipTaskMarkdown;
}
if (redactedWakeContext.paperclipTaskMarkdownCompact) {
context.paperclipTaskMarkdownCompact =
redactedWakeContext.paperclipTaskMarkdownCompact;
}
if (redactedWakeContext.paperclipTaskMarkdownAssignment) {
context.paperclipTaskMarkdownAssignment =
redactedWakeContext.paperclipTaskMarkdownAssignment;
}
if (redactedWakeContext.paperclipTaskMarkdownAssignmentCompact) {
context.paperclipTaskMarkdownAssignmentCompact =
redactedWakeContext.paperclipTaskMarkdownAssignmentCompact;
}
}
if (issueRef) {
const digest = (value: string | null | undefined) =>
value?.trim()
? createHash("sha256")
.update(value.trim().replace(/[\u0000-\u0008\u000b\u000c\u000e-\u001f\u007f]/g, ""))
.digest("hex")
: null;
const redactedIssue = parseObject(context.paperclipIssue);
context.paperclipTurnContext = {
version: 1,
assignment: {
owner: "task_markdown",
description: {
id: issueRef.id,
revision: digest(readNonEmptyString(redactedIssue.description)),
},
},
events: {
owner: "wake_prompt",
comments: safeWakeComments.map((comment) => ({
id: comment.id,
revision: digest(comment.body),
})),
},
} satisfies PaperclipTurnContext;
} else {
delete context.paperclipTurnContext;
}
return { continuationSummary, executionContinuation, paperclipWakePayload, safeWakeComments, safeWakeCommentContext };
}
File diff suppressed because it is too large. Load diff
@@ -79,211 +79,13 @@ const previousRunId = "40000000-0000-4000-8000-000000000004";
const currentRunId = "50000000-0000-4000-8000-000000000005";
const normalizedSessionId = "60000000-0000-4000-8000-000000000006";
const CONDITIONAL_READER_TOOL_CONTRACT_FINGERPRINT = `sha256:${createHash(
"sha256",
)
.update(
JSON.stringify({
schema: "paperclip.native-tool-contract.v1",
tools: [
{ name: "register_deliverable", version: 1 },
{
name: "read_current_wake_comments",
semanticContract: "paperclip.server-current-wake-comments.v1",
version: 1,
},
],
}),
)
.digest("hex")}`;
const PRE_CHAT_ATTACHMENT_REUSE_TOOL_CONTRACT_FINGERPRINT = `sha256:${createHash(
"sha256",
)
.update(
JSON.stringify({
schema: "paperclip.native-tool-contract.v2",
executionTargetKind: "local",
advertisementPolicy: {
readCurrentWakeComments: "always_advertised_binding_gated.v1",
registerDeliverable: "local_workspace_only.v1",
},
tools: [
{ name: "register_deliverable", version: 1 },
{
name: "read_current_wake_comments",
semanticContract: "paperclip.server-current-wake-comments.v1",
version: 1,
},
],
}),
)
.digest("hex")}`;
const PRE_CHAT_ATTACHMENT_READ_TOOL_CONTRACT_FINGERPRINT = `sha256:${createHash(
"sha256",
)
.update(
JSON.stringify({
schema: "paperclip.native-tool-contract.v3",
executionTargetKind: "local",
advertisementPolicy: {
readCurrentWakeComments: "always_advertised_binding_gated.v1",
historicalChatAttachments:
"always_advertised_conversation_binding_gated.v1",
registerDeliverable: "local_workspace_only.v1",
},
tools: [
{ name: "register_deliverable", version: 1 },
{
name: "read_current_wake_comments",
semanticContract: "paperclip.server-current-wake-comments.v1",
version: 1,
},
{
name: "list_chat_attachments",
semanticContract: "paperclip.server-chat-attachment-reuse.v1",
version: 1,
},
{
name: "reuse_chat_attachment",
semanticContract: "paperclip.server-chat-attachment-reuse.v1",
version: 1,
},
],
}),
)
.digest("hex")}`;
const PRE_STRUCTURED_HUMAN_INPUT_TOOL_CONTRACT_FINGERPRINT = `sha256:${createHash(
"sha256",
)
.update(
JSON.stringify({
schema: "paperclip.native-tool-contract.v4",
executionTargetKind: "local",
advertisementPolicy: {
readCurrentWakeComments: "always_advertised_binding_gated.v1",
historicalChatAttachments:
"always_advertised_conversation_binding_gated.v1",
registerDeliverable: "local_workspace_only.v1",
readChatAttachment: "always_advertised_run_scope_local_staging.v1",
},
tools: [
{ name: "register_deliverable", version: 1 },
{
name: "read_current_wake_comments",
semanticContract: "paperclip.server-current-wake-comments.v1",
version: 1,
},
{
name: "list_chat_attachments",
semanticContract: "paperclip.server-chat-attachment-reuse.v1",
version: 1,
},
{
name: "reuse_chat_attachment",
semanticContract: "paperclip.server-chat-attachment-reuse.v1",
version: 1,
},
{
name: "read_chat_attachment",
semanticContract: "paperclip.server-chat-attachment-read.v1",
version: 1,
},
],
}),
)
.digest("hex")}`;
const PRE_RESPONSE_WAKE_YIELD_TOOL_CONTRACT_FINGERPRINT = `sha256:${createHash(
"sha256",
)
.update(
JSON.stringify({
schema: "paperclip.native-tool-contract.v5",
executionTargetKind: "local",
advertisementPolicy: {
readCurrentWakeComments: "always_advertised_binding_gated.v1",
historicalChatAttachments:
"always_advertised_conversation_binding_gated.v1",
registerDeliverable: "local_workspace_only.v1",
readChatAttachment: "always_advertised_run_scope_local_staging.v1",
structuredHumanInput:
"always_advertised_run_issue_agent_binding_gated.v1",
},
tools: [
{ name: "register_deliverable", version: 1 },
{
name: "read_current_wake_comments",
semanticContract: "paperclip.server-current-wake-comments.v1",
version: 1,
},
{
name: "list_chat_attachments",
semanticContract: "paperclip.server-chat-attachment-reuse.v1",
version: 1,
},
{
name: "reuse_chat_attachment",
semanticContract: "paperclip.server-chat-attachment-reuse.v1",
version: 1,
},
{
name: "read_chat_attachment",
semanticContract: "paperclip.server-chat-attachment-read.v1",
version: 1,
},
{ name: "request_human_input", version: 1 },
],
}),
)
.digest("hex")}`;
const PRE_EXPLICIT_CHAT_ATTACHMENT_GUIDANCE_TOOL_CONTRACT_FINGERPRINT = `sha256:${createHash(
"sha256",
)
.update(
JSON.stringify({
schema: "paperclip.native-tool-contract.v7",
executionTargetKind: "local",
advertisementPolicy: {
readCurrentWakeComments: "always_advertised_binding_gated.v1",
historicalChatAttachments:
"always_advertised_conversation_binding_gated.v1",
registerDeliverable: "local_workspace_only.v1",
readChatAttachment: "always_advertised_run_scope_local_staging.v1",
structuredHumanInput:
"always_advertised_run_issue_agent_binding_gated.v1",
semanticCompletion: "finish_response_wake_concrete_object.v2",
},
tools: [
{ name: "register_deliverable", version: 1 },
{
name: "read_current_wake_comments",
semanticContract: "paperclip.server-current-wake-comments.v1",
version: 1,
},
{
name: "list_chat_attachments",
semanticContract: "paperclip.server-chat-attachment-reuse.v1",
version: 1,
},
{
name: "reuse_chat_attachment",
semanticContract: "paperclip.server-chat-attachment-reuse.v1",
version: 1,
},
{
name: "read_chat_attachment",
semanticContract: "paperclip.server-chat-attachment-read.v1",
version: 1,
},
{ name: "request_human_input", version: 1 },
],
}),
)
.digest("hex")}`;
function execution(
runId: string,
@@ -440,6 +242,8 @@ describe("capability-gated connection tool refresh", () => {
current.runtimeContext.instructions.bundle.digest = "b".repeat(64);
current.runtimeContext.aggregateDigest = canonicalNativeRuntimeContextDigest(current.runtimeContext);
expect(rebindNativeSessionCheckpoint({ previousRun: previousRun(), currentExecution: current, toolRefreshOnResume: true, refreshTools: true })).toBeNull();
current.runtimeContext = nativeRuntimeContextFixture();
expect(rebindNativeSessionCheckpoint({ previousRun: previousRun(), currentExecution: current, toolRefreshOnResume: true, refreshTools: true })).not.toBeNull();
current.binding.agentId = randomUUID();
expect(rebindNativeSessionCheckpoint({ previousRun: previousRun(), currentExecution: current, toolRefreshOnResume: true, refreshTools: true })).toBeNull();
});
@@ -743,6 +547,13 @@ const recoveryFakeCodex = resolve(
runnerDiagnostics.push(chunk.slice(-4_096));
if (runnerDiagnostics.length > 32) runnerDiagnostics.shift();
};
// The production resolver prefers installed Codex over PATH. Explicitly
// bind this disposable fixture so it can never invoke a real provider.
const codexCommandResolver = await vi.importActual<{ resolveCodexCommand: () => string }>(
"../../../../packages/paperclip-runner/src/drivers/codex/codex-command.js",
);
const resolveFixtureCodex = vi.spyOn(codexCommandResolver, "resolveCodexCommand")
.mockReturnValue(join(bin, "codex"));
process.env.PAPERCLIP_RUNNER_STATE_DIR = stateBase;
try {
await Promise.all(
@@ -1227,6 +1038,7 @@ const recoveryFakeCodex = resolve(
Promise.resolve().then(() => session.close({ reason: "Recovery fixture cleanup" })),
));
const closeFailures = closed.filter((result) => result.status === "rejected");
resolveFixtureCodex.mockRestore();
runnerPrpWebSocketInternals.resetForTests();
server.closeAllConnections();
await new Promise<void>((done) => server.close(() => done()));
@@ -1910,6 +1722,12 @@ describe("rebindNativeSessionCheckpoint", () => {
});
it.each([
{ contract: "conditional current-wake reader", retainedFingerprint: "sha256:fe7a2e17ca825228ea2f51f23acdc82db65c91da5ec6add330fefa39b65e1c85" },
{ contract: "same-conversation attachment reuse", retainedFingerprint: "sha256:2b84f2105c2cea10e0e5f37cf4a6ba9bf7fec90d8c8c19c655a0e21d3a8dab5f" },
{ contract: "historical file inspection", retainedFingerprint: "sha256:9b558a575dfa4e3dfaf97b7139b47d1676ab3e0307953ec85f2af0b3f443567c" },
{ contract: "structured human questions", retainedFingerprint: "sha256:1c3cd7e3223f6ed28e1c44a0ba3c961dd4c0c52e52c9c1100a4e41a83384259c" },
{ contract: "response-wake completion yield", retainedFingerprint: "sha256:09887bbf1f6a3674fcf529d872fe2853cc1d6554376019dd1a9b8d607d7eb7fb" },
{ contract: "explicit chat-attachment invocation guidance", retainedFingerprint: "sha256:d797c8bc1baef8407a8c80f2c4b28da63b2a4258a47cf52ee9b1fca45535f5e7" },
{
contract: "completion descriptions",
// Deployed v8 / finish_response_wake_concrete_object.v2 local catalog.
@@ -1942,7 +1760,13 @@ describe("rebindNativeSessionCheckpoint", () => {
},
])(
"refreshes retained $contract without changing the Paperclip task or prior history",
({ retainedFingerprint }) => {
({ contract, retainedFingerprint }) => {
if (contract === "explicit chat-attachment invocation guidance") {
expect(LIST_CHAT_ATTACHMENTS_TOOL_DEFINITION.description).toContain("Call with {}");
expect(LIST_CHAT_ATTACHMENTS_TOOL_DEFINITION.description).toContain("1 through 50");
expect(REUSE_CHAT_ATTACHMENT_TOOL_DEFINITION.description).toContain("All four arguments are required");
expect(REUSE_CHAT_ATTACHMENT_TOOL_DEFINITION.description).toContain("same key unchanged on every retry");
}
expect(NATIVE_TOOL_CONTRACT_FINGERPRINT).not.toBe(retainedFingerprint);
const prior = previousRun({
nativeToolContractFingerprint: retainedFingerprint,
@@ -1993,35 +1817,7 @@ describe("rebindNativeSessionCheckpoint", () => {
},
);
it("rotates a provider thread created while the current-wake reader was conditionally advertised", () => {
expect(CONDITIONAL_READER_TOOL_CONTRACT_FINGERPRINT).not.toBe(
NATIVE_TOOL_CONTRACT_FINGERPRINT,
);
expect(
rebindNativeSessionCheckpoint({
previousRun: previousRun({
nativeToolContractFingerprint:
CONDITIONAL_READER_TOOL_CONTRACT_FINGERPRINT,
}),
currentExecution: execution(currentRunId),
}),
).toBeNull();
});
it("rotates a provider thread that predates same-conversation attachment reuse", () => {
expect(PRE_CHAT_ATTACHMENT_REUSE_TOOL_CONTRACT_FINGERPRINT).not.toBe(
NATIVE_TOOL_CONTRACT_FINGERPRINT,
);
expect(
rebindNativeSessionCheckpoint({
previousRun: previousRun({
nativeToolContractFingerprint:
PRE_CHAT_ATTACHMENT_REUSE_TOOL_CONTRACT_FINGERPRINT,
}),
currentExecution: execution(currentRunId),
}),
).toBeNull();
});
it("binds the provider tool catalog to the local or remote execution target", () => {
const remoteFingerprint = nativeToolContractFingerprintForTarget("remote");
@@ -2044,77 +1840,10 @@ describe("rebindNativeSessionCheckpoint", () => {
).not.toBeNull();
});
it("rotates provider threads whose retained catalog cannot inspect historical files", () => {
expect(PRE_CHAT_ATTACHMENT_READ_TOOL_CONTRACT_FINGERPRINT).not.toBe(
NATIVE_TOOL_CONTRACT_FINGERPRINT,
);
expect(
rebindNativeSessionCheckpoint({
previousRun: previousRun({
nativeToolContractFingerprint:
PRE_CHAT_ATTACHMENT_READ_TOOL_CONTRACT_FINGERPRINT,
}),
currentExecution: execution(currentRunId),
}),
).toBeNull();
});
it("rotates direct provider threads whose retained catalog cannot ask structured questions", () => {
expect(PRE_STRUCTURED_HUMAN_INPUT_TOOL_CONTRACT_FINGERPRINT).not.toBe(
NATIVE_TOOL_CONTRACT_FINGERPRINT,
);
expect(
rebindNativeSessionCheckpoint({
previousRun: previousRun({
nativeToolContractFingerprint:
PRE_STRUCTURED_HUMAN_INPUT_TOOL_CONTRACT_FINGERPRINT,
}),
currentExecution: execution(currentRunId),
}),
).toBeNull();
});
it("rotates direct provider threads whose retained completion tool cannot yield for a response", () => {
expect(PRE_RESPONSE_WAKE_YIELD_TOOL_CONTRACT_FINGERPRINT).not.toBe(
NATIVE_TOOL_CONTRACT_FINGERPRINT,
);
expect(
rebindNativeSessionCheckpoint({
previousRun: previousRun({
nativeToolContractFingerprint:
PRE_RESPONSE_WAKE_YIELD_TOOL_CONTRACT_FINGERPRINT,
}),
currentExecution: execution(currentRunId),
}),
).toBeNull();
});
it("rotates provider threads whose retained attachment tools lack explicit invocation guidance", () => {
expect(LIST_CHAT_ATTACHMENTS_TOOL_DEFINITION.description).toContain(
"Call with {}",
);
expect(LIST_CHAT_ATTACHMENTS_TOOL_DEFINITION.description).toContain(
"1 through 50",
);
expect(REUSE_CHAT_ATTACHMENT_TOOL_DEFINITION.description).toContain(
"All four arguments are required",
);
expect(REUSE_CHAT_ATTACHMENT_TOOL_DEFINITION.description).toContain(
"same key unchanged on every retry",
);
expect(
PRE_EXPLICIT_CHAT_ATTACHMENT_GUIDANCE_TOOL_CONTRACT_FINGERPRINT,
).not.toBe(NATIVE_TOOL_CONTRACT_FINGERPRINT);
expect(
rebindNativeSessionCheckpoint({
previousRun: previousRun({
nativeToolContractFingerprint:
PRE_EXPLICIT_CHAT_ATTACHMENT_GUIDANCE_TOOL_CONTRACT_FINGERPRINT,
}),
currentExecution: execution(currentRunId),
}),
).toBeNull();
});
it("permits legacy retry rebinding only before the replacement acquired authority", () => {
const source = {
@@ -107,31 +107,7 @@ describe("runner API catalog", () => {
expect(operation.responses).toBeDefined();
expect(operation.authorization.actor).toBeTruthy();
});
it.each(
runnerApiCatalog().filter((operation) => operation.transport === "rest"),
)(
"resolves the catalog route $operationId inside the bound origin",
(operation) => {
const pathParams = Object.fromEntries(
operation.parameters
.filter((parameter) => parameter.in === "path")
.map((parameter) => [
parameter.name,
parameter.name === "companyId" ? context.companyId : "fixture-id",
]),
);
const url = runnerApiUrl(
operation,
{ operationId: operation.operationId, pathParams },
context,
"https://paperclip.test",
);
expect(url.origin).toBe("https://paperclip.test");
expect(url.pathname).not.toContain("{");
expect(operation.responses).toBeDefined();
expect(operation.authorization.actor).toBeTruthy();
},
);
it("ranks natural language, explains dedicated alternatives, and supports exact lookup", () => {
expect(
searchRunnerApi({ query: "create project" }).results.map(
@@ -0,0 +1,50 @@
import path from 'node:path';
import type { CompanySkillVersionFileInventoryEntry, SkillRepositoryPackage } from '@paperclipai/shared';
import { z } from 'zod';
import { assertSkillSnapshotPath, skillFileBytes } from './skill-snapshot.js';
export const REPOSITORY_CONTENT_DIR = '.paperclip-repository';
export const MAX_REPOSITORY_FILES = 9_990;
export const MAX_REPOSITORY_BYTES = 61 * 1024 * 1024;
const manifestSchema = z.object({
version: z.literal(1),
skills: z.array(z.string().min(1).max(4096)).min(1).max(1000),
requirements: z.string().max(4000).optional(),
}).strict();
/** Declarative metadata only: importing never invokes a repository's installer. */
export function inspectRepositoryManifest(files: CompanySkillVersionFileInventoryEntry[], skillPaths: string[]) {
const file = files.find(file => file.path === 'paperclip.skills.json');
if (!file) return {};
const manifest = manifestSchema.parse(JSON.parse(skillFileBytes(file).toString('utf8')));
const declared = [...new Set(manifest.skills.map(assertSkillSnapshotPath))];
if (declared.some(value => !skillPaths.includes(value))) throw new Error('Manifest skills must name existing SKILL.md entrypoints.');
return { skillPaths: declared, requirements: manifest.requirements ?? null };
}
export function repositoryPackageInspection(files: CompanySkillVersionFileInventoryEntry[], skillPaths: string[], warnings: string[], error: string | null): SkillRepositoryPackage {
let manifest: ReturnType<typeof inspectRepositoryManifest> = {};
try { manifest = inspectRepositoryManifest(files, skillPaths); }
catch { warnings = [...warnings, 'paperclip.skills.json — Invalid manifest; skills were discovered automatically.']; }
return { files: files.map(file => ({ path: file.path, kind: file.kind, sizeBytes: skillFileBytes(file).length,
encoding: file.encoding ?? 'utf8', executable: file.executable ?? false })), requirements: null, ...manifest, error, warnings };
}
/** A portable discovery entrypoint directs the agent to the untouched original.
* The full tree travels inside the skill on every existing adapter and sandbox
* transport. Never rewrite authored relative links or use host-only symlinks.
*/
export function repositorySkillProjection(files: CompanySkillVersionFileInventoryEntry[], skillPath: string): CompanySkillVersionFileInventoryEntry[] {
assertSkillSnapshotPath(skillPath);
const skill = files.find(file => file.path === skillPath);
if (!skill || skill.encoding === 'base64') throw new Error('Repository skill entrypoint is missing or not text.');
const frontmatter = skill.content.match(/^---\r?\n[\s\S]*?\r?\n---(?:\r?\n|$)/)?.[0];
if (!frontmatter) throw new Error('Repository skill entrypoint has no frontmatter.');
const target = `${REPOSITORY_CONTENT_DIR}/${skillPath}`;
const wrapper = `${frontmatter.trimEnd()}\n\nRead and follow the complete skill instructions in [${skillPath}](${target.split('/').map(encodeURIComponent).join('/')}). Resolve the original skill's relative file references from its directory inside \`${REPOSITORY_CONTENT_DIR}/\`. That directory contains the pinned repository, including supporting skills and shared files.\n`;
// Host-specific discovery policy belongs beside the generated entrypoint too.
const policyPath = path.posix.join(path.posix.dirname(skillPath), 'agents/openai.yaml');
return [{ path: 'SKILL.md', kind: 'skill', content: wrapper },
...files.filter(file => file.path === policyPath).map(file => ({ ...file, path: 'agents/openai.yaml' })),
...files.map(file => ({ ...file, path: `${REPOSITORY_CONTENT_DIR}/${assertSkillSnapshotPath(file.path)}` }))];
}
@@ -6,12 +6,14 @@ import type { SkillSourceScanProgress } from '@paperclipai/shared';
import { parseGitHubSkillRepositoryUrl } from '@paperclipai/shared';
import { unprocessable } from '../errors.js';
import { buildGitAuthInvocation } from './git-credentials.js';
import { MAX_REPOSITORY_BYTES, MAX_REPOSITORY_FILES } from './skill-repository-package.js';
export type GitSkillTreeEntry = { path: string; type: string; mode: string; sha: string; size?: number };
export interface GitSkillSnapshot {
commitSha: string;
entries: GitSkillTreeEntry[];
readBlob: (entry: GitSkillTreeEntry, signal?: AbortSignal) => Promise<Buffer>;
readBlobs?: (entries: GitSkillTreeEntry[], signal?: AbortSignal) => Promise<Map<string, Buffer>>;
release: () => Promise<void>;
}
export interface GitSkillSnapshotOptions {
@@ -69,12 +71,12 @@ async function directoryBytes(directory: string): Promise<number> {
return size;
}
async function runGit(directory: string, args: string[], token: string, options: GitSkillSnapshotOptions & { download?: boolean; maxOutput?: number } = {}): Promise<Buffer> {
async function runGit(directory: string, args: string[], token: string, options: GitSkillSnapshotOptions & { download?: boolean; maxOutput?: number; input?: string } = {}): Promise<Buffer> {
options.signal?.throwIfAborted();
return new Promise((resolve, reject) => {
const child = spawn('git', ['-c', `core.hooksPath=${os.devNull}`, '-c', 'init.templateDir=', '-c', 'protocol.allow=never',
'-c', 'protocol.https.allow=always', '-c', 'http.followRedirects=false', '-c', 'gc.auto=0', '-c', 'maintenance.auto=false', ...args],
{ cwd: directory, env: gitEnvironment(token), stdio: ['ignore', 'pipe', 'pipe'], detached: process.platform !== 'win32' });
{ cwd: directory, env: gitEnvironment(token), stdio: [options.input === undefined ? 'ignore' : 'pipe', 'pipe', 'pipe'], detached: process.platform !== 'win32' });
const chunks: Buffer[] = [];
let bytes = 0, stderr = '', unfinishedLine = '';
let failure: unknown;
@@ -116,12 +118,12 @@ async function runGit(directory: string, args: string[], token: string, options:
}
})().catch(stop).finally(() => { reporting = null; if (latest && !failure && !closed) report(); });
};
child.stdout.on('data', (chunk: Buffer) => {
child.stdout!.on('data', (chunk: Buffer) => {
bytes += chunk.length;
if (bytes > (options.maxOutput ?? MAX_TREE_BYTES)) stop(unprocessable('Repository contents exceed the scan limit. Use a smaller skills repository.'));
else chunks.push(chunk);
});
child.stderr.on('data', (chunk: Buffer) => {
child.stderr!.on('data', (chunk: Buffer) => {
stderr = (stderr + chunk.toString('utf8')).slice(-8192);
const lines = (unfinishedLine + chunk.toString('utf8')).split(/[\r\n]/);
unfinishedLine = lines.pop()!.slice(-4096);
@@ -151,6 +153,10 @@ async function runGit(directory: string, args: string[], token: string, options:
}
resolve(Buffer.concat(chunks));
});
if (options.input !== undefined) {
child.stdin!.on('error', () => stop(unprocessable('Could not read repository files. Try again.')));
child.stdin!.end(options.input);
}
});
}
@@ -183,6 +189,37 @@ function acquire(cached: NonNullable<ReturnType<typeof snapshots.get>>, key: str
if (released || !/^[a-f0-9]{40}$/.test(entry.sha) || entry.type !== 'blob' || (entry.size ?? Infinity) > 1024 * 1024) throw unprocessable('Invalid snapshot file.');
return runGit(cached.directory, ['cat-file', 'blob', entry.sha], '', { signal, maxOutput: 1024 * 1024 });
},
readBlobs: async (entries, signal) => {
signal?.throwIfAborted();
if (released || entries.length > MAX_REPOSITORY_FILES) throw unprocessable('Invalid repository file batch.');
let declaredBytes = 0;
for (const entry of entries) {
if (!/^[a-f0-9]{40}$/.test(entry.sha) || entry.type !== 'blob' || !['100644', '100755'].includes(entry.mode)
|| !Number.isSafeInteger(entry.size) || entry.size! < 0 || entry.size! > 1024 * 1024) throw unprocessable('Invalid snapshot file.');
declaredBytes += entry.size!;
}
if (declaredBytes > MAX_REPOSITORY_BYTES) throw unprocessable('Repository package exceeds the 61 MB runtime limit.');
const unique = [...new Map(entries.map(entry => [entry.sha, entry])).values()];
const blobs = new Map<string, Buffer>();
if (!unique.length) return blobs;
const bytes = await runGit(cached.directory, ['cat-file', '--batch'], '', {
signal, input: unique.map(entry => `${entry.sha}\n`).join(''),
maxOutput: MAX_REPOSITORY_BYTES + MAX_REPOSITORY_FILES * 80,
});
let offset = 0;
for (const entry of unique) {
const newline = bytes.indexOf(10, offset);
if (newline < offset || newline - offset > 80) throw unprocessable('Git returned an incomplete repository file batch.');
const header = bytes.subarray(offset, newline).toString('ascii');
if (header !== `${entry.sha} blob ${entry.size}`) throw unprocessable('Git returned an unexpected repository file.');
const start = newline + 1, end = start + entry.size!;
if (end >= bytes.length || bytes[end] !== 10) throw unprocessable('Git returned an incomplete repository file.');
blobs.set(entry.sha, bytes.subarray(start, end));
offset = end + 1;
}
if (offset !== bytes.length) throw unprocessable('Git returned unexpected repository file data.');
return blobs;
},
release: async () => {
if (released) return;
released = true;
+36 -13
View File
@@ -1,7 +1,7 @@
import { createHash, randomUUID } from 'node:crypto';
import path from 'node:path';
import { and, eq, isNull, lt, or, sql } from 'drizzle-orm';
import { companySkillSources as sources, companySkillSourceEntries as entries, companySkills, type Db } from '@paperclipai/db';
import { companySkillSources as sources, companySkillSourceEntries as entries, companySkills, companySkillRepositorySnapshots, type Db } from '@paperclipai/db';
import type { CompanySkill, SkillSource, SkillSourceCreateRequest, SkillSourceDiscoveryRequest, SkillSourceSelectionRequest, SkillSourceRefreshResult, SkillSourcePreviewRequest } from '@paperclipai/shared';
import { normalizeAgentUrlKey } from '@paperclipai/shared';
import { conflict, notFound, unprocessable } from '../errors.js';
@@ -41,7 +41,7 @@ export function skillSourceService(db: Db) {
}
async function discover(input: SkillSourceDiscoveryRequest, context: SkillSourceContext, options?: SkillScanOptions) {
await context.authorize('skills.import', { sourceType: 'git', sourceLocator: input.repositoryUrl });
const { skills: _files, defaultBranch: _defaultBranch, ...result } = await scanGitHubSkills(input, context.read(input.connectionId ?? null), { ...options, retainFiles: false });
const { skills: _files, repositoryFiles: _repositoryFiles, defaultBranch: _defaultBranch, ...result } = await scanGitHubSkills(input, context.read(input.connectionId ?? null), { ...options, retainFiles: false });
return result;
}
async function preview(input: SkillSourcePreviewRequest, context: SkillSourceContext) {
@@ -70,9 +70,22 @@ export function skillSourceService(db: Db) {
return `github/${scan.repositoryId}/${identity}/${normalizeAgentUrlKey(name) || 'skill'}`;
}
async function publish(tx: Tx, source: SourceRow, scan: ScannedSkillSource, selectedPaths: string[], excludedFolders: string[], context: SkillSourceContext, selectionReviewed: boolean): Promise<Omit<SkillSourceRefreshResult, 'source'>> {
const mode = scan.packageMode ?? 'skills';
if (mode === 'repository' && (scan.repositoryPackage?.error || !scan.repositoryFiles?.length)) {
throw unprocessable(scan.repositoryPackage?.error ?? 'Repository package has no files. Scan it again before importing.');
}
let repositorySnapshotId: string | undefined;
const repositoryHash = mode === 'repository' ? skillSnapshotHash(scan.repositoryFiles!) : null;
if (repositoryHash) {
await tx.insert(companySkillRepositorySnapshots).values({ companyId: source.companyId, contentHash: repositoryHash, fileInventory: scan.repositoryFiles! })
.onConflictDoNothing();
const [snapshot] = await tx.select({ id: companySkillRepositorySnapshots.id }).from(companySkillRepositorySnapshots).where(and(
eq(companySkillRepositorySnapshots.companyId, source.companyId), eq(companySkillRepositorySnapshots.contentHash, repositoryHash)));
repositorySnapshotId = snapshot!.id;
}
const previous = await tx.select().from(entries).where(and(eq(entries.companyId, source.companyId), eq(entries.sourceId, source.id)));
const imported: CompanySkill[] = [], updated: CompanySkill[] = [];
const warnings = [...scan.warnings];
const warnings = [...new Set([...scan.warnings, ...(scan.repositoryPackage?.warnings ?? [])])];
let unchanged = 0;
const selected = new Set(selectedPaths);
const scanned = new Set(scan.skills.map(candidate => candidate.path));
@@ -86,18 +99,19 @@ export function skillSourceService(db: Db) {
let skillId = old?.skillId ?? null;
const current = skillId ? await skills.getById(source.companyId, skillId, tx) : null;
if (selection === 'selected' && !candidate.error) {
const hash = skillSnapshotHash(candidate.files);
const hash = repositoryHash ? createHash('sha256').update(`${repositoryHash}:${candidate.path}`).digest('hex') : skillSnapshotHash(candidate.files);
const slug = normalizeAgentUrlKey(candidate.name) || 'skill';
const key = current?.key ?? newSkillKey(scan, candidate.path, candidate.name, source.id);
const ownerRepo = scan.fullName.split('/');
const metadata = { ...(current?.metadata ?? {}), sourceKind: 'github', hostname: 'github.com',
owner: ownerRepo[0], repo: ownerRepo[1], ref: current?.metadata?.snapshotHash === hash && current.currentVersionId ? current.sourceRef : scan.commitSha, trackingRef: scan.trackingRef,
repoSkillDir: path.posix.dirname(candidate.path) === '.' ? '' : path.posix.dirname(candidate.path),
skillSourceId: source.id, skillSourcePath: candidate.path, skillSourceState: 'synced', snapshotHash: hash };
skillSourceId: source.id, skillSourcePath: candidate.path, skillSourceState: 'synced', snapshotHash: hash, packageMode: mode };
const trustFiles = mode === 'repository' ? scan.repositoryFiles! : candidate.files;
const values = { name: candidate.name, description: candidate.description, markdown: candidate.files.find(file => file.path === 'SKILL.md')!.content,
sourceType: 'github', sourceLocator: scan.repositoryUrl, sourceRef: current?.metadata?.snapshotHash === hash && current.currentVersionId ? current.sourceRef : scan.commitSha,
fileInventory: candidate.files.map(({ path, kind }) => ({ path, kind })),
trustLevel: candidate.files.some(file => file.kind === 'script' || file.executable) ? 'scripts_executables' : candidate.files.some(file => file.kind === 'asset' || file.kind === 'other') ? 'assets' : 'markdown_only',
trustLevel: trustFiles.some(file => file.kind === 'script' || file.executable) ? 'scripts_executables' : trustFiles.some(file => file.kind === 'asset' || file.kind === 'other') ? 'assets' : 'markdown_only',
metadata, updatedAt: new Date() };
const changed = !current?.currentVersionId || current.metadata?.snapshotHash !== hash;
if (current && changed) await tx.update(companySkills).set(values).where(and(eq(companySkills.companyId, source.companyId), eq(companySkills.id, current.id)));
@@ -105,13 +119,14 @@ export function skillSourceService(db: Db) {
const [created] = await tx.insert(companySkills).values({ ...values, companyId: source.companyId, key, slug, installCount: 1 }).returning({ id: companySkills.id });
skillId = created!.id;
}
else if (current.metadata?.skillSourceState !== 'synced') {
await tx.update(companySkills).set({ metadata }).where(and(eq(companySkills.companyId, source.companyId), eq(companySkills.id, current.id)));
else if (current.metadata?.skillSourceState !== 'synced' || current.trustLevel !== values.trustLevel) {
await tx.update(companySkills).set({ metadata, trustLevel: values.trustLevel }).where(and(eq(companySkills.companyId, source.companyId), eq(companySkills.id, current.id)));
}
const installed = (await skills.getById(source.companyId, skillId!, tx))!;
if (changed) {
const version = await skills.createVersion(source.companyId, skillId!, { label: `GitHub ${scan.commitSha.slice(0, 8)}` }, context.actor, {
database: tx, skipInventoryRefresh: true, skill: installed, fileInventory: candidate.files,
...(repositorySnapshotId ? { repositorySnapshotId, repositorySkillPath: candidate.path } : {}),
});
installed.currentVersionId = version.id;
(current ? updated : imported).push(installed);
@@ -121,6 +136,10 @@ export function skillSourceService(db: Db) {
skillSourceState: selection !== 'selected' ? 'not_syncing' : 'update_failed' } }).where(and(eq(companySkills.companyId, source.companyId), eq(companySkills.id, current.id)));
}
if (candidate.error && selection === 'selected') warnings.push(`${candidate.path}: ${candidate.error}`);
if (selection === 'selected') {
warnings.push(...(candidate.inspection?.warnings.filter(warning => warning.includes('not included') || warning.includes('couldn’t be included')) ?? []));
warnings.push(...(candidate.inspection?.references.map(reference => `${reference.resolvedPath} — Referenced by ${candidate.path}; not included.`) ?? []));
}
await tx.insert(entries).values({ companyId: source.companyId, sourceId: source.id, path: candidate.path, name: candidate.name,
description: candidate.description, inspection: candidate.inspection ?? null, skillId, selection, present: true, error: candidate.error })
.onConflictDoUpdate({ target: [entries.sourceId, entries.path], set: { name: candidate.name, description: candidate.description, inspection: candidate.inspection ?? null, skillId, selection, present: true, error: candidate.error } });
@@ -140,10 +159,11 @@ export function skillSourceService(db: Db) {
or(eq(sources.repositoryId, scan.repositoryId), eq(sources.repositoryUrl, scan.repositoryUrl)),
)) : [];
await tx.update(sources).set({ repositoryId: scan.repositoryId, repositoryUrl: scan.repositoryUrl, fullName: scan.fullName,
trackingRef: existingBranch ? 'HEAD' : scan.trackingRef, excludedFolders, lastSuccessAt: new Date(), lastScanCommit: scan.commitSha, lastError: warnings.length ? `${warnings.length} warning(s). Review the source’s skills.` : null,
packageMode: mode, repositoryPackage: scan.repositoryPackage ?? null,
trackingRef: existingBranch ? 'HEAD' : scan.trackingRef, excludedFolders, lastSuccessAt: new Date(), lastScanCommit: scan.commitSha, lastError: null,
revision: source.revision + 1, leaseToken: null, leaseExpiresAt: null }).where(scope(source.companyId, source.id));
await context.audit(tx, source.id, 'company.skill_source_refreshed', { commit: scan.commitSha, importedCount: imported.length, updatedCount: updated.length, unchanged, warningCount: warnings.length });
return { imported, updated, unchanged, warnings };
return { imported, updated, unchanged, warnings: [...new Set(warnings)] };
}
async function create(companyId: string, input: SkillSourceCreateRequest, context: SkillSourceContext, staged?: ScannedSkillSource) {
await context.authorize('skills.import', { sourceType: 'git', sourceLocator: input.repositoryUrl });
@@ -159,7 +179,7 @@ export function skillSourceService(db: Db) {
}
return db.transaction(async tx => {
const [source] = await tx.insert(sources).values({ id, companyId, repositoryId: scan.repositoryId, repositoryUrl: scan.repositoryUrl, fullName: scan.fullName,
trackingRef: scan.trackingRef, connectionId: scan.connectionId ?? input.connectionId ?? null, lastAttemptAt: new Date() }).onConflictDoNothing().returning();
packageMode: scan.packageMode ?? 'skills', trackingRef: scan.trackingRef, connectionId: scan.connectionId ?? input.connectionId ?? null, lastAttemptAt: new Date() }).onConflictDoNothing().returning();
if (!source) throw conflict('This repository and branch are already in Sources. Manage its skills there.');
const result = await publish(tx, source, scan, input.selectedPaths, input.excludedFolders ?? [], context, true);
return { ...result, source: await detail(companyId, id, tx) };
@@ -178,7 +198,7 @@ export function skillSourceService(db: Db) {
source = leased;
try {
const connectionId = selection?.connectionId !== undefined ? selection.connectionId : source.connectionId;
const scan = staged ?? await scanGitHubSkills({ repositoryUrl: source.repositoryUrl, trackingRef: source.trackingRef }, context.read(connectionId));
const scan = staged ?? await scanGitHubSkills({ repositoryUrl: source.repositoryUrl, trackingRef: source.trackingRef, packageMode: selection?.packageMode ?? source.packageMode }, context.read(connectionId));
if (source.repositoryId && source.repositoryId !== scan.repositoryId) throw conflict('The repository at this URL has changed identity. Add it as a new source.');
const current = await detail(companyId, id);
const selectedPaths = selection?.selectedPaths ?? current.entries.filter(entry => entry.selection === 'selected').map(entry => entry.path);
@@ -241,8 +261,11 @@ export function skillSourceService(db: Db) {
if (!trackingRef) throw unprocessable('GitHub branch or commit was not found.');
}
await context.authorize('skills.import', { sourceType: 'git', sourceLocator: repositoryUrl });
const scan = await scanGitHubSkills({ repositoryUrl, trackingRef, commitSha }, read);
let scan = await scanGitHubSkills({ repositoryUrl, trackingRef, commitSha }, read);
const existing = previous.find(source => source.trackingRef === scan.trackingRef || (source.trackingRef === 'HEAD' && scan.trackingRef === scan.defaultBranch));
if (existing?.packageMode === 'repository') {
scan = await scanGitHubSkills({ repositoryUrl, trackingRef: scan.trackingRef, commitSha: scan.commitSha, packageMode: 'repository' }, read);
}
const selectedPaths = scan.candidates.filter(candidate =>
(!prefix || candidate.path === prefix || candidate.path.startsWith(`${prefix}/`)) &&
(!parsed.requestedSkillSlug || normalizeAgentUrlKey(candidate.name) === parsed.requestedSkillSlug || candidate.path.split('/').at(-2) === parsed.requestedSkillSlug)