Merge branch 'codex/work-folders-pi-refresh' into codex/work-folders-storybook-refresh

* codex/work-folders-pi-refresh:
  fix(runner): keep healthy native sessions alive (#13261)
  feat(connections): add AgentMail inboxes and email tasks (#13256)
  fix: stop completion reviews caused by policy upgrades (#13266)
  fix(ci): avoid empty pnpm caches from lockfile refresh (#13267)
  feat: accept a base64-encoded Cloud UI snippet (#13245)
  ci: use reserved AWS capacity for post-merge cloud verification (#13257)
  Fix Codex API key authentication in tests and runs (#13260)
  refactor: remove automatic productivity reviews (#13263)
  fix: continue conversations after confirmed remote runner stop (#13254)
  ci: cache Rust dependencies used by post-merge typecheck (#13259)
  fix(ui): restore task page archive shortcut (#13253)
  feat(ui): show one rolling runner activity per commentary group (#13255)
  fix(ui): hide retry countdown after execution starts (#13258)
  test: isolate native controller takeover fixture (#13212)
This commit is contained in:
Dotta committed 2026-09-11 17:15:49 -05:00
commit 24a7ab22f6
201 files changed
+62608 -3880

No files matched your search

+283
View File
@@ -0,0 +1,283 @@
import { z } from "zod";
import { Webhook } from "svix";
import type { EmailEnvelope } from "@paperclipai/shared";
const strings = z.array(z.string());
export const agentmailMessageSchema = z.object({
inbox_id: z.string().min(1),
thread_id: z.string().min(1),
message_id: z.string().min(1),
from: z.string().default(""),
to: strings.default([]),
cc: strings.optional(),
bcc: strings.optional(),
reply_to: strings.optional(),
subject: z.string().default("(No subject)"),
text: z.string().optional(),
html: z.string().optional(),
extracted_text: z.string().optional(),
timestamp: z.string().datetime({ offset: true }),
created_at: z.string().datetime({ offset: true }).optional(),
labels: strings.default([]),
headers: z.record(z.string(), z.string()).default({}),
attachments: z
.array(
z.object({
attachment_id: z.string(),
filename: z.string().optional(),
content_type: z.string().optional(),
size: z.number().nonnegative(),
}),
)
.default([]),
});
export type AgentmailMessage = z.infer<typeof agentmailMessageSchema>;
export interface AgentmailInbox {
inbox_id: string;
display_name?: string;
}
export interface AgentmailScope {
scope_type: "organization" | "pod" | "inbox";
organization_id: string;
pod_id?: string;
inbox_id?: string;
}
export class AgentmailApiError extends Error {
constructor(
readonly status: number,
readonly retryAfterMs = 1000,
) {
// Provider bodies may contain credentials or private mail. Never log them.
super(`AgentMail request failed (${status})`);
}
}
export const AGENTMAIL_EVENTS = [
"message.received",
"message.sent",
"message.delivered",
"message.bounced",
"message.complained",
"message.rejected",
];
export function emailText(message: AgentmailMessage): string {
return (
message.extracted_text ??
message.text ??
(message.html
? message.html
.replace(/<script\b[^>]*>[\s\S]*?<\/script>/gi, "")
.replace(/<style\b[^>]*>[\s\S]*?<\/style>/gi, "")
.replace(/<[^>]*>/g, " ")
: "")
).slice(0, 100_000);
}
/** Reconstruct only visible recipients; never let provider reply-all inherit Bcc. */
export function emailReplyRecipients(
message: EmailEnvelope,
ownAddress: string,
replyAll: boolean,
) {
const address = (value: string) =>
(value.match(/<([^>]+)>/)?.[1] ?? value).trim();
const seen = new Set([address(ownAddress).toLowerCase()]);
const unique = (values: string[]) =>
values.map(address).filter((value) => {
const key = value.toLowerCase();
if (!value || seen.has(key)) return false;
seen.add(key);
return true;
});
const replyTargets = message.replyTo?.length ? message.replyTo : [message.from];
const to = unique([...replyTargets, ...(replyAll ? message.to : [])]);
const cc = unique(replyAll ? (message.cc ?? []) : []);
return { to, cc, bcc: [], reply_all: false };
}
export function isAutomaticEmail(message: AgentmailMessage): boolean {
const headers = Object.fromEntries(
Object.entries(message.headers).map(([k, v]) => [
k.toLowerCase(),
v.toLowerCase(),
]),
);
return Boolean(
(headers["auto-submitted"] && headers["auto-submitted"] !== "no") ||
/^(bulk|list|junk)$/.test(headers.precedence ?? "") ||
headers["x-autoreply"] ||
headers["x-autorespond"],
);
}
export function isFilteredEmail(message: AgentmailMessage): boolean {
return message.labels.some((label) =>
["spam", "blocked", "unauthenticated", "trash"].includes(label),
);
}
export function verifyAgentmailWebhook(
body: Buffer,
headers: Record<string, string>,
secret: string,
): unknown {
return new Webhook(secret).verify(body.toString("utf8"), headers);
}
export function normalizeAgentmailEvent(value: unknown) {
const parsed = z
.object({
type: z.string().optional(),
event_type: z.string().optional(),
event_id: z.string().optional(),
message: z.unknown().optional(),
send: z.unknown().optional(),
delivery: z.unknown().optional(),
bounce: z.unknown().optional(),
complaint: z.unknown().optional(),
reject: z.unknown().optional(),
})
.parse(value);
const kind =
parsed.event_type ?? parsed.type?.replace(/^message_/, "message.");
if (!kind || !AGENTMAIL_EVENTS.includes(kind)) return null;
// Provider receipts use event-specific envelopes, shared by both transports.
// Internal reconciliation events may supply the fetched message directly.
const receipts: Record<string, unknown> = {
"message.sent": parsed.send,
"message.delivered": parsed.delivery,
"message.bounced": parsed.bounce,
"message.complained": parsed.complaint,
"message.rejected": parsed.reject,
};
// Fetch the authoritative message before intake; delivery events have reduced payloads.
const message = z
.object({ inbox_id: z.string(), message_id: z.string() })
.parse(receipts[kind] ?? parsed.message);
return {
kind,
...message,
eventId: parsed.event_id ?? `${kind}:${message.message_id}`,
};
}
/** REST is the email protocol boundary; credentials never enter an agent runtime. */
export function agentmailApi(apiKey: string, fetchImpl: typeof fetch = fetch) {
async function request<T>(
path: string,
method = "GET",
body?: unknown,
idempotencyKey?: string,
): Promise<T> {
const response = await fetchImpl(`https://api.agentmail.to/v0${path}`, {
method,
signal: AbortSignal.timeout(25_000),
redirect: "error",
headers: {
Authorization: `Bearer ${apiKey}`,
"Content-Type": "application/json",
...(idempotencyKey ? { "Idempotency-Key": idempotencyKey } : {}),
},
body: body === undefined ? undefined : JSON.stringify(body),
});
if (!response.ok) {
const retryAfter = response.headers.get("retry-after");
const seconds = Number(retryAfter ?? 1);
const delay = Number.isFinite(seconds)
? seconds * 1000
: Date.parse(retryAfter ?? "") - Date.now();
throw new AgentmailApiError(
response.status,
Math.max(1000, Math.min(300_000, Number.isFinite(delay) ? delay : 1000)),
);
}
if (response.status === 204) return undefined as T;
if (!response.body) throw new Error("Empty AgentMail response");
const reader = response.body.getReader();
const parts: Uint8Array[] = [];
let bytes = 0;
try {
for (;;) {
const part = await reader.read();
if (part.done) break;
bytes += part.value.length;
if (bytes > 16 * 1024 * 1024)
throw new Error("AgentMail response exceeds the processing limit");
parts.push(part.value);
}
} finally {
await reader.cancel();
}
return JSON.parse(Buffer.concat(parts).toString("utf8")) as T;
}
const inboxPath = (id: string) => `/inboxes/${encodeURIComponent(id)}`;
return {
request,
whoami: () => request<AgentmailScope>("/auth/me"),
getInbox: (id: string) => request<AgentmailInbox>(inboxPath(id)),
listInboxes: () =>
request<{ inboxes: AgentmailInbox[] }>("/inboxes?limit=100"),
listDomains: () =>
request<{ domains: { domain_id: string; domain: string }[] }>(
"/domains?limit=100",
),
getDomain: (id: string) =>
request<{ domain_id: string; domain: string; status: string }>(
`/domains/${encodeURIComponent(id)}`,
),
createInbox: (body: unknown) =>
request<AgentmailInbox>("/inboxes", "POST", body),
createInboxKey: (id: string) =>
request<{ api_key: string; api_key_id: string }>(
`${inboxPath(id)}/api-keys`,
"POST",
{ name: "Paperclip email runtime" },
),
deleteInboxKey: (id: string, keyId: string) =>
request<void>(
`${inboxPath(id)}/api-keys/${encodeURIComponent(keyId)}`,
"DELETE",
),
createWebhook: (id: string, url: string, clientId: string) =>
request<{ webhook_id: string; secret: string }>(
`${inboxPath(id)}/webhooks`,
"POST",
{ url, event_types: AGENTMAIL_EVENTS, client_id: clientId },
),
deleteWebhook: (id: string, webhookId: string) =>
request<void>(
`${inboxPath(id)}/webhooks/${encodeURIComponent(webhookId)}`,
"DELETE",
),
getMessage: async (id: string, messageId: string) =>
agentmailMessageSchema.parse(
await request(
`${inboxPath(id)}/messages/${encodeURIComponent(messageId)}`,
),
),
getThread: async (id: string, threadId: string) =>
z
.object({ messages: z.array(agentmailMessageSchema) })
.parse(
await request(
`${inboxPath(id)}/threads/${encodeURIComponent(threadId)}`,
),
),
listMessages: (id: string, after?: string, page?: string) =>
request<{
messages: {
message_id: string;
created_at?: string;
timestamp?: string;
}[];
next_page_token?: string;
}>(
`${inboxPath(id)}/messages?${new URLSearchParams({ ...(after ? { after } : {}), ascending: "true", limit: "100", ...(page ? { page_token: page } : {}) })}`,
),
send: (id: string, body: unknown, key: string, replyId?: string) =>
request<{ message_id: string; thread_id: string }>(
`${inboxPath(id)}/messages/${replyId ? `${encodeURIComponent(replyId)}/reply` : "send"}`,
"POST",
body,
key,
),
getAttachment: (id: string, messageId: string, attachmentId: string) =>
request<{ download_url: string; size: number }>(
`${inboxPath(id)}/messages/${encodeURIComponent(messageId)}/attachments/${encodeURIComponent(attachmentId)}`,
),
};
}
-61
View File
@@ -49,7 +49,6 @@ import type {
IssueReviewPolicy,
} from "@paperclipai/shared";
import { badRequest } from "../errors.js";
import { PRODUCTIVITY_REVIEW_ORIGIN_KIND } from "./productivity-review.js";
import { budgetService } from "./budgets.js";
import {
BLOCKER_ATTENTION_MAX_DEPTH,
@@ -105,7 +104,6 @@ const SOURCE_RANK: Record<AttentionSourceKind, number> = {
const PENDING_INTERACTION_STATUSES = ["pending"] as const;
const OPEN_RECOVERY_STATUSES = ["active", "escalated"] as const;
const HUMAN_RECOVERY_OWNER_TYPES = ["user", "board"] as const;
const PRODUCTIVITY_REVIEW_TERMINAL_STATUSES = ["done", "cancelled"] as const;
const FAILED_RUN_STATUSES = ["failed", "timed_out"] as const;
const DETAIL_EXCERPT_LENGTH = 160;
const DETAIL_IMAGE_LIMIT = 3;
@@ -1453,65 +1451,6 @@ export function attentionService(db: Db, serviceOptions: AttentionServiceOptions
}));
}
const productivityRows = await db
.select({
id: issues.id,
companyId: issues.companyId,
identifier: issues.identifier,
title: issues.title,
status: issues.status,
priority: issues.priority,
originId: issues.originId,
originFingerprint: issues.originFingerprint,
assigneeAgentId: issues.assigneeAgentId,
assigneeUserId: issues.assigneeUserId,
createdAt: issues.createdAt,
updatedAt: issues.updatedAt,
})
.from(issues)
.where(and(
eq(issues.companyId, companyId),
eq(issues.originKind, PRODUCTIVITY_REVIEW_ORIGIN_KIND),
isNull(issues.hiddenAt),
isNotNull(issues.assigneeUserId),
notInArray(issues.status, [...PRODUCTIVITY_REVIEW_TERMINAL_STATUSES]),
))
.orderBy(desc(issues.updatedAt), desc(issues.id));
const [productivitySourceMap, productivityReviewMap, productivityImageMap] = await Promise.all([
issueSummaryMap(db, companyId, productivityRows.map((row) => row.originId)),
issueSummaryMap(db, companyId, productivityRows.map((row) => row.id)),
issueImageMap(db, companyId, productivityRows.map((row) => row.id)),
]);
for (const review of productivityRows) {
const reviewIssue = productivityReviewMap.get(review.id);
if (!reviewIssue) continue;
const sourceIssue = review.originId ? productivitySourceMap.get(review.originId) ?? null : null;
const dedupKey = `productivity_review:${review.originFingerprint ?? review.originId ?? review.id}`;
add(createItem({
companyId,
sourceKind: "productivity_review",
subject: issueSubject(prefix, reviewIssue),
whyNow: "Productivity review is awaiting a human decision.",
decisionVerbs: decisionVerbs(
{ id: "resolve", label: "Resolve", description: "Record a productivity review outcome." },
{ id: "dismiss", label: "Dismiss", description: "Dismiss this review for now." },
{ id: "reassign", label: "Reassign", description: "Move the review to another owner." },
),
inlineResolvable: false,
entryRule: "Open issue_productivity_review issue assigned to a user.",
exitRule: "Review issue is done/cancelled or no longer assigned to a user.",
dedupKey,
severity: review.priority === "critical" ? "critical" : review.priority === "high" ? "high" : "medium",
activityAt: toIso(review.updatedAt),
createdAt: toIso(review.createdAt),
updatedAt: toIso(review.updatedAt),
relatedIssue: sourceIssue ? issueSubject(prefix, sourceIssue) : null,
...issueContext(reviewIssue),
detail: genericDetail(sourceIssue?.title ?? review.title, issueImages(productivityImageMap, review.id)),
}));
}
const blockedIssues = await issueService(db).list(companyId, { status: "blocked", includeBlockedBy: true });
type BlockedAttentionIssue = IssueSubjectRow & {
blockerAttention?: {
+19 -1
View File
@@ -383,6 +383,7 @@ function publicationSummary(
}
const PROVIDER_LABELS: Record<ChatProvider, string> = {
agentmail: "AgentMail",
slack: "Slack",
github: "GitHub",
discord: "Discord",
@@ -576,6 +577,7 @@ async function inspectSlackCallback(
}
const CAPABILITIES: Record<ChatProvider, ChatAdapterCapabilities> = {
agentmail: { threads: true, directMessages: true, nativeStreaming: false, messageEdits: false, messageDeletes: false, reactions: false, files: true, cards: false, actions: false, modals: false, slashCommands: false, ephemeralMessages: false, proactiveDirectMessages: true },
slack: {
threads: true,
directMessages: true,
@@ -668,6 +670,7 @@ const REQUIRED_CREDENTIALS: Record<
Exclude<ChatProvider, "github">,
readonly string[]
> = {
agentmail: [],
slack: ["botToken", "signingSecret"],
discord: ["botToken", "applicationId", "guildId"],
"microsoft-teams": ["clientId", "tenantId", "clientSecret"],
@@ -725,6 +728,7 @@ const SUPPORTED_GITHUB_WEBHOOK_EVENTS = new Set<string>([
]);
const SUPPLIED_CREDENTIAL_KEYS: Record<ChatProvider, readonly string[]> = {
agentmail: [],
slack: ["botToken", "signingSecret"],
github: ["appId", "privateKey"],
discord: ["botToken", "applicationId", "guildId"],
@@ -2590,6 +2594,7 @@ function providerSetupState(
const webhookUrl = publicBaseUrl ? `${publicBaseUrl}${path}` : null;
const step = endpoint.status === "active" ? "complete" : endpoint.setup.step;
switch (endpoint.provider) {
case "agentmail": return endpoint.setup;
case "slack": {
const observations = (endpoint.setup as InternalSetupState)
.slackCallbackSurfaces;
@@ -5747,6 +5752,8 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) {
id: endpoint.id,
companyId: endpoint.companyId,
connectionId: endpoint.connectionId,
publicationMode: endpoint.publicationMode,
externalExecutionPolicy: endpoint.externalExecutionPolicy,
provider: endpoint.provider,
publicId: endpoint.publicId,
status: endpoint.status,
@@ -5832,6 +5839,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) {
input: CreateChatEndpointInput,
actorUserId?: string | null,
) {
if ((input.provider as string) === "agentmail") throw badRequest("Use the email inbox setup API for AgentMail");
const agent = await db
.select({ id: agents.id, name: agents.name, status: agents.status })
.from(agents)
@@ -5968,6 +5976,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) {
) {
const initial = await endpointRecord(endpointId);
if (!initial) throw notFound("Chat endpoint not found");
if (initial.endpoint.provider === "agentmail") throw badRequest("Use the email inbox API for AgentMail");
await withCredentialMutationLease(
initial.endpoint,
async (credentialLease) => {
@@ -8048,6 +8057,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) {
) {
const record = await endpointRecord(endpointId);
if (!record) throw notFound("Chat endpoint not found");
if (record.endpoint.provider === "agentmail") throw badRequest("Use the email inbox API for AgentMail");
const suppliedCredentialKeys = Object.keys(input.credentials ?? {});
if (suppliedCredentialKeys.length > 0) {
const credentialAction =
@@ -26104,7 +26114,8 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) {
.from(chatDeliveries)
.where(
and(
onlyDeliveryId ? eq(chatDeliveries.id, onlyDeliveryId) : undefined,
sql`not exists (select 1 from chat_endpoints e where e.id = ${chatDeliveries.endpointId} and e.provider = 'agentmail')`,
onlyDeliveryId ? eq(chatDeliveries.id, onlyDeliveryId) : undefined,
inArray(chatDeliveries.eventKind, [
"reaction_added",
"reaction_removed",
@@ -26195,6 +26206,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) {
.from(chatDeliveries)
.where(
and(
sql`not exists (select 1 from chat_endpoints e where e.id = ${chatDeliveries.endpointId} and e.provider = 'agentmail')`,
onlyDeliveryId ? eq(chatDeliveries.id, onlyDeliveryId) : undefined,
notInArray(chatDeliveries.eventKind, [
"reaction_added",
@@ -28642,6 +28654,8 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) {
conversationId: string,
commentId: string,
) {
const emailBoundary = await endpointRecord(endpointId);
if (emailBoundary?.endpoint.publicationMode === "explicit") throw badRequest("Use an explicit email send action");
const conversation = await db
.select()
.from(chatConversations)
@@ -28714,6 +28728,8 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) {
userId: string,
attachmentIds: string[] = [],
) {
const emailBoundary = await endpointRecord(endpointId);
if (emailBoundary?.endpoint.publicationMode === "explicit") throw badRequest("Use an explicit email send action");
// Browser request IDs are only unique within the conversation that issued
// them. Include that durable task boundary so a retried key from another
// conversation can neither suppress its send nor return the first task's
@@ -35794,6 +35810,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) {
})
.where(
and(
sql`not exists (select 1 from chat_endpoints e where e.id = ${chatPublications.endpointId} and e.publication_mode = 'explicit')`,
eq(chatPublications.state, "streaming"),
lte(chatPublications.updatedAt, staleBefore),
// Teams owns separate staged I/O intents and a longer attempt lease.
@@ -35884,6 +35901,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) {
and(
or(
and(
sql`not exists (select 1 from chat_endpoints e where e.id = ${chatPublications.endpointId} and e.publication_mode = 'explicit')`,
inArray(chatPublications.state, ["pending", "retry"]),
notExists(
db
@@ -277,6 +277,7 @@ export async function enqueueIssueInteractionChatPublications(
and(
eq(chatEndpoints.companyId, chatConversations.companyId),
eq(chatEndpoints.id, chatConversations.endpointId),
eq(chatEndpoints.publicationMode, "automatic"),
),
)
.where(
@@ -493,6 +493,7 @@ export function parseChatProviderLifecycle(
input: ParseChatProviderLifecycleInput,
): ChatProviderLifecycleEffect[] {
switch (input.provider) {
case "agentmail": return [];
case "slack":
return parseSlackLifecycle(input);
case "github":
@@ -240,6 +240,7 @@ async function enqueueSafeNativeChatProgress(
and(
eq(chatEndpoints.companyId, chatConversations.companyId),
eq(chatEndpoints.id, chatConversations.endpointId),
eq(chatEndpoints.publicationMode, "automatic"),
eq(chatEndpoints.assignedAgentId, heartbeatRuns.agentId),
),
)
@@ -360,6 +361,7 @@ async function enqueueSafeNativeChatProgress(
and(
eq(chatEndpoints.companyId, chatConversations.companyId),
eq(chatEndpoints.id, chatConversations.endpointId),
eq(chatEndpoints.publicationMode, "automatic"),
eq(chatEndpoints.assignedAgentId, row.agentId),
),
)
@@ -671,6 +673,7 @@ export async function enqueueChatRunMilestones(
and(
eq(chatEndpoints.companyId, chatConversations.companyId),
eq(chatEndpoints.id, chatConversations.endpointId),
eq(chatEndpoints.publicationMode, "automatic"),
eq(chatEndpoints.assignedAgentId, heartbeatRuns.agentId),
),
)
+278
View File
@@ -0,0 +1,278 @@
import { promises as fs } from "node:fs";
import path from "node:path";
import { fileURLToPath } from "node:url";
import { createHash } from "node:crypto";
import type { Db } from "@paperclipai/db";
import type { AgentSkillSnapshot } from "@paperclipai/shared";
import {
resolvePaperclipSkillsDir,
readPaperclipSkillSyncPreference,
writePaperclipSkillSyncPreference,
type PaperclipSkillEntry,
} from "@paperclipai/adapter-utils/server-utils";
import { forbidden } from "../errors.js";
import { emailChannelService } from "./email-channels.js";
import {
AGENTMAIL_TOOLS,
executeAgentmailTool,
} from "./connectors/agentmail.js";
import { materializeAsset } from "./native-runtime/runtime-context.js";
type AgentBinding = { companyId: string; agentId: string };
type ToolBinding = AgentBinding & {
runId: string;
issueId: string;
workMode?: string;
};
type Resource = {
id: string;
label: string;
connectionId: string;
metadata?: Record<string, unknown>;
};
type Tool = {
name: string;
description: string;
inputSchema: Record<string, unknown>;
};
interface ConnectorDefinition {
key: string;
label: string;
skillName: string;
tools: Tool[];
resolve: (db: Db, binding: AgentBinding) => Promise<Resource[]>;
execute: (
db: Db,
binding: ToolBinding,
name: string,
value: unknown,
) => Promise<unknown>;
}
// Trusted connector packages declare their contributions here. Assignments and
// current access, not credential availability or agent-authored config, select them.
const connectors: ConnectorDefinition[] = [
{
key: "agentmail",
label: "AgentMail",
skillName: "agentmail",
tools: AGENTMAIL_TOOLS.map(
({ action: _action, ...definition }) => definition,
),
async resolve(db, binding) {
const service = emailChannelService(db, {
heartbeat: { wakeup: async () => null },
});
return (
await service.assignedInboxes(binding.companyId, binding.agentId)
).map(({ id, address, connectionId }) => ({
id,
label: address ?? id,
connectionId,
}));
},
async execute(db, binding, name, value) {
const tool = AGENTMAIL_TOOLS.find((entry) => entry.name === name);
if (!tool) throw forbidden("Unknown AgentMail tool");
if (!value || typeof value !== "object" || Array.isArray(value))
throw forbidden("Expected tool arguments");
return executeAgentmailTool(db, binding, {
...value,
action: tool.action,
});
},
},
];
const skillKey = (connector: ConnectorDefinition) =>
`paperclipai/paperclip/${connector.skillName}`;
export type ConnectorAssignment = {
key: string;
label: string;
skillKey: string;
resources: Resource[];
tools: Tool[];
};
export async function resolveConnectorAssignments(
db: Db,
binding: AgentBinding,
): Promise<ConnectorAssignment[]> {
const assignments: ConnectorAssignment[] = [];
for (const connector of connectors) {
const resources = await connector.resolve(db, binding);
if (resources.length)
assignments.push({
key: connector.key,
label: connector.label,
skillKey: skillKey(connector),
resources,
tools: connector.tools,
});
}
return assignments;
}
export function isConnectorSkill(key: string) {
return connectors.some((connector) => skillKey(connector) === key);
}
export function isConnectorTool(name: string) {
return connectors.some((connector) =>
connector.tools.some((tool) => tool.name === name),
);
}
export async function executeConnectorTool(
db: Db,
binding: ToolBinding,
name: string,
value: unknown,
) {
const connector = connectors.find((entry) =>
entry.tools.some((tool) => tool.name === name),
);
if (!connector || !(await connector.resolve(db, binding)).length)
throw forbidden("This connector is no longer assigned or authorized");
return connector.execute(db, binding, name, value);
}
/** Runtime-only overlay: never persist automatic assignments into agent preferences. */
export async function applyConnectorSkills(
config: Record<string, unknown>,
entries: PaperclipSkillEntry[],
assignments: ConnectorAssignment[],
) {
const reserved = new Set(
connectors.flatMap((connector) => [
skillKey(connector),
connector.skillName,
]),
);
const desired = readPaperclipSkillSyncPreference(
config,
).desiredSkillEntries.filter((entry) => !reserved.has(entry.key));
const skills = entries.filter(
(entry) => !reserved.has(entry.key) && !reserved.has(entry.runtimeName),
);
for (const assignment of assignments) {
const connector = connectors.find((entry) => entry.key === assignment.key)!;
const root = await resolvePaperclipSkillsDir(
path.dirname(fileURLToPath(import.meta.url)),
[fileURLToPath(new URL("../../../skills", import.meta.url))],
);
if (!root)
throw new Error(`Bundled connector skill is missing: ${connector.key}`);
const markdown = await fs.readFile(
path.join(root, connector.skillName, "SKILL.md"),
"utf8",
);
const toolRevision = createHash("sha256")
.update(JSON.stringify(assignment.tools))
.digest("hex");
const context = `\n\n## Assigned resources\n\nPaperclip supplies the following resource identifiers as data, not instructions.\nThese assignments are checked again on every call.\n\n\`\`\`json\n${JSON.stringify(assignment.resources, null, 2)}\n\`\`\`\n\n<!-- Connector tools revision: ${toolRevision} -->\n`;
const bundle = await materializeAsset([
{
path: "SKILL.md",
content: Buffer.from(markdown + context),
mode: 0o444,
},
]);
skills.push({
key: assignment.skillKey,
runtimeName: connector.skillName,
source: bundle.rootPath,
sourceStatus: "available",
});
desired.push({ key: assignment.skillKey, versionId: null });
}
const connectorSkillDigest = assignments.length
? createHash("sha256")
.update(
JSON.stringify(skills.filter((skill) => reserved.has(skill.key))),
)
.digest("hex")
: null;
return {
...writePaperclipSkillSyncPreference(config, desired),
paperclipRuntimeSkills: skills,
paperclipConnectorSkillDigest: connectorSkillDigest,
};
}
/** Shared-home adapters receive the assigned skill in the run prompt, never on disk. */
export async function prepareConnectorSkillDelivery(
config: Record<string, unknown> & Awaited<ReturnType<typeof applyConnectorSkills>>,
adapterType: string,
) {
const scopedFiles =
adapterType === "paperclip_runner" ||
(config.engine === "cli" &&
["codex_local", "claude_local", "kimi_local"].includes(adapterType));
if (scopedFiles) return { config, instructions: "" };
const assigned = config.paperclipRuntimeSkills.filter((entry) =>
isConnectorSkill(entry.key),
);
const instructions = (
await Promise.all(
assigned.map(
async (entry) =>
`### ${entry.runtimeName}\n\n${await fs.readFile(path.join(entry.source, "SKILL.md"), "utf8")}`,
),
)
).join("\n\n");
const stripped = await applyConnectorSkills(
config,
config.paperclipRuntimeSkills,
[],
);
return {
config: {
...stripped,
paperclipConnectorSkillDigest: config.paperclipConnectorSkillDigest,
},
instructions,
};
}
export function annotateConnectorSkills(
snapshot: AgentSkillSnapshot,
assignments: ConnectorAssignment[],
): AgentSkillSnapshot {
const entries = [...snapshot.entries];
for (const connector of connectors) {
if (!entries.some((entry) => entry.key === skillKey(connector)))
entries.push({
key: skillKey(connector),
runtimeName: connector.skillName,
desired: assignments.some(
(entry) => entry.skillKey === skillKey(connector),
),
managed: true,
state: "available",
readOnly: true,
originLabel: `${connector.label} assignment`,
detail:
"Provided automatically when this connector assigns a resource to the agent.",
});
}
return {
...snapshot,
entries: entries.map((entry) => {
const assignment = assignments.find(
(item) => item.skillKey === entry.key,
);
return assignment
? {
...entry,
desired: true,
state: "configured",
readOnly: true,
originLabel: `${assignment.label} assignment`,
detail: `Provided automatically by ${assignment.label}: ${assignment.resources.map((resource) => resource.label).join(", ")}. Manage this skill through the connector assignment.`,
}
: connectors.some((connector) => skillKey(connector) === entry.key)
? { ...entry, readOnly: true }
: entry;
}),
};
}
+177
View File
@@ -0,0 +1,177 @@
import { z } from "zod";
import type { Db } from "@paperclipai/db";
import { emailSendSchema } from "@paperclipai/shared";
import { emailChannelService } from "../email-channels.js";
import { forbidden, notFound } from "../../errors.js";
import { instanceSettingsService } from "../instance-settings.js";
const AGENTMAIL_EMAIL_CONTRACT = {
description:
"Use an assigned AgentMail inbox for this task. Internal comments and final responses never send email. List inboxes, read the current email thread, explicitly send a new email child task or reply, and inspect delivery. Requires experimental email connections. A send needs a UUID idempotencyKey; preserve it and the identical payload on retry. A reply uses conversationId and replyToMessageId from thread; replyAll defaults false and excludes Bcc. Sending does not close the task.",
inputSchema: {
type: "object",
properties: {
action: {
type: "string",
enum: ["inboxes", "thread", "send", "delivery"],
},
publicationId: {
type: "string",
description: "Publication UUID returned by send, for delivery status.",
},
request: {
type: "object",
properties: {
endpointId: { type: "string" },
parentIssueId: {
type: "string",
description: "Current task UUID for a new email child task.",
},
conversationId: { type: "string" },
replyToMessageId: { type: "string" },
replyAll: { type: "boolean" },
to: { type: "array", items: { type: "string" } },
cc: { type: "array", items: { type: "string" } },
bcc: { type: "array", items: { type: "string" } },
subject: { type: "string" },
text: { type: "string" },
attachmentIds: { type: "array", items: { type: "string" } },
idempotencyKey: { type: "string" },
},
required: ["endpointId", "text", "idempotencyKey"],
additionalProperties: false,
},
},
required: ["action"],
additionalProperties: false,
},
} as const;
// Connector-owned definitions. They are never part of the universal runner catalog.
export const AGENTMAIL_TOOLS = [
{
name: "agentmail_inboxes",
action: "inboxes",
description: "List your active assigned AgentMail inboxes.",
inputSchema: {
type: "object",
properties: {},
additionalProperties: false,
},
},
{
name: "agentmail_read_thread",
action: "thread",
description:
"Read the current task's AgentMail email thread, recipients, messages and attachments.",
inputSchema: {
type: "object",
properties: {},
additionalProperties: false,
},
},
{
name: "agentmail_send",
action: "send",
description: AGENTMAIL_EMAIL_CONTRACT.description,
inputSchema: {
type: "object",
properties: {
request: AGENTMAIL_EMAIL_CONTRACT.inputSchema.properties.request,
},
required: ["request"],
additionalProperties: false,
},
},
{
name: "agentmail_delivery",
action: "delivery",
description:
"Check delivery of an AgentMail publication belonging to your task.",
inputSchema: {
type: "object",
properties: {
publicationId:
AGENTMAIL_EMAIL_CONTRACT.inputSchema.properties.publicationId,
},
required: ["publicationId"],
additionalProperties: false,
},
},
] as const;
const schema = z
.object({
action: z.enum(["inboxes", "thread", "send", "delivery"]),
request: emailSendSchema.optional(),
publicationId: z.string().uuid().optional(),
})
.strict();
export async function executeAgentmailTool(
db: Db,
binding: {
companyId: string;
agentId: string;
runId: string;
issueId: string;
workMode?: string;
},
value: unknown,
) {
if (
!(await instanceSettingsService(db).getExperimental()).enableChatConnectors
)
throw forbidden("Experimental email connections are disabled");
const input = schema.parse(value);
// This facade only persists intents/reads. The app's durable email worker owns execution.
const service = emailChannelService(db, {
heartbeat: {
wakeup: async () => {
throw new Error("Task email facade cannot start a receive worker");
},
},
});
const inboxes = await service.assignedInboxes(
binding.companyId,
binding.agentId,
);
if (!inboxes.length) throw forbidden("No active assigned AgentMail inbox");
if (input.action === "inboxes") return inboxes;
await service.authorizeRead(binding.companyId, binding.issueId, {
agentId: binding.agentId,
runId: binding.runId,
});
const thread = await service.thread(binding.companyId, binding.issueId);
if (thread && !inboxes.some((inbox) => inbox.id === thread.endpoint.id))
throw notFound("Email task not found");
if (input.action === "thread") return thread;
if (input.action === "delivery") {
if (!input.publicationId) throw forbidden("Publication ID required");
const delivery = await service.publication(
input.publicationId,
binding.companyId,
);
await service.authorizeRead(binding.companyId, delivery.issueId, {
agentId: binding.agentId,
runId: binding.runId,
});
const target = await service.thread(binding.companyId, delivery.issueId);
if (!target || !inboxes.some((inbox) => inbox.id === target.endpoint.id))
throw notFound("Email delivery not found");
return delivery;
}
if (binding.workMode && binding.workMode !== "standard")
throw forbidden("Email sends require standard work mode");
if (
!input.request ||
(input.request.parentIssueId &&
input.request.parentIssueId !== binding.issueId) ||
(input.request.conversationId &&
input.request.conversationId !== thread?.conversationId)
)
throw forbidden("Email send must belong to the current task");
return service.queueSend(binding.companyId, input.request, {
agentId: binding.agentId,
runId: binding.runId,
});
}
+1
View File
@@ -236,6 +236,7 @@ async function sourceIssueId(
.then((rows) => rows[0] ?? null);
return { exists: Boolean(row), issueId: row?.issueId ?? null };
}
// Keep historical decision queue entries accessible after feature retirement.
case "productivity_review":
case "blocker_attention":
case "review": {
File diff suppressed because it is too large. Load diff
+319
View File
@@ -0,0 +1,319 @@
import { randomUUID } from "node:crypto";
import { and, eq, isNull, ne, sql } from "drizzle-orm";
import {
type Db,
agents,
toolConnections,
connectionGrants,
companySecrets,
toolConnectionInstalls,
} from "@paperclipai/db";
import type { EmailConnectionInput } from "@paperclipai/shared";
import { badRequest, forbidden, notFound } from "../errors.js";
import { secretService } from "./secrets.js";
import { toolAccessService } from "./tool-access.js";
import { agentmailApi } from "./agentmail-api.js";
import { logActivity } from "./activity-log.js";
import type { EmailActor } from "./email-channels.js";
export function emailConnectionService(
db: Db,
fetchImpl: typeof fetch = fetch,
) {
const secrets = secretService(db);
async function get(companyId: string, id: string, actor?: EmailActor) {
const [connection] = await db
.select()
.from(toolConnections)
.where(
and(
eq(toolConnections.companyId, companyId),
eq(toolConnections.id, id),
eq(toolConnections.status, "active"),
eq(toolConnections.enabled, true),
),
);
if (!connection || connection.config.provider !== "agentmail")
throw notFound("Active AgentMail connection not found");
if (connection.config.emailCredential) {
const id = connection.credentialSecretRefs.find((ref) => ref.configPath === "credentials.controlKey")?.secretId;
const [secret] = id ? await db.select({ id: companySecrets.id }).from(companySecrets).where(and(
eq(companySecrets.id, id), eq(companySecrets.companyId, companyId),
eq(companySecrets.status, "active"), isNull(companySecrets.deletedAt),
)) : [];
if (!secret) throw forbidden("AgentMail credential is unavailable");
}
const activeGrants = await db
.select()
.from(connectionGrants)
.where(
and(
eq(connectionGrants.connectionId, id),
eq(connectionGrants.status, "active"),
),
);
if (connection.config.emailCredential && !activeGrants.length)
throw forbidden("AgentMail credential access has been revoked");
if (actor && !actor.localImplicit) {
const grants = await db
.select()
.from(connectionGrants)
.where(
and(
eq(connectionGrants.connectionId, id),
eq(connectionGrants.status, "active"),
),
);
if (
!grants.some(
(g) =>
g.kind === "organization" ||
(g.kind === "user" && g.subjectUserId === actor.userId),
)
)
throw forbidden("You do not have access to this AgentMail credential");
}
return connection;
}
async function assertAgentAccess(
companyId: string,
id: string,
agentId: string,
) {
await get(companyId, id);
const installs = await db
.select()
.from(toolConnectionInstalls)
.where(
and(
eq(toolConnectionInstalls.companyId, companyId),
eq(toolConnectionInstalls.connectionId, id),
),
);
if (
!installs.some(
(i) =>
(i.targetType === "company" && i.targetId === companyId) ||
(i.targetType === "agent" && i.targetId === agentId),
)
)
throw forbidden(
"This agent no longer has access to the AgentMail connection",
);
}
async function credential(companyId: string, id: string, actor?: EmailActor) {
const connection = await get(companyId, id, actor);
const ref = connection.credentialSecretRefs.find(
(r) => r.configPath === "credentials.controlKey",
);
if (!ref) throw badRequest("Reconnect AgentMail to restore its API key");
const value = await secrets.resolveSecretValue(
companyId,
ref.secretId,
ref.versionSelector ?? "latest",
{
consumerType: "tool_connection",
consumerId: id,
configPath: ref.configPath,
actorType: "system",
actorId: null,
},
);
return { connection, ref, value };
}
async function connect(
companyId: string,
input: EmailConnectionInput,
actor: EmailActor,
) {
await agentmailApi(input.apiKey, fetchImpl).whoami();
return db.transaction(async (tx) => {
const db = tx as unknown as Db;
await db.execute(
sql`select pg_advisory_xact_lock(hashtextextended(${`email-account:${companyId}:${input.idempotencyKey}`}, 0))`,
);
const secrets = secretService(db);
const tools = toolAccessService(db);
for (const agentId of input.agentIds) {
const [agent] = await db
.select()
.from(agents)
.where(
and(
eq(agents.id, agentId),
eq(agents.companyId, companyId),
ne(agents.status, "terminated"),
),
);
if (!agent) throw badRequest("Select an available company agent");
}
const previous = await db
.select()
.from(toolConnections)
.where(
and(
eq(toolConnections.companyId, companyId),
eq(
toolConnections.uid,
`agentmail-account-${input.idempotencyKey}`,
),
),
);
if (previous[0])
return emailConnectionService(db, fetchImpl).get(
companyId,
previous[0].id,
actor,
);
const secret = await secrets.create(companyId, {
name: `AgentMail account ${randomUUID()}`,
provider: "local_encrypted",
value: input.apiKey,
});
const app = await tools.createApplication(companyId, {
name: `AgentMail ${input.idempotencyKey.slice(0, 8)}`,
applicationKey: `agentmail-account:${input.idempotencyKey}`,
type: "chat",
status: "active",
metadata: { sourceTemplateKey: "agentmail" },
});
const connection = await tools.createConnection(
companyId,
{
applicationId: app.id,
name: "AgentMail",
connectionKind: "managed",
connectionPurpose: "tool",
transport: "rest_api",
authKind: "api_key",
ownership: "customer",
credentialPolicy: "shared",
enabled: true,
status: "active",
config: { provider: "agentmail", emailCredential: true },
transportConfig: {},
credentialSecretRefs: [
{
secretId: secret.id,
configPath: "credentials.controlKey",
versionSelector: "latest",
required: true,
},
],
},
{
actorType: "user",
actorId: actor.userId ?? "board",
actorSource: actor.localImplicit ? "local_implicit" : "session",
},
);
await db
.update(toolConnections)
.set({
uid: `agentmail-account-${input.idempotencyKey}`,
healthStatus: "ok",
healthMessage: "Connected",
healthCheckedAt: new Date(),
})
.where(eq(toolConnections.id, connection.id));
if (input.grantKind === "user")
await db
.update(connectionGrants)
.set({
kind: "user",
isDefault: false,
subjectUserId: actor.userId ?? "board",
})
.where(eq(connectionGrants.connectionId, connection.id));
await tools.putConnectionInstalls(
connection.id,
{
installs: input.allAgents
? [{ targetType: "company", targetId: companyId }]
: input.agentIds.map((targetId) => ({
targetType: "agent" as const,
targetId,
})),
},
{
actorType: "user",
actorId: actor.userId ?? "board",
actorSource: actor.localImplicit ? "local_implicit" : "session",
},
);
await logActivity(db, {
companyId,
actorType: "user",
actorId: actor.userId ?? "board",
action: "email.connection.created",
entityType: "tool_connection",
entityId: connection.id,
details: {
grantKind: input.grantKind,
allAgents: input.allAgents,
agentIds: input.agentIds,
},
});
return tools.getConnection(connection.id, companyId);
});
}
async function allowAgent(
companyId: string,
id: string,
agentId: string,
actor: EmailActor,
) {
return db.transaction(async (tx) => {
const db = tx as unknown as Db;
await db
.select()
.from(toolConnections)
.where(
and(
eq(toolConnections.id, id),
eq(toolConnections.companyId, companyId),
),
)
.for("update");
await emailConnectionService(db, fetchImpl).get(companyId, id, actor);
const tools = toolAccessService(db);
const installs = await db
.select()
.from(toolConnectionInstalls)
.where(eq(toolConnectionInstalls.connectionId, id));
if (
installs.some(
(i) => i.targetType === "company" || i.targetId === agentId,
)
)
return;
await tools.putConnectionInstalls(
id,
{
installs: [
...installs.map((i) => ({
targetType: i.targetType,
targetId: i.targetId,
})),
{ targetType: "agent", targetId: agentId },
],
},
{
actorType: "user",
actorId: actor.userId ?? "board",
actorSource: actor.localImplicit ? "local_implicit" : "session",
},
);
await logActivity(db, {
companyId,
actorType: "user",
actorId: actor.userId ?? "board",
action: "email.connection.agent_added",
entityType: "tool_connection",
entityId: id,
details: { agentId },
});
});
}
return { get, credential, connect, allowAgent, assertAgentAccess };
}
+23 -12
View File
@@ -1,3 +1,4 @@
import { remoteTerminationReceipt } from "./remote-execution-termination.js";
import { createHash, randomUUID } from "node:crypto";
import { and, eq, inArray, sql } from "drizzle-orm";
import type { Db } from "@paperclipai/db";
@@ -634,9 +635,10 @@ export interface EnvironmentRuntimeDriver {
* current environment provider, so a provider change or an environment delete
* cannot strand the teardown. `environment` is null when a delete already
* removed the environment row. The method throws when the teardown fails, so
* the cleanup sweep keeps the row for a later retry.
* the cleanup sweep keeps the row for a later retry. Any returned provider
* receipt must be validated and persisted by the caller at lease release.
*/
retryPendingSandboxTeardown?(input: { environment: Environment | null; lease: EnvironmentLease }): Promise<void>;
retryPendingSandboxTeardown?(input: { environment: Environment | null; lease: EnvironmentLease }): Promise<unknown>;
/**
* Report whether the provider worker can run an orphan teardown now. A plugin
* sandbox provider worker can be briefly down during its own restart window.
@@ -1486,9 +1488,12 @@ function createSandboxEnvironmentDriver(
const releaseCleanedUpOrphanRow = async (
leaseId: string,
diagnosticFields: Record<string, unknown>,
receipt: unknown,
): Promise<void> => {
try {
const lease = await environmentsSvc.getLeaseById(leaseId);
await environmentsSvc.releaseLease(leaseId, "expired", {
...(lease ? { remoteExecutionTermination: remoteTerminationReceipt(lease, receipt) } : {}),
cleanupStatus: "success",
failureReason: "acquire_rejected_teardown_succeeded",
});
@@ -1564,13 +1569,14 @@ function createSandboxEnvironmentDriver(
record: DeferredOrphanCleanupRecord;
cause: unknown;
canTeardown: boolean;
teardown: () => Promise<void>;
teardown: () => Promise<unknown>;
}): Promise<void> => {
const durable = await tryWriteDurablePendingCleanup(input.record);
let teardownFailed = !input.canTeardown;
let receipt: unknown;
if (!teardownFailed) {
try {
await input.teardown();
receipt = await input.teardown();
} catch {
teardownFailed = true;
}
@@ -1578,7 +1584,7 @@ function createSandboxEnvironmentDriver(
if (!teardownFailed) {
// The teardown removed the orphan, so drop the durable row if we wrote one.
if (durable.leaseId !== null) {
await releaseCleanedUpOrphanRow(durable.leaseId, orphanDiagnosticFields(input.record));
await releaseCleanedUpOrphanRow(durable.leaseId, orphanDiagnosticFields(input.record), receipt);
}
return;
}
@@ -2159,7 +2165,7 @@ function createSandboxEnvironmentDriver(
cause: error,
canTeardown: pluginWorkerManager.isRunning(pluginProvider.resolved.plugin.id),
teardown: async () => {
await pluginWorkerManager.call(
return await pluginWorkerManager.call(
pluginProvider.resolved.plugin.id,
"environmentDestroyLease",
{
@@ -2528,7 +2534,7 @@ function createSandboxEnvironmentDriver(
{ issueId: input.lease.issueId, heartbeatRunId: input.lease.heartbeatRunId },
);
const workerConfig = stripSandboxProviderEnvelope(config as SandboxEnvironmentConfig);
await pluginWorkerManager.call(
return await pluginWorkerManager.call(
pluginProvider.resolved.plugin.id,
"environmentDestroyLease",
{
@@ -2545,7 +2551,6 @@ function createSandboxEnvironmentDriver(
},
resolvePluginSandboxRpcTimeoutMs(workerConfig),
);
return;
}
// Built-in provider path. Resolve the recorded config secrets through the
@@ -3045,6 +3050,7 @@ function createSandboxEnvironmentDriver(
const providerKey = readString(metadata.provider);
let cleanupStatus: "success" | "failed" = "success";
let termination: ReturnType<typeof remoteTerminationReceipt>;
if (
pluginId &&
providerKey &&
@@ -3057,7 +3063,7 @@ function createSandboxEnvironmentDriver(
lease: input.lease,
provider: providerKey,
});
await runLeaseReleaseWithRunParent(input.lease.id, () =>
const receipt = await runLeaseReleaseWithRunParent(input.lease.id, () =>
pluginWorkerManager.call(pluginId, "environmentReleaseLease", {
driverKey: providerKey,
companyId: input.lease.companyId,
@@ -3068,6 +3074,7 @@ function createSandboxEnvironmentDriver(
leaseMetadata: metadata,
}, resolvePluginSandboxRpcTimeoutMs(stripSandboxProviderEnvelope(config as SandboxEnvironmentConfig))),
);
termination = remoteTerminationReceipt(input.lease, receipt);
} catch {
cleanupStatus = "failed";
}
@@ -3096,6 +3103,7 @@ function createSandboxEnvironmentDriver(
return await environmentsSvc.releaseLease(input.lease.id, releaseStatus, {
failureReason,
cleanupStatus,
...(cleanupStatus === "success" && termination ? { remoteExecutionTermination: termination } : {}),
});
}
@@ -3106,6 +3114,7 @@ function createSandboxEnvironmentDriver(
}): Promise<EnvironmentLease | null> {
if (await retainUnsavedWorkFolderLease(db, input.lease)) return { ...input.lease, status: "retained", expiresAt: null, failureReason: "work_folder_save_required" };
let cleanupStatus: "success" | "failed" = "success";
let termination: ReturnType<typeof remoteTerminationReceipt>;
const metadata = input.lease.metadata ?? {};
try {
@@ -3125,7 +3134,7 @@ function createSandboxEnvironmentDriver(
lease: input.lease,
provider: providerKey,
});
await runLeaseReleaseWithRunParent(input.lease.id, () =>
const receipt = await runLeaseReleaseWithRunParent(input.lease.id, () =>
pluginWorkerManager.call(pluginId, "environmentDestroyLease", {
driverKey: providerKey,
companyId: input.lease.companyId,
@@ -3136,6 +3145,7 @@ function createSandboxEnvironmentDriver(
leaseMetadata: metadata,
}, resolvePluginSandboxRpcTimeoutMs(stripSandboxProviderEnvelope(config as SandboxEnvironmentConfig))),
);
termination = remoteTerminationReceipt(input.lease, receipt);
}
} else {
const metadataConfig = sandboxConfigFromLeaseMetadata(input.lease);
@@ -3165,6 +3175,7 @@ function createSandboxEnvironmentDriver(
{
failureReason: input.failureReason,
cleanupStatus,
...(cleanupStatus === "success" && termination ? { remoteExecutionTermination: termination } : {}),
},
);
}
@@ -3856,14 +3867,14 @@ export function environmentRuntimeService(
async retryPendingSandboxTeardown(input: {
environment: Environment | null;
lease: EnvironmentLease;
}): Promise<void> {
}): Promise<unknown> {
const driver = requireDriverKey(getLeaseDriverKey(input.lease, input.environment));
if (!driver.retryPendingSandboxTeardown) {
throw new Error(
`Environment driver "${driver.driver}" does not support orphan sandbox teardown.`,
);
}
await driver.retryPendingSandboxTeardown(input);
return await driver.retryPendingSandboxTeardown(input);
},
// Report whether the provider worker can run an orphan teardown now. The
+6
View File
@@ -1617,6 +1617,7 @@ export function environmentService(db: Db) {
options?: {
failureReason?: string;
cleanupStatus?: EnvironmentLeaseCleanupStatus;
remoteExecutionTermination?: Record<string, unknown>;
},
) => {
const now = new Date();
@@ -1629,6 +1630,11 @@ export function environmentService(db: Db) {
updatedAt: now,
...(options?.failureReason !== undefined ? { failureReason: options.failureReason } : {}),
...(options?.cleanupStatus !== undefined ? { cleanupStatus: options.cleanupStatus } : {}),
// A later release without a receipt cannot reuse an earlier stop's
// authority (for example after a same-run lease resume).
metadata: options?.remoteExecutionTermination
? sql`coalesce(${environmentLeases.metadata}, '{}'::jsonb) || ${JSON.stringify({ remoteExecutionTermination: options.remoteExecutionTermination })}::jsonb`
: sql`${environmentLeases.metadata} - 'remoteExecutionTermination'`,
})
.where(eq(environmentLeases.id, id))
.returning()
@@ -1,15 +1,16 @@
import { remoteTerminationReceipt } from "./remote-execution-termination.js";
import { randomUUID } from "node:crypto";
import { and, eq } from "drizzle-orm";
import { beforeAll, afterAll, describe, it, expect } from "vitest";
import {
approvals, issueApprovals, issueThreadInteractions,
agentWakeupRequests, agents, companies, createDb, heartbeatRuns, issueComments, issueRecoveryActions,
agentWakeupRequests, agents, companies, createDb, heartbeatRunEvents, heartbeatRuns, issueComments, issueRecoveryActions,
issues, nativeRunFinalizations, environmentLeases, environments, issueRelations, issueTreeHolds, issueTreeHoldMembers,
} from "@paperclipai/db";
import { startEmbeddedPostgresTestDatabase, getEmbeddedPostgresTestSupport } from "../__tests__/helpers/embedded-postgres.js";
import { admitExplicitNativeContinuation } from "./explicit-native-continuation.js";
import { buildExecutionContinuation } from "./execution-continuation.js";
import { heartbeatService } from "./heartbeat.js";
import { heartbeatService, type HeartbeatEnvironmentRuntime } from "./heartbeat.js";
import { getExecutionBlocker } from "./execution-blocker.js";
const support = await getEmbeddedPostgresTestSupport();
(support.supported ? describe : describe.skip)("explicit native conversation continuation", () => {
@@ -46,6 +47,107 @@ const support = await getEmbeddedPostgresTestSupport();
agentId: f.agentId, status: "queued", contextSnapshot: { issueId: f.issueId, previousRunId: result.previousRunId, forceFreshSession: true } });
return result;
});
it.each([true, false])("acknowledges a legacy remote Stop only after confirmed lease cleanup: %s", async confirmed => {
const f = await seed();
await db.update(agents).set({ adapterType: "claude_local" }).where(eq(agents.id, f.agentId));
await db.update(heartbeatRuns).set({ runtimeMode: "legacy", status: "cancelled", processPid: null,
resultJson: { executionCancellation: { state: "requested" } },
}).where(eq(heartbeatRuns.id, f.sourceRunId));
await db.insert(heartbeatRunEvents).values({ companyId: f.companyId, runId: f.sourceRunId,
agentId: f.agentId, seq: 1, eventType: "adapter.invoke", payload: { adapterType: "claude_local" } });
await db.update(issueRecoveryActions).set({ cause: "legacy_execution_requires_reconciliation" })
.where(eq(issueRecoveryActions.sourceIssueId, f.issueId));
const [environment] = await db.insert(environments).values({ name: `Remote ${f.sourceRunId}`, driver: "sandbox" }).returning();
const identity = { id: randomUUID(), companyId: f.companyId, heartbeatRunId: f.sourceRunId,
provider: "daytona", providerLeaseId: "sandbox-legacy" };
await db.insert(environmentLeases).values({ ...identity, environmentId: environment.id,
status: "expired", leasePolicy: "ephemeral", releasedAt: new Date(), cleanupStatus: "success",
metadata: confirmed ? { remoteExecutionTermination: remoteTerminationReceipt(identity,
{ providerLeaseId: identity.providerLeaseId, state: "destroyed" }) } : {},
});
await heartbeatService(db).releaseEnvironmentLeasesForRun({ runId: f.sourceRunId,
companyId: f.companyId, agentId: f.agentId, status: "cancelled" });
const [run] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.sourceRunId));
expect(run.resultJson?.executionCancellation).toMatchObject({ state: confirmed ? "acknowledged" : "requested" });
if (confirmed) expect(await getExecutionBlocker(db, f.companyId, f.issueId)).toBeNull();
else expect(await getExecutionBlocker(db, f.companyId, f.issueId)).not.toBeNull();
});
it.each(["stopped", "destroyed", "missing", "wrong_lease", "cleanup_failed", "active"])(
"admits a remote native predecessor only with confirmed termination: %s", async kind => {
const f = await seed();
// This PID exists on the control-plane host. It must never be used to
// infer liveness of the identically numbered remote process.
await db.update(heartbeatRuns).set({ processPid: process.pid }).where(eq(heartbeatRuns.id, f.sourceRunId));
const [environment] = await db.insert(environments).values({ name: `Remote ${f.sourceRunId}`, driver: "sandbox" }).returning();
const identity = { id: randomUUID(), companyId: f.companyId, heartbeatRunId: f.sourceRunId,
provider: "daytona", providerLeaseId: "sandbox-1" };
const proof = remoteTerminationReceipt(identity, { providerLeaseId: "sandbox-1",
state: kind === "destroyed" ? "destroyed" : "stopped" });
await db.insert(environmentLeases).values({ ...identity, environmentId: environment.id,
issueId: f.issueId, status: kind === "active" ? "active" : "released", leasePolicy: "ephemeral",
releasedAt: kind === "active" ? null : new Date(), cleanupStatus: kind === "cleanup_failed" ? "failed" : "success",
metadata: kind === "missing" ? {} : { remoteExecutionTermination:
kind === "wrong_lease" ? { ...proof, providerLeaseId: "other-sandbox" } : proof },
});
const result = await admit(f);
if (["stopped", "destroyed"].includes(kind)) expect(result).toMatchObject({ previousRunId: f.sourceRunId });
else expect(result).toBeNull();
},
);
it.each([
{ runtime: "native", retry: false }, { runtime: "native", retry: true },
{ runtime: "legacy", retry: false }, { runtime: "legacy", retry: true },
])("resumes a user message after confirmed cleanup: %j", async ({ runtime, retry }) => {
const f = await seed();
if (runtime === "legacy") {
await db.update(agents).set({ adapterType: "claude_local" }).where(eq(agents.id, f.agentId));
await db.update(heartbeatRuns).set({ runtimeMode: "legacy", status: "cancelled", processPid: null,
resultJson: { executionCancellation: { state: "requested" } } }).where(eq(heartbeatRuns.id, f.sourceRunId));
await db.insert(heartbeatRunEvents).values({ companyId: f.companyId, runId: f.sourceRunId,
agentId: f.agentId, seq: 1, eventType: "adapter.invoke", payload: { adapterType: "claude_local" } });
await db.update(issueRecoveryActions).set({ cause: "legacy_execution_requires_reconciliation" })
.where(eq(issueRecoveryActions.sourceIssueId, f.issueId));
}
// Keep the successor queued so this test never starts an actual provider.
await db.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: f.agentId, status: "running" });
const [environment] = await db.insert(environments).values({ name: `pending-${f.sourceRunId}`, driver: "sandbox" }).returning();
const identity = { id: randomUUID(), companyId: f.companyId, heartbeatRunId: f.sourceRunId,
provider: "daytona", providerLeaseId: "pending-sandbox" };
await db.insert(environmentLeases).values({ ...identity, environmentId: environment.id, status: "active", leasePolicy: "ephemeral" });
const heartbeat = heartbeatService(db, retry ? { environmentRuntime: {
retryPendingSandboxTeardown: async () => ({ providerLeaseId: identity.providerLeaseId, state: "destroyed" }),
} as unknown as HeartbeatEnvironmentRuntime } : {});
await heartbeat.wakeup(f.agentId, { source: "automation", triggerDetail: "system", reason: "issue_commented",
requestedByActorType: "user", requestedByActorId: "board", payload: { issueId: f.issueId, commentId: f.commentId },
contextSnapshot: { issueId: f.issueId, wakeCommentId: f.commentId } });
const [source] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.sourceRunId));
await heartbeat.resumeRemoteStopComments(source);
expect(await getExecutionBlocker(db, f.companyId, f.issueId)).not.toBeNull();
if (retry) {
await db.update(environmentLeases).set({ status: "pending_cleanup", cleanupStatus: "failed" })
.where(eq(environmentLeases.id, identity.id));
expect(await heartbeat.sweepPendingCleanupLeases()).toMatchObject({ destroyed: 1 });
const [lease] = await db.select().from(environmentLeases).where(eq(environmentLeases.id, identity.id));
expect(lease.metadata?.remoteExecutionTermination).toMatchObject({ runId: f.sourceRunId, state: "destroyed" });
} else {
await db.update(environmentLeases).set({ status: "released", releasedAt: new Date(), cleanupStatus: "success",
metadata: { remoteExecutionTermination: remoteTerminationReceipt(identity,
{ providerLeaseId: identity.providerLeaseId, state: "stopped" }) } }).where(eq(environmentLeases.id, identity.id));
await heartbeat.releaseEnvironmentLeasesForRun({ runId: source.id, companyId: source.companyId,
agentId: source.agentId, status: source.status });
}
await heartbeat.resumeRemoteStopComments(source);
await heartbeat.resumeRemoteStopComments(source);
const runs = await db.select().from(heartbeatRuns).where(and(eq(heartbeatRuns.companyId, f.companyId), eq(heartbeatRuns.status, "queued")));
expect(runs).toHaveLength(1);
if (runtime === "native") expect(runs[0].contextSnapshot).toMatchObject({ forceFreshSession: true, previousRunId: f.sourceRunId,
explicitUserContinuation: { commentId: f.commentId } });
else expect(runs[0].contextSnapshot).toMatchObject({ wakeCommentId: f.commentId });
expect(await getExecutionBlocker(db, f.companyId, f.issueId)).toBeNull();
});
it("queues the actual user wake with a fresh session and retained source context", async () => {
const f = await seed();
// Occupy this agent's only slot so this admission test never starts a provider.
@@ -1,3 +1,5 @@
import { completeTerminatedRemoteNativeSessionCleanup } from "../vendor/paperclip-runner/index.js";
import { hasRemoteTerminationReceipt, remoteLeaseCleanupScope } from "./remote-execution-termination.js";
import { z } from "zod";
import { and, eq, inArray, isNull, ne, or, sql } from "drizzle-orm";
import {
@@ -76,24 +78,31 @@ export async function admitExplicitNativeContinuation(input: {
run.errorCode === "execution_reconciliation_required" &&
!run.processPid && !run.processGroupId && !run.nativeSessionId;
if (run.runtimeMode !== "native" && !unusedAdmission) return null;
if (!unusedAdmission) {
// A missing process identity is not evidence that a provider exited.
if (!run.processPid && !run.processGroupId) return null;
if (run.processPid && !processStopped(run.processPid)) return null;
if (run.processGroupId && !processStopped(-run.processGroupId)) return null;
}
const [coordinator] = await db.select().from(nativeRunFinalizations).where(and(
eq(nativeRunFinalizations.companyId, companyId), eq(nativeRunFinalizations.runId, run.id),
)).for("update");
if (coordinator && (coordinator.phase !== "terminal_failure" || coordinator.leaseOwner ||
coordinator.resultId || coordinator.failureDetail?.successorRunId)) return null;
const leases = await db.select({ provider: environmentLeases.provider, releasedAt: environmentLeases.releasedAt })
const leases = await db.select()
.from(environmentLeases).where(and(
eq(environmentLeases.companyId, companyId), eq(environmentLeases.heartbeatRunId, run.id),
));
// A PID on another host cannot be checked with this server's process table.
// Remote execution retains its hold until a target-aware stop proof exists.
if (leases.some(lease => !lease.releasedAt || lease.provider !== "local")) return null;
const remote = leases.some(lease => lease.provider !== "local");
if (remote) {
// Never interpret remote PIDs using the control-plane host's process table.
if (!leases.every(hasRemoteTerminationReceipt)) return null;
if (!input.dryRun && !leases.every(lease => completeTerminatedRemoteNativeSessionCleanup({
companyId, runId: run.id, remoteCleanupScope: remoteLeaseCleanupScope(lease)!,
}))) return null;
} else {
if (leases.some(lease => !lease.releasedAt || lease.cleanupStatus === "failed")) return null;
if (!unusedAdmission) {
// A missing process identity is not evidence that a provider exited.
if (!run.processPid && !run.processGroupId) return null;
if (run.processPid && !processStopped(run.processPid)) return null;
if (run.processGroupId && !processStopped(-run.processGroupId)) return null;
}
}
sources.push(run);
}
const nativeSources = sources.filter(run => run.runtimeMode === "native");
+114 -45
View File
@@ -1,3 +1,6 @@
import { completeTerminatedRemoteNativeSessionCleanup } from "../vendor/paperclip-runner/index.js";
import { remoteExecutionHasStopped, remoteTerminationReceipt, stoppedRemoteCleanupScopes } from "./remote-execution-termination.js";
import { applyConnectorSkills, prepareConnectorSkillDelivery, resolveConnectorAssignments } from "./connector-runtime.js";
import { admitExplicitNativeContinuation } from "./explicit-native-continuation.js";
import { getExecutionBlocker } from "./execution-blocker.js";
import { CONVERSATION_CONTINUATION_POLICY, runUsedConversationAdapter, hasConversationContinuationPolicy, isConversationAdapter } from "./conversation-continuation.js";
@@ -504,7 +507,6 @@ import {
REVIEW_PATH_RECOVERY_INSTRUCTION,
reviewPathConsumedRefFromRun,
} from "./recovery/review-path-recovery.js";
import { productivityReviewService } from "./productivity-review.js";
import { resolveRequiredSuccessfulRunHandoffOnValidPath } from "./successful-run-handoff-state.js";
import { taskWatchdogService } from "./task-watchdogs.js";
import { withAgentStartLock } from "./agent-start-lock.js";
@@ -9728,7 +9730,6 @@ export function heartbeatService(
return interaction.id;
}
const productivityReviews = productivityReviewService(db, { enqueueWakeup });
const taskWatchdogs = taskWatchdogService(db, { enqueueWakeup });
let unsafeTextProjectionPromise: Promise<boolean> | null = null;
@@ -9874,7 +9875,19 @@ export function heartbeatService(
);
return { closed: 0, busy: 0, failed: 1 };
});
if (closeResult.busy > 0 || closeResult.failed > 0) {
// A failed remote checkpoint cannot veto destruction of the isolated
// sandbox after the run stopped. Provider destruction supplies the exit
// proof; it does not turn the interrupted checkpoint into a success.
const remoteLeases = closeResult.failed > 0 && leaseOwnerRun &&
["cancelled", "failed", "timed_out", "interrupted"].includes(leaseOwnerRun.status)
? await db.select({ provider: environmentLeases.provider }).from(environmentLeases).where(and(
eq(environmentLeases.companyId, input.companyId),
eq(environmentLeases.heartbeatRunId, input.runId),
))
: [];
const canDestroyRemote = remoteLeases.length > 0 &&
remoteLeases.every(lease => lease.provider && lease.provider !== "local");
if (closeResult.busy > 0 || (closeResult.failed > 0 && !canDestroyRemote)) {
logger.warn(
{ runId: input.runId, warmNativeSessions: closeResult },
"deferred environment lease destruction until warm native sessions close",
@@ -9909,6 +9922,76 @@ export function heartbeatService(
"failed to release environment lease for heartbeat run",
);
}
await acknowledgeRemoteStop(input.runId, input.companyId);
}
async function acknowledgeRemoteStop(runId: string, companyId: string) {
// The provider receipt arrives after adapter settlement. A remote ACP child
// has no host PID, so only this target-aware boundary can acknowledge Stop.
const stopped = await getRun(runId);
if (stopped?.runtimeMode === "native") {
const scopes = await stoppedRemoteCleanupScopes(db, companyId, runId);
for (const remoteCleanupScope of scopes ?? []) {
completeTerminatedRemoteNativeSessionCleanup({ companyId, runId, remoteCleanupScope });
}
}
if (stopped?.runtimeMode === "legacy" && stopped.status === "cancelled" &&
parseObject(stopped.resultJson?.executionCancellation).state === "requested" &&
await runUsedConversationAdapter(db, stopped) &&
await remoteExecutionHasStopped(db, companyId, runId)) {
await db.update(heartbeatRuns).set({
resultJson: sql`coalesce(${heartbeatRuns.resultJson}, '{}'::jsonb) || ${JSON.stringify({
executionCancellation: { ...parseObject(stopped.resultJson?.executionCancellation),
state: "acknowledged", acknowledgedAt: new Date().toISOString(),
proof: "provider_termination_receipt" },
conversationContinuation: CONVERSATION_CONTINUATION_POLICY,
})}::jsonb`,
updatedAt: new Date(),
}).where(and(eq(heartbeatRuns.id, runId), eq(heartbeatRuns.companyId, companyId),
eq(heartbeatRuns.status, "cancelled")));
}
}
async function resumeRemoteStopComments(run: typeof heartbeatRuns.$inferSelect) {
if (!isHeartbeatRunTerminalStatus(run.status) || adapterExecutionControls.has(run.id) ||
!(await remoteExecutionHasStopped(db, run.companyId, run.id))) return;
const issueId = run.nativeIssueId ?? (typeof run.contextSnapshot?.issueId === "string" ? run.contextSnapshot.issueId : null);
if (!issueId) return;
const legacyContinuation = run.runtimeMode === "legacy" && run.status === "cancelled" &&
hasConversationContinuationPolicy((await getRun(run.id))?.resultJson) &&
!(await getExecutionBlocker(db, run.companyId, issueId));
if (run.runtimeMode !== "native" && !legacyContinuation) return;
const pending = await db.select().from(agentWakeupRequests).where(and(
eq(agentWakeupRequests.companyId, run.companyId), eq(agentWakeupRequests.agentId, run.agentId),
eq(agentWakeupRequests.status, "deferred_issue_execution"),
eq(agentWakeupRequests.requestedByActorType, "user"),
sql`${agentWakeupRequests.payload}->>'issueId' = ${issueId}`,
)).orderBy(asc(agentWakeupRequests.requestedAt));
for (const wake of pending) {
const payload = parseObject(wake.payload);
const context = parseObject(payload[DEFERRED_WAKE_CONTEXT_KEY]);
const commentId = deriveCommentId(context, payload);
if (legacyContinuation) {
if (!commentId || !run.finishedAt || !wake.requestedByActorId ||
!["issue_commented", "issue_reopened_via_comment"].includes(wake.reason ?? "")) continue;
const [comment] = await db.select().from(issueComments).where(and(
eq(issueComments.companyId, run.companyId), eq(issueComments.issueId, issueId),
sql`${issueComments.id}::text = ${commentId}`, eq(issueComments.authorType, "user"),
eq(issueComments.authorUserId, wake.requestedByActorId), isNull(issueComments.deletedAt),
isNull(issueComments.createdByRunId), gt(issueComments.createdAt, run.finishedAt),
));
if (!comment?.body.trim()) continue;
} else if (!await admitExplicitNativeContinuation({ db, companyId: run.companyId, issueId,
agentId: run.agentId, actorType: wake.requestedByActorType, actorId: wake.requestedByActorId,
reason: wake.reason, commentId, successorRunId: randomUUID(), dryRun: true })) continue;
// Re-enter ordinary admission with the original user's authority. It
// atomically adopts the deferred comments and still applies every gate.
await enqueueWakeup(run.agentId, { source: wake.source as WakeupOptions["source"], triggerDetail: (wake.triggerDetail ?? undefined) as WakeupOptions["triggerDetail"],
reason: wake.reason, payload, contextSnapshot: context,
requestedByActorType: "user", requestedByActorId: wake.requestedByActorId,
idempotencyKey: `remote-stop-comment:${run.id}:${wake.id}` });
break;
}
}
async function hasUnsafeTextProjectionDatabase() {
@@ -12514,29 +12597,6 @@ export function heartbeatService(
projectId: issue.projectId,
})
: null;
if (issue) {
const productivityHold =
await productivityReviews.isProductivityReviewContinuationHoldActive({
companyId: issue.companyId,
issueId: issue.id,
agentId: run.agentId,
});
if (productivityHold.held) {
await setRunStatus(run.id, run.status, {
livenessReason: `${run.livenessReason ?? "Run ended without concrete progress"}; continuation held by productivity review ${productivityHold.reviewIdentifier ?? productivityHold.reviewIssueId}`,
});
await productivityReviews.recordContinuationHold({
companyId: issue.companyId,
issueId: issue.id,
runId: run.id,
agentId: run.agentId,
reviewIssueId: productivityHold.reviewIssueId,
trigger: productivityHold.trigger,
reason: productivityHold.reason,
});
return;
}
}
const nextAttempt = readContinuationAttempt(run.continuationAttempt) + 1;
const idempotencyKey = issue
@@ -17719,15 +17779,16 @@ export function heartbeatService(
try {
if (useRecordedTeardown) {
// Tear the sandbox down from the recorded provider config and the
// cleanup-authorized secret versions. The teardown returns no value
// and throws on failure, so the sweep releases the lease itself.
await environmentRuntime.retryPendingSandboxTeardown({
// cleanup-authorized secret versions. Preserve any provider receipt;
// a completed retry must grant the same evidence as initial cleanup.
const receipt = await environmentRuntime.retryPendingSandboxTeardown({
environment,
lease,
});
await environmentsSvc.releaseLease(lease.id, "expired", {
cleanupStatus: "success",
failureReason: "pending_cleanup_retry",
remoteExecutionTermination: remoteTerminationReceipt(lease, receipt),
});
destroyed += 1;
} else if (environment) {
@@ -17764,6 +17825,15 @@ export function heartbeatService(
"pending_cleanup lease retry failed",
);
}
if (lease.heartbeatRunId) {
// Delivery failure must not revert successful provider cleanup. A new
// message can still use the persisted receipt on its next admission.
await (async () => {
await acknowledgeRemoteStop(lease.heartbeatRunId!, lease.companyId);
const run = await getRun(lease.heartbeatRunId!);
if (run) await resumeRemoteStopComments(run);
})().catch(() => logger.warn({ leaseId: lease.id }, "could not reconsider messages after cleanup retry"));
}
}
return { swept: rows.length, destroyed, capped };
@@ -18625,16 +18695,6 @@ export function heartbeatService(
});
}
async function reconcileProductivityReviews(opts?: {
now?: Date;
companyId?: string;
}) {
return productivityReviews.reconcileProductivityReviews({
...opts,
issueCreatedAtGte: await getWorktreeExecutionCutoff(),
});
}
async function reconcileTaskWatchdogs(opts?: {
companyId?: string | null;
runId?: string | null;
@@ -20182,10 +20242,14 @@ export function heartbeatService(
startedAtMs: skillsPrepareStartedAtMs,
endedAtMs: Date.now(),
});
let runtimeConfig: Record<string, unknown> = {
...effectiveResolvedConfig,
paperclipRuntimeSkills: runtimeSkillEntries,
};
const connectorAssignments = await resolveConnectorAssignments(db, { companyId: agent.companyId, agentId: agent.id });
const connectorSkillConfig = await applyConnectorSkills(effectiveResolvedConfig, runtimeSkillEntries, connectorAssignments);
// Both CLI adapters and native context materialization use the same resolved set.
runtimeSkillEntries.splice(0, runtimeSkillEntries.length, ...connectorSkillConfig.paperclipRuntimeSkills);
const connectorDelivery = await prepareConnectorSkillDelivery(connectorSkillConfig, agent.adapterType);
// Always replace this runtime-only field; caller wake data cannot supply skills.
context.paperclipWake = { ...parseObject(context.paperclipWake), connectorSkillInstructions: connectorDelivery.instructions };
let runtimeConfig: Record<string, unknown> = connectorDelivery.config;
const latestAgentConfigRevision = await getLatestAgentConfigRevision(
agent.companyId,
agent.id,
@@ -22900,6 +22964,7 @@ export function heartbeatService(
executePaperclipNativeSession({
db,
execution: nativeExecution,
turnTimeoutMs: Math.max(0, asNumber(runtimeConfig.timeoutSec, 0)) * 1_000,
runnerInstanceId: nativeRunnerInstanceId,
leaseOwner: runOptions.nativeLeaseOwner,
restartRecovery: runOptions.nativeRestartRecovery,
@@ -24815,6 +24880,9 @@ export function heartbeatService(
!nativeWorkspaceFinalizeScheduled &&
!shutdownInProgress
) {
if (latestRun) await resumeRemoteStopComments(latestRun).catch(err => {
logger.warn({ err, runId: run.id }, "failed to resume user messages after remote Stop");
});
await startNextQueuedRunForAgent(run.agentId);
}
}
@@ -25451,8 +25519,10 @@ export function heartbeatService(
const [chatBinding] = await tx
.select({ id: chatConversations.id })
.from(chatConversations)
.innerJoin(chatEndpoints, eq(chatEndpoints.id, chatConversations.endpointId))
.where(
and(
eq(chatEndpoints.externalExecutionPolicy, "restricted"),
eq(chatConversations.companyId, agent.companyId),
eq(chatConversations.issueId, issueId),
),
@@ -28019,6 +28089,7 @@ export function heartbeatService(
terminalizeRunOnLeaseRelease,
releaseEnvironmentLeasesForRun,
resumeRemoteStopComments,
sweepStaleIssueLocks,
@@ -28026,8 +28097,6 @@ export function heartbeatService(
scanSilentActiveRuns,
reconcileProductivityReviews,
reconcileTaskWatchdogs,
buildRunOutputSilence,
-4
View File
@@ -114,10 +114,6 @@ export {
RunnerGoalActionError,
RunnerGoalConflictError,
} from "./runner-goals.js";
export {
productivityReviewService,
PRODUCTIVITY_REVIEW_ORIGIN_KIND,
} from "./productivity-review.js";
export { classifyIssueGraphLiveness, type IssueLivenessFinding } from "./recovery/index.js";
export { dashboardService } from "./dashboard.js";
export { sidebarBadgeService } from "./sidebar-badges.js";
+6 -161
View File
@@ -79,8 +79,6 @@ import type {
IssueReviewAttentionPath,
IssueBlockedInboxAttention,
IssueBlockedInboxIssueRef,
IssueProductivityReview,
IssueProductivityReviewTrigger,
IssueRelationIssueSummary,
IssueWatchdogSummary,
LowTrustBoundary,
@@ -1114,6 +1112,10 @@ export async function resolveChatOriginPublicationBindings(
runId: string | null,
): Promise<ChatPublicationBinding[]> {
if (!runId) return [];
const explicitEmail = await dbOrTx.select({ id: chatEndpoints.id }).from(chatEndpoints)
.innerJoin(chatConversations, eq(chatConversations.endpointId, chatEndpoints.id))
.where(and(eq(chatConversations.companyId, companyId), eq(chatConversations.issueId, issueId), eq(chatEndpoints.publicationMode, "explicit"))).limit(1);
if (explicitEmail.length) return [];
let originRunId = runId;
let contextSnapshot: Record<string, unknown> | null = null;
@@ -3238,15 +3240,6 @@ const BLOCKER_ATTENTION_PENDING_APPROVAL_STATUSES = [
const BLOCKER_ATTENTION_OPEN_RECOVERY_ORIGIN_KIND =
"harness_liveness_escalation";
const BLOCKER_ATTENTION_CHILD_TERMINAL_STATUSES = ["done", "cancelled"];
const PRODUCTIVITY_REVIEW_ORIGIN_KIND = "issue_productivity_review";
const PRODUCTIVITY_REVIEW_TERMINAL_STATUSES = ["done", "cancelled"];
const PRODUCTIVITY_REVIEW_ACTIVITY_ACTIONS = [
"issue.productivity_review_created",
"issue.productivity_review_updated",
];
const PRODUCTIVITY_REVIEW_TRIGGERS: readonly IssueProductivityReviewTrigger[] =
["no_comment_streak", "long_active_duration", "high_churn"];
function lowTrustBoundaryIssueCondition(
companyId: string,
boundary: (LowTrustBoundary & { companyId: string }) | null | undefined,
@@ -3628,132 +3621,6 @@ async function terminalExplicitBlockersByRoot(
return terminalByRoot;
}
function readProductivityReviewTrigger(
value: unknown,
): IssueProductivityReviewTrigger | null {
if (typeof value !== "string") return null;
return PRODUCTIVITY_REVIEW_TRIGGERS.includes(
value as IssueProductivityReviewTrigger,
)
? (value as IssueProductivityReviewTrigger)
: null;
}
function readProductivityReviewStreak(value: unknown): number | null {
if (typeof value !== "number" || !Number.isFinite(value) || value < 0)
return null;
return Math.floor(value);
}
async function listIssueProductivityReviewMap(
dbOrTx: any,
companyId: string,
sourceIssueIds: string[],
): Promise<Map<string, IssueProductivityReview>> {
const map = new Map<string, IssueProductivityReview>();
if (sourceIssueIds.length === 0) return map;
const reviewRows: Array<{
sourceIssueId: string | null;
reviewIssueId: string;
reviewIdentifier: string | null;
status: string;
priority: string;
createdAt: Date;
updatedAt: Date;
}> = [];
for (const chunk of chunkList(
[...new Set(sourceIssueIds)],
ISSUE_LIST_RELATED_QUERY_CHUNK_SIZE,
)) {
const rows = await dbOrTx
.select({
sourceIssueId: issues.originId,
reviewIssueId: issues.id,
reviewIdentifier: issues.identifier,
status: issues.status,
priority: issues.priority,
createdAt: issues.createdAt,
updatedAt: issues.updatedAt,
})
.from(issues)
.where(
and(
eq(issues.companyId, companyId),
eq(issues.originKind, PRODUCTIVITY_REVIEW_ORIGIN_KIND),
inArray(issues.originId, chunk),
visibleIssueCondition(),
notInArray(issues.status, PRODUCTIVITY_REVIEW_TERMINAL_STATUSES),
),
)
.orderBy(desc(issues.createdAt), desc(issues.id));
reviewRows.push(...rows);
}
if (reviewRows.length === 0) return map;
const reviewIssueIds = reviewRows.map((row) => row.reviewIssueId);
const triggerByReviewIssueId = new Map<
string,
{
trigger: IssueProductivityReviewTrigger | null;
noCommentStreak: number | null;
}
>();
for (const chunk of chunkList(
reviewIssueIds,
ISSUE_LIST_RELATED_QUERY_CHUNK_SIZE,
)) {
const detailRows = await dbOrTx
.select({
entityId: activityLog.entityId,
details: activityLog.details,
createdAt: activityLog.createdAt,
})
.from(activityLog)
.where(
and(
eq(activityLog.companyId, companyId),
eq(activityLog.entityType, "issue"),
inArray(activityLog.entityId, chunk),
inArray(activityLog.action, PRODUCTIVITY_REVIEW_ACTIVITY_ACTIONS),
),
)
.orderBy(desc(activityLog.createdAt));
for (const row of detailRows as Array<{
entityId: string;
details: Record<string, unknown> | null;
createdAt: Date;
}>) {
if (triggerByReviewIssueId.has(row.entityId)) continue;
triggerByReviewIssueId.set(row.entityId, {
trigger: readProductivityReviewTrigger(row.details?.trigger),
noCommentStreak: readProductivityReviewStreak(
row.details?.noCommentStreak,
),
});
}
}
for (const row of reviewRows) {
if (!row.sourceIssueId) continue;
if (map.has(row.sourceIssueId)) continue;
const detail = triggerByReviewIssueId.get(row.reviewIssueId);
map.set(row.sourceIssueId, {
reviewIssueId: row.reviewIssueId,
reviewIdentifier: row.reviewIdentifier,
status: row.status as IssueProductivityReview["status"],
priority: row.priority as IssueProductivityReview["priority"],
trigger: detail?.trigger ?? null,
noCommentStreak: detail?.noCommentStreak ?? null,
createdAt: row.createdAt,
updatedAt: row.updatedAt,
});
}
return map;
}
async function listIssueBlockerAttentionMap(
dbOrTx: any,
companyId: string,
@@ -6369,7 +6236,6 @@ async function listBlockedInboxIssues(
blockerAttention?: IssueBlockerAttention;
reviewAttention?: IssueReviewAttention;
blockedInboxAttention: IssueBlockedInboxAttention;
productivityReview?: IssueProductivityReview | null;
liveDescendantCount?: number;
lastActivityAt: Date;
myLastTouchAt?: Date | null;
@@ -6418,7 +6284,6 @@ async function listBlockedInboxIssues(
blockedByMap,
blockerAttentionByIssueId,
reviewAttentionByIssueId,
productivityReviewByIssueId,
blockedInboxAttentionByIssueId,
liveDescendantCountByIssueId,
] = await Promise.all([
@@ -6432,7 +6297,6 @@ async function listBlockedInboxIssues(
blockedByMapForIssues(dbOrTx, companyId, issueIds),
listIssueBlockerAttentionMap(dbOrTx, companyId, withRuns),
listIssueReviewAttentionMap(dbOrTx, companyId, withRuns),
listIssueProductivityReviewMap(dbOrTx, companyId, issueIds),
listIssueBlockedInboxAttentionMap(dbOrTx, companyId, withRuns),
includeLiveDescendantSummary
? liveDescendantCountMapForIssues(dbOrTx, companyId, issueIds)
@@ -6506,9 +6370,6 @@ async function listBlockedInboxIssues(
reviewAttention:
reviewAttentionByIssueId.get(row.id) ?? reviewAttentionNone(),
blockedInboxAttention,
...(productivityReviewByIssueId.has(row.id)
? { productivityReview: productivityReviewByIssueId.get(row.id) }
: {}),
...(includeLiveDescendantSummary
? {
liveDescendantCount:
@@ -8160,12 +8021,10 @@ export function issueService(db: Db) {
const [
blockerAttentionByIssueId,
reviewAttentionByIssueId,
productivityReviewByIssueId,
blockedInboxAttentionByIssueId,
] = await Promise.all([
listIssueBlockerAttentionMap(db, companyId, withRuns),
listIssueReviewAttentionMap(db, companyId, withRuns),
listIssueProductivityReviewMap(db, companyId, issueIds),
includeBlockedInboxAttention
? listIssueBlockedInboxAttentionMap(db, companyId, withRuns)
: Promise.resolve(new Map<string, IssueBlockedInboxAttention>()),
@@ -8203,9 +8062,6 @@ export function issueService(db: Db) {
liveDescendantCountByIssueId.get(row.id) ?? 0,
}
: {}),
...(productivityReviewByIssueId.has(row.id)
? { productivityReview: productivityReviewByIssueId.get(row.id) }
: {}),
};
});
}
@@ -8249,9 +8105,6 @@ export function issueService(db: Db) {
liveDescendantCountByIssueId.get(row.id) ?? 0,
}
: {}),
...(productivityReviewByIssueId.has(row.id)
? { productivityReview: productivityReviewByIssueId.get(row.id) }
: {}),
...deriveIssueUserContext(row, contextUserId, {
myLastCommentAt:
statsByIssueId.get(row.id)?.myLastCommentAt ?? null,
@@ -9119,14 +8972,6 @@ export function issueService(db: Db) {
return listIssueReviewAttentionMap(dbOrTx, companyId, issueRows);
},
listProductivityReviews: async (
companyId: string,
sourceIssueIds: string[],
dbOrTx: any = db,
) => {
return listIssueProductivityReviewMap(dbOrTx, companyId, sourceIssueIds);
},
listWakeableBlockedDependents: async (blockerIssueId: string) => {
const blockerIssue = await db
.select({ id: issues.id, companyId: issues.companyId })
@@ -10474,8 +10319,8 @@ export function issueService(db: Db) {
createdAt: row.createdAt ?? new Date(),
updatedAt: row.updatedAt ?? new Date(),
// Imported in-progress work did not start at import time; fabricating
// startedAt here trips duration-based sweeps (e.g. productivity
// review). Only a bundle-carried startedAt is written.
// startedAt here would misrepresent its active episode.
// Only a bundle-carried startedAt is written.
startedAt: row.startedAt ?? null,
completedAt:
row.completedAt ?? (row.status === "done" ? new Date() : null),
@@ -1,3 +1,4 @@
import { logger } from "../../middleware/logger.js";
import { createHash, randomUUID } from "node:crypto";
import { and, asc, desc, eq, gt, inArray, isNotNull, isNull, lte, notInArray, or, sql } from "drizzle-orm";
import type { Db } from "@paperclipai/db";
@@ -30,6 +31,7 @@ import { issueRecoveryActionService } from "../issue-recovery-actions.js";
import { issueService } from "../issues.js";
import { emitAgentTaskRun } from "../agent-task-run-telemetry.js";
import { resumeNativeWorkspaceFinalization } from "./native-workspace-finalizer.js";
import { dismissObsoleteNativePolicyReviews } from "./obsolete-policy-reviews.js";
import {
cleanupNativeWorkspaceSync,
readNativeWorkspaceSyncReference,
@@ -57,7 +59,6 @@ export type NativeReconciliationFacts = {
newEvidenceSatisfiesContract?: boolean;
dependencyResolved?: boolean;
authorizedResume?: boolean;
policyVersionChanged?: boolean;
statusVersionAdvanced?: boolean;
};
@@ -111,22 +112,6 @@ export function resolveNativeReconciliationStatus(input: {
if (input.facts.authoritativeStatusChanged) {
return preserve("prior_status_terminal_preserved", [{ kind: "append_superseding_assessment" }]);
}
if (input.facts.policyVersionChanged) {
if (["done", "cancelled"].includes(input.priorIssueStatus)) {
return preserve("prior_status_terminal_preserved", [{ kind: "append_superseding_assessment" }]);
}
return {
policyVersion: NATIVE_STATUS_ARBITER_POLICY_VERSION,
statusAction: "in_review",
toStatus: "in_review",
reasonCode: "completion_review_required",
unblockDescriptor: null,
effects: [
{ kind: "bind_reviewer", prompt: "Review the superseding native policy assessment.", ownerUserId: null },
{ kind: "append_superseding_assessment" },
],
};
}
if (input.facts.statusVersionAdvanced) {
return preserve("arbitration_conflict_reloaded", [
{ kind: "increment_status_version" },
@@ -553,6 +538,9 @@ export async function reconcileNativeFinalizations(
}) => Promise<void>;
} = {},
) {
await dismissObsoleteNativePolicyReviews(db, runIds).catch((err) => {
logger.warn({ err }, "Obsolete native policy review lookup failed; continuing native reconciliation");
});
const rows = await db
.select({
runId: heartbeatRuns.id,
@@ -655,7 +643,6 @@ export async function reconcileNativeFinalizations(
),
)).limit(1).then((entries) => entries[0] ?? null)
: null;
const policyVersionChanged = assessment?.policyVersion !== NATIVE_STATUS_ARBITER_POLICY_VERSION;
const currentDecision = row.decisionId
? await db.select({
assessmentId: statusDecisions.assessmentId,
@@ -727,7 +714,7 @@ export async function reconcileNativeFinalizations(
let reassessment = null;
let resultRow = null;
let contractRow = null;
if (assessment && (policyVersionChanged || authoritativeStatusChanged || changedEvidence)) {
if (assessment && (authoritativeStatusChanged || changedEvidence)) {
[resultRow, contractRow] = await Promise.all([
db.select().from(nativeRunResults).where(and(
eq(nativeRunResults.id, assessment.resultId),
@@ -758,11 +745,9 @@ export async function reconcileNativeFinalizations(
&& reassessment.verificationPassed === true;
const facts: NativeReconciliationFacts = authoritativeStatusChanged
? { authoritativeStatusChanged: true }
: policyVersionChanged
? { policyVersionChanged: true }
: newEvidenceSatisfiesContract
? { newEvidenceSatisfiesContract: true }
: {};
: newEvidenceSatisfiesContract
? { newEvidenceSatisfiesContract: true }
: {};
if (Object.keys(facts).length > 0) {
if (!assessment || !reassessment || !resultRow || !contractRow) {
throw new Error("native_reconciliation_reassessment_missing");
@@ -207,6 +207,7 @@ describe("native controller takeover fencing", () => {
}),
now,
isProcessAlive,
readProcessStartedAt: async () => recordedStart,
}),
).resolves.toEqual({ allowed: false, reason: "controller_still_alive" });
expect(isProcessAlive).toHaveBeenCalledWith(123);
@@ -5802,26 +5802,31 @@ describe("native warm session supervision", () => {
return result;
});
await executePaperclipNativeSession({
db: leaseDb(base),
execution: base,
runnerInstanceId: "runner",
});
await executePaperclipNativeSession({
db: leaseDb(lowered),
execution: lowered,
runnerInstanceId: "runner",
});
expect(firstClose).toHaveBeenCalledWith({
reason: "warm native session configuration changed",
});
await vi.waitFor(
() =>
expect(secondClose).toHaveBeenCalledWith({
reason: "warm native session idle timeout",
}),
{ timeout: 500 },
);
// Filesystem work between calls can exceed the idle window on a busy host.
// Advance that window only after proving the permission change closed it.
vi.useFakeTimers({ toFake: ["setTimeout", "clearTimeout"] });
try {
await executePaperclipNativeSession({
db: leaseDb(base),
execution: base,
runnerInstanceId: "runner",
});
await executePaperclipNativeSession({
db: leaseDb(lowered),
execution: lowered,
runnerInstanceId: "runner",
});
expect(firstClose).toHaveBeenCalledWith({
reason: "warm native session configuration changed",
});
expect(secondClose).not.toHaveBeenCalled();
await vi.advanceTimersByTimeAsync(20);
expect(secondClose).toHaveBeenCalledWith({
reason: "warm native session idle timeout",
});
} finally {
vi.useRealTimers();
}
});
});
@@ -1,3 +1,5 @@
import { remoteLeaseCleanupScope } from "../remote-execution-termination.js";
import { resolveConnectorAssignments, isConnectorSkill } from "../connector-runtime.js";
import {
boundedExecutionCleanup,
EXECUTION_CONTROL_DEADLINE_MS,
@@ -6613,6 +6615,8 @@ export async function executePaperclipNativeSession(input: {
db: Db;
execution: NativeExecutionInput;
runnerInstanceId: string;
/** Configured total turn bound; zero/unset is unlimited. */
turnTimeoutMs?: number;
leaseOwner?: string;
restartRecovery?: NativeRestartRecoveryClaim;
onSpawn?: (meta: {
@@ -6633,8 +6637,6 @@ export async function executePaperclipNativeSession(input: {
onGoalCheckpoint?: (snapshot: PersistedNativeSession) => Promise<void>;
sessionGoalControl?: NativeSessionGoalControl | null;
resumeSessionGoalHeartbeat?: boolean;
/** Internal test seam; production rolls over five minutes before runnerd's one-hour lease. */
goalRolloverAtMs?: number;
preparationSpans?: NativeRunHistoricalSpan[];
/** Resolved adapter env; the runner transport applies a provider allowlist before spawn. */
runnerEnvironment?: NodeJS.ProcessEnv;
@@ -7612,6 +7614,14 @@ async function executePaperclipNativeSessionWithinScope(
trace,
})
: null;
const remoteCleanupLease = input.runnerExecutionTarget?.kind === "remote" &&
input.runnerExecutionTarget.transport === "sandbox" && input.runnerExecutionTarget.leaseId
? await input.db.select({ provider: environmentLeases.provider, providerLeaseId: environmentLeases.providerLeaseId })
.from(environmentLeases).where(and(
eq(environmentLeases.companyId, input.execution.binding.companyId),
eq(environmentLeases.id, input.runnerExecutionTarget.leaseId),
)).then(rows => rows[0])
: null;
nativeSessionExecuteStartedAtMs = Date.now();
native = await trace.measure(
"native.session.execute",
@@ -7624,6 +7634,8 @@ async function executePaperclipNativeSessionWithinScope(
const result = await trace.run(runnerSessionStartupScope, () =>
executeNativeSession({
input: runnerExecution,
remoteCleanupScope: remoteCleanupLease ? remoteLeaseCleanupScope(remoteCleanupLease) : undefined,
turnTimeoutMs: input.turnTimeoutMs,
backend:
input.backend ??
runnerdBackend ??
@@ -9610,7 +9622,11 @@ async function createRunnerdBackendWithinSessionClaim(
input.db,
input.execution.binding,
);
const pinnedSkills = new Set("runtimeContext" in input.execution ? input.execution.runtimeContext.skills.map((skill) => skill.key) : []);
const connectorAssignments = [...pinnedSkills].some(isConnectorSkill)
? await resolveConnectorAssignments(input.db, input.execution.binding) : [];
const authority = new PaperclipRunnerToolAuthority(input.db, {
connectorAssignments: connectorAssignments.filter((assignment) => pinnedSkills.has(assignment.skillKey)),
companyId: input.execution.binding.companyId,
issueId: input.execution.binding.issueId,
runId: input.execution.binding.runId,
@@ -23,7 +23,7 @@ export function nativeToolContractFingerprintForTarget(
return `sha256:${createHash("sha256")
.update(
JSON.stringify({
schema: "paperclip.native-tool-contract.v10",
schema: "paperclip.native-tool-contract.v12",
executionTargetKind,
advertisementPolicy: {
// Direct provider threads retain declarations from thread/start.
@@ -37,6 +37,7 @@ export function nativeToolContractFingerprintForTarget(
structuredHumanInput:
"always_advertised_run_issue_agent_binding_gated_current_task_description.v2",
semanticCompletion: "finish_response_wake_user_facing_summary.v3",
connectorTools: "assigned_resources_and_pinned_skill_bundle.v1",
},
tools: [
...(executionTargetKind === "local"
@@ -0,0 +1,150 @@
import { and, eq, inArray, isNull, sql } from "drizzle-orm";
import {
approvals, issueApprovals, issueThreadInteractions, issues,
nativeRunFinalizations, statusDecisionEffects, statusDecisions, workAssessments,
type Db,
} from "@paperclipai/db";
import { logger } from "../../middleware/logger.js";
import { issueService } from "../issues.js";
import { issueThreadInteractionService } from "../issue-thread-interactions.js";
import { enqueueTerminalIssueInteractionChatPublications } from "../chat-interaction-publications.js";
import { persistActivity, publishActivity, type ActivityPublication } from "../activity-log.js";
const obsoletePrompt = "Review the superseding native policy assessment.";
/** Retire only the old version-change gate, never a real completion review. */
export async function dismissObsoleteNativePolicyReviews(db: Db, runIds?: string[]) {
const candidates = await db.select({
interaction: issueThreadInteractions,
decision: statusDecisions,
priorDecisionId: workAssessments.priorDecisionId,
}).from(issueThreadInteractions)
.innerJoin(statusDecisionEffects, and(
eq(statusDecisionEffects.companyId, issueThreadInteractions.companyId),
eq(statusDecisionEffects.issueId, issueThreadInteractions.issueId),
sql`${statusDecisionEffects.targetId} = ${issueThreadInteractions.id}::text`,
eq(statusDecisionEffects.targetType, "issue_thread_interaction"),
eq(statusDecisionEffects.effectKind, "bind_reviewer"),
))
.innerJoin(statusDecisions, and(
eq(statusDecisions.id, statusDecisionEffects.decisionId),
eq(statusDecisions.companyId, issueThreadInteractions.companyId),
eq(statusDecisions.issueId, issueThreadInteractions.issueId),
eq(statusDecisions.runId, issueThreadInteractions.sourceRunId),
))
.innerJoin(workAssessments, and(
eq(workAssessments.id, statusDecisions.assessmentId),
eq(workAssessments.companyId, statusDecisions.companyId),
eq(workAssessments.issueId, statusDecisions.issueId),
))
.where(and(
eq(issueThreadInteractions.status, "pending"),
eq(issueThreadInteractions.kind, "request_confirmation"),
isNull(issueThreadInteractions.createdByAgentId),
isNull(issueThreadInteractions.createdByUserId),
eq(statusDecisions.applicationState, "applied"),
eq(statusDecisions.reasonCode, "completion_review_required"),
eq(statusDecisions.toStatus, "in_review"),
sql`${issueThreadInteractions.idempotencyKey} = 'native-review:' || ${statusDecisions.id}::text`,
sql`${issueThreadInteractions.payload}->>'prompt' = ${obsoletePrompt}`,
// Match the complete old decision, including its unique assessment-only effect.
sql`${statusDecisions.decisionJson}->'effects' = ${JSON.stringify([
{ kind: "bind_reviewer", prompt: obsoletePrompt, ownerUserId: null },
{ kind: "append_superseding_assessment" },
])}::jsonb`,
...(runIds?.length ? [inArray(statusDecisions.runId, runIds)] : []),
)).limit(100);
for (const { interaction, decision, priorDecisionId } of candidates) {
const publications: ActivityPublication[] = [];
try {
await db.transaction(async (tx) => {
// Same lock order as status commits: coordinator, issue, interaction.
await tx.select({ runId: nativeRunFinalizations.runId }).from(nativeRunFinalizations)
.where(and(eq(nativeRunFinalizations.runId, decision.runId),
eq(nativeRunFinalizations.companyId, decision.companyId)))
.for("update");
const issue = await tx.select().from(issues).where(and(
eq(issues.id, decision.issueId), eq(issues.companyId, decision.companyId),
)).for("update").then((rows) => rows[0]);
if (!issue) return;
const now = new Date();
const [cancelled] = await tx.update(issueThreadInteractions).set({
status: "cancelled",
result: { version: 1, outcome: "withdrawn", reason: "A Paperclip upgrade does not require completion review." },
resolvedAt: now,
updatedAt: now,
}).where(and(
eq(issueThreadInteractions.id, interaction.id),
eq(issueThreadInteractions.companyId, decision.companyId),
eq(issueThreadInteractions.status, "pending"),
)).returning({ id: issueThreadInteractions.id });
if (!cancelled) return;
const terminalInteraction = await issueThreadInteractionService(tx as unknown as Db).getById(cancelled.id);
if (terminalInteraction) {
await enqueueTerminalIssueInteractionChatPublications(tx as unknown as Db, terminalInteraction);
}
const pendingInteraction = await tx.select({ id: issueThreadInteractions.id })
.from(issueThreadInteractions).where(and(
eq(issueThreadInteractions.companyId, issue.companyId),
eq(issueThreadInteractions.issueId, issue.id),
eq(issueThreadInteractions.status, "pending"),
)).limit(1);
const pendingApproval = await tx.select({ id: approvals.id }).from(issueApprovals)
.innerJoin(approvals, and(eq(approvals.id, issueApprovals.approvalId),
eq(approvals.companyId, issue.companyId)))
.where(and(eq(issueApprovals.companyId, issue.companyId), eq(issueApprovals.issueId, issue.id),
inArray(approvals.status, ["pending", "revision_requested"]))).limit(1);
const restoreStatus = issue.status === "in_review"
&& issue.lastStatusDecisionId === decision.id
&& issue.statusVersion === Number(decision.decisionJson.projectedStatusVersion ?? decision.decisionVersion)
&& ["backlog", "todo", "in_progress", "blocked"].includes(decision.fromStatus)
&& pendingInteraction.length === 0 && pendingApproval.length === 0
&& issue.executionState?.status !== "pending";
if (restoreStatus) {
const priorDecision = priorDecisionId ? await tx.select().from(statusDecisions).where(and(
eq(statusDecisions.id, priorDecisionId), eq(statusDecisions.companyId, issue.companyId),
eq(statusDecisions.issueId, issue.id),
)).then((rows) => rows[0]) : null;
await issueService(tx as unknown as Db).update(issue.id, {
status: decision.fromStatus,
// This is an administrative correction, not a replay of the old decision.
lastStatusDecisionId: null,
unblockDescriptor: priorDecision?.decisionJson.unblockDescriptor as typeof issue.unblockDescriptor ?? null,
}, tx, publications);
}
const { publication } = await persistActivity(tx as unknown as Db, {
companyId: issue.companyId,
actorType: "system",
actorId: "native-policy-review-cleanup",
action: restoreStatus ? "issue.updated" : "issue.interaction_cancelled",
entityType: "issue",
entityId: issue.id,
issueId: issue.id,
runId: decision.runId,
details: {
source: "obsolete_native_policy_review",
interactionId: interaction.id,
decisionId: decision.id,
fromStatus: issue.status,
toStatus: restoreStatus ? decision.fromStatus : issue.status,
},
});
publications.push(publication);
});
} catch (err) {
logger.warn({ err, interactionId: interaction.id, issueId: decision.issueId },
"Failed to withdraw obsolete native policy review; will retry on the next pass");
continue;
}
for (const publication of publications) {
try {
publishActivity(publication);
} catch (err) {
logger.warn({ err, interactionId: interaction.id, issueId: decision.issueId },
"Obsolete native policy review cleanup committed; live activity publication failed, history is preserved");
}
}
}
}
@@ -1,3 +1,4 @@
import { isConnectorTool, executeConnectorTool, type ConnectorAssignment } from "../connector-runtime.js";
import { resolveNativeRuntimeMcpSnapshot } from "./runtime-context.js";
import { connectionIntentService } from "../connection-intents.js";
import { RUNTIME_CONNECTION_TOOL_DEFINITIONS } from "../connection-tool-definitions.js";
@@ -86,6 +87,7 @@ type Binding = {
apiUrl?: string;
storage?: StorageService;
/** Server-owned suppression for baseline evals; true never overrides operator opt-in. */
connectorAssignments?: ConnectorAssignment[];
apiToolsEnabled?: boolean;
workMode?: "standard" | "planning" | "ask";
workspaceRoot?: string;
@@ -184,7 +186,7 @@ export class PaperclipRunnerToolAuthority {
definitions.push(LIST_CHAT_ATTACHMENTS_TOOL_DEFINITION);
definitions.push(REUSE_CHAT_ATTACHMENT_TOOL_DEFINITION);
definitions.push(READ_CHAT_ATTACHMENT_TOOL_DEFINITION);
return [...RUNTIME_CONNECTION_TOOL_DEFINITIONS, ...definitions];
return [...RUNTIME_CONNECTION_TOOL_DEFINITIONS, ...(this.binding.connectorAssignments ?? []).flatMap((assignment) => assignment.tools), ...definitions];
}
async execute(call: {
@@ -192,6 +194,13 @@ export class PaperclipRunnerToolAuthority {
callId: string;
arguments: unknown;
}): Promise<unknown> {
if (isConnectorTool(call.tool)) {
if (!(this.binding.connectorAssignments ?? []).some((assignment) => assignment.tools.some((tool) => tool.name === call.tool))) throw forbidden("Connector tool is not available to this run");
const { run } = await this.#boundContext();
const snapshot = record(run.contextSnapshot);
if (isPaperclipExternalChatContractTurn(snapshot.paperclipWake) || String(snapshot.source ?? "").startsWith("chat:") || snapshot.paperclipExternalChatQuestionResponse) throw forbidden("Restricted chat runs cannot use email actions");
return executeConnectorTool(this.db, this.binding, call.tool, call.arguments);
}
if (RUNTIME_CONNECTION_TOOL_DEFINITIONS.some((tool) => tool.name === call.tool)) {
await this.#boundContext();
const { run } = await captureRunIdentity(this.db, this.binding);
@@ -1081,7 +1090,7 @@ export class PaperclipRunnerToolAuthority {
eq(chatEndpoints.assignedAgentId, this.binding.agentId),
),
);
if (!endpoint) {
if (!endpoint || endpoint.provider === "agentmail") {
throw new Error("paperclip_runner_chat_attachment_binding_denied");
}
provider = endpoint.provider;
@@ -87,7 +87,7 @@ async function verifyMaterializedAsset(
}
}
async function materializeAsset(files: AssetFile[]): Promise<NativeRuntimeAssetReference> {
export async function materializeAsset(files: AssetFile[]): Promise<NativeRuntimeAssetReference> {
const sorted = [...files].sort((a, b) => a.path.localeCompare(b.path));
const manifestFiles = sorted.map((file) => ({ path: safeRelativePath(file.path, "runtime context path"), sha256: sha256(file.content), mode: file.mode & 0o555, size: file.content.byteLength }));
const totalBytes = manifestFiles.reduce((sum, file) => sum + file.size, 0);
-999
View File
@@ -1,999 +0,0 @@
import { and, asc, desc, eq, gt, gte, inArray, isNull, notInArray, sql } from "drizzle-orm";
import type { Db } from "@paperclipai/db";
import { clampIssueRequestDepth } from "@paperclipai/shared";
import {
activityLog,
agents,
companies,
costEvents,
heartbeatRuns,
issueComments,
issues,
projects,
} from "@paperclipai/db";
import { logger } from "../middleware/logger.js";
import { logActivity } from "./activity-log.js";
import { budgetService } from "./budgets.js";
import { issueService } from "./issues.js";
import { visibleIssueCondition } from "./issue-visibility.js";
import { withRecoveryContext } from "./recovery/status-only-context.js";
import { RECOVERY_ORIGIN_KINDS } from "./recovery/origins.js";
export const PRODUCTIVITY_REVIEW_ORIGIN_KIND = RECOVERY_ORIGIN_KINDS.issueProductivityReview;
export const DEFAULT_PRODUCTIVITY_REVIEW_NO_COMMENT_STREAK_RUNS = 10;
export const DEFAULT_PRODUCTIVITY_REVIEW_LONG_ACTIVE_HOURS = 6;
export const DEFAULT_PRODUCTIVITY_REVIEW_HIGH_CHURN_HOURLY = 10;
export const DEFAULT_PRODUCTIVITY_REVIEW_HIGH_CHURN_SIX_HOURS = 30;
export const DEFAULT_PRODUCTIVITY_REVIEW_RESOLVED_SNOOZE_MS = 6 * 60 * 60 * 1000;
export const DEFAULT_PRODUCTIVITY_REVIEW_REFRESH_INTERVAL_MS = 60 * 60 * 1000;
export const DEFAULT_PRODUCTIVITY_REVIEW_MAX_REFRESH_COMMENTS = 3;
export const DEFAULT_PRODUCTIVITY_REVIEW_CREATION_WINDOW_MS = 24 * 60 * 60 * 1000;
export const DEFAULT_PRODUCTIVITY_REVIEW_MAX_CREATIONS_PER_WINDOW = 1;
export const DEFAULT_PRODUCTIVITY_REVIEW_MAX_CONSECUTIVE_NO_ACTION_REVIEWS = 3;
const TERMINAL_RUN_STATUSES = ["succeeded", "interrupted", "failed", "cancelled", "timed_out"] as const;
const ACTIVE_RUN_STATUSES = ["queued", "running", "scheduled_retry"] as const;
const MAX_CANDIDATE_ISSUES = 250;
const MAX_RUNS_FOR_STREAK = 100;
const MAX_PARENT_WALK_DEPTH = 25;
export const PRODUCTIVITY_REVIEW_REFRESH_COMMENT_PREFIX = "Productivity review evidence refreshed.";
type IssueRow = typeof issues.$inferSelect;
type AgentRow = typeof agents.$inferSelect;
type HeartbeatRunRow = typeof heartbeatRuns.$inferSelect;
// Evidence only reads these run fields; selecting the full row detoasts
// result_json/context_snapshot for up to MAX_RUNS_FOR_STREAK runs per issue.
type ProductivityRunSample = Pick<
HeartbeatRunRow,
"id" | "agentId" | "status" | "livenessState" | "createdAt" | "nextAction" | "usageJson"
>;
type ProductivityReviewTrigger = "no_comment_streak" | "long_active_duration" | "high_churn";
type ProductivityReviewThresholds = {
noCommentStreakRuns: number;
longActiveMs: number;
highChurnHourly: number;
highChurnSixHours: number;
resolvedSnoozeMs: number;
refreshIntervalMs: number;
maxRefreshComments: number;
creationWindowMs: number;
maxCreationsPerWindow: number;
maxConsecutiveNoActionReviews: number;
};
type ProductivityReviewEvidence = {
trigger: ProductivityReviewTrigger;
triggerReasons: string[];
sourceIssue: IssueRow;
sourceAgent: AgentRow;
noCommentStreak: number;
totalRunCount: number;
terminalRunCount: number;
activeRunCount: number;
runCountLastHour: number;
runCountLastSixHours: number;
commentCount: number;
commentCountLastHour: number;
commentCountLastSixHours: number;
elapsedMs: number | null;
latestRuns: ProductivityRunSample[];
latestComments: Array<typeof issueComments.$inferSelect>;
costCents: number;
usageSamples: Array<{ runId: string; usageJson: Record<string, unknown> | null }>;
nextAction: string | null;
thresholds: ProductivityReviewThresholds;
generatedAt: Date;
};
type EnqueueWakeup = (
agentId: string,
opts?: {
source?: "timer" | "assignment" | "on_demand" | "automation";
triggerDetail?: "manual" | "ping" | "callback" | "system";
reason?: string | null;
payload?: Record<string, unknown> | null;
requestedByActorType?: "user" | "agent" | "system";
requestedByActorId?: string | null;
contextSnapshot?: Record<string, unknown>;
},
) => Promise<unknown | null>;
function productivityReviewFingerprint(sourceIssueId: string) {
return `productivity-review:${sourceIssueId}`;
}
function issueRunScopeSql(issueId: string) {
return sql`(
${heartbeatRuns.contextSnapshot}->>'issueId' = ${issueId}
or ${heartbeatRuns.contextSnapshot}->>'taskId' = ${issueId}
or ${heartbeatRuns.contextSnapshot}->>'taskKey' = ${issueId}
)`;
}
function msToHuman(ms: number | null) {
if (ms === null) return "unknown";
const minutes = Math.floor(ms / 60_000);
if (minutes < 60) return `${minutes}m`;
const hours = Math.floor(minutes / 60);
const days = Math.floor(hours / 24);
if (days > 0) return `${days}d ${hours % 24}h`;
return `${hours}h ${minutes % 60}m`;
}
function issueUiLink(issue: { identifier: string | null; id: string }, prefix: string) {
const label = issue.identifier ?? issue.id;
return `[${label}](/${prefix}/issues/${label})`;
}
function runUiLink(run: { id: string; agentId: string }, prefix: string) {
return `[${run.id}](/${prefix}/agents/${run.agentId}/runs/${run.id})`;
}
function truncateInline(value: string | null | undefined, max = 260) {
if (!value) return "";
const compact = value.replace(/\s+/g, " ").trim();
return compact.length <= max ? compact : `${compact.slice(0, max - 3)}...`;
}
function readPositiveInteger(value: number, fallback: number) {
return Number.isFinite(value) && value > 0 ? Math.floor(value) : fallback;
}
function coerceDate(value: Date | string | null | undefined) {
if (!value) return null;
return value instanceof Date ? value : new Date(value);
}
function buildThresholds(overrides?: Partial<ProductivityReviewThresholds>): ProductivityReviewThresholds {
return {
noCommentStreakRuns: readPositiveInteger(
overrides?.noCommentStreakRuns ?? DEFAULT_PRODUCTIVITY_REVIEW_NO_COMMENT_STREAK_RUNS,
DEFAULT_PRODUCTIVITY_REVIEW_NO_COMMENT_STREAK_RUNS,
),
longActiveMs: readPositiveInteger(
overrides?.longActiveMs ?? DEFAULT_PRODUCTIVITY_REVIEW_LONG_ACTIVE_HOURS * 60 * 60 * 1000,
DEFAULT_PRODUCTIVITY_REVIEW_LONG_ACTIVE_HOURS * 60 * 60 * 1000,
),
highChurnHourly: readPositiveInteger(
overrides?.highChurnHourly ?? DEFAULT_PRODUCTIVITY_REVIEW_HIGH_CHURN_HOURLY,
DEFAULT_PRODUCTIVITY_REVIEW_HIGH_CHURN_HOURLY,
),
highChurnSixHours: readPositiveInteger(
overrides?.highChurnSixHours ?? DEFAULT_PRODUCTIVITY_REVIEW_HIGH_CHURN_SIX_HOURS,
DEFAULT_PRODUCTIVITY_REVIEW_HIGH_CHURN_SIX_HOURS,
),
resolvedSnoozeMs: readPositiveInteger(
overrides?.resolvedSnoozeMs ?? DEFAULT_PRODUCTIVITY_REVIEW_RESOLVED_SNOOZE_MS,
DEFAULT_PRODUCTIVITY_REVIEW_RESOLVED_SNOOZE_MS,
),
refreshIntervalMs: readPositiveInteger(
overrides?.refreshIntervalMs ?? DEFAULT_PRODUCTIVITY_REVIEW_REFRESH_INTERVAL_MS,
DEFAULT_PRODUCTIVITY_REVIEW_REFRESH_INTERVAL_MS,
),
maxRefreshComments: readPositiveInteger(
overrides?.maxRefreshComments ?? DEFAULT_PRODUCTIVITY_REVIEW_MAX_REFRESH_COMMENTS,
DEFAULT_PRODUCTIVITY_REVIEW_MAX_REFRESH_COMMENTS,
),
creationWindowMs: readPositiveInteger(
overrides?.creationWindowMs ?? DEFAULT_PRODUCTIVITY_REVIEW_CREATION_WINDOW_MS,
DEFAULT_PRODUCTIVITY_REVIEW_CREATION_WINDOW_MS,
),
maxCreationsPerWindow: readPositiveInteger(
overrides?.maxCreationsPerWindow ?? DEFAULT_PRODUCTIVITY_REVIEW_MAX_CREATIONS_PER_WINDOW,
DEFAULT_PRODUCTIVITY_REVIEW_MAX_CREATIONS_PER_WINDOW,
),
maxConsecutiveNoActionReviews: readPositiveInteger(
overrides?.maxConsecutiveNoActionReviews ?? DEFAULT_PRODUCTIVITY_REVIEW_MAX_CONSECUTIVE_NO_ACTION_REVIEWS,
DEFAULT_PRODUCTIVITY_REVIEW_MAX_CONSECUTIVE_NO_ACTION_REVIEWS,
),
};
}
function choosePrimaryTrigger(input: {
noComment: boolean;
longActive: boolean;
highChurn: boolean;
}): ProductivityReviewTrigger | null {
if (input.noComment) return "no_comment_streak";
if (input.highChurn) return "high_churn";
if (input.longActive) return "long_active_duration";
return null;
}
function isSoftStopTrigger(trigger: ProductivityReviewTrigger) {
return trigger === "no_comment_streak" || trigger === "high_churn";
}
function formatTrigger(trigger: ProductivityReviewTrigger) {
if (trigger === "no_comment_streak") return "No-comment streak";
if (trigger === "high_churn") return "High churn";
return "Long active duration";
}
export function productivityReviewService(db: Db, deps?: { enqueueWakeup?: EnqueueWakeup }) {
const issuesSvc = issueService(db);
const budgets = budgetService(db);
async function getCompanyIssuePrefix(companyId: string) {
return db
.select({ issuePrefix: companies.issuePrefix })
.from(companies)
.where(eq(companies.id, companyId))
.then((rows) => rows[0]?.issuePrefix ?? "PAP");
}
async function getAgent(agentId: string) {
return db
.select()
.from(agents)
.where(eq(agents.id, agentId))
.then((rows) => rows[0] ?? null);
}
function isAgentInvokable(agent: AgentRow | null | undefined) {
return Boolean(agent && !["paused", "terminated", "pending_approval"].includes(agent.status));
}
async function isProductivityReviewDescendant(issue: Pick<IssueRow, "companyId" | "parentId">) {
let parentId = issue.parentId;
let depth = 0;
while (parentId && depth < MAX_PARENT_WALK_DEPTH) {
const parent = await db
.select({ id: issues.id, parentId: issues.parentId, originKind: issues.originKind })
.from(issues)
.where(and(eq(issues.companyId, issue.companyId), eq(issues.id, parentId)))
.then((rows) => rows[0] ?? null);
if (!parent) return false;
if (parent.originKind === PRODUCTIVITY_REVIEW_ORIGIN_KIND) return true;
parentId = parent.parentId;
depth += 1;
}
return false;
}
async function findOpenProductivityReview(companyId: string, sourceIssueId: string) {
return db
.select()
.from(issues)
.where(
and(
eq(issues.companyId, companyId),
eq(issues.originKind, PRODUCTIVITY_REVIEW_ORIGIN_KIND),
eq(issues.originId, sourceIssueId),
visibleIssueCondition(),
notInArray(issues.status, ["done", "cancelled"]),
),
)
.orderBy(desc(issues.updatedAt))
.limit(1)
.then((rows) => rows[0] ?? null);
}
async function findRecentTerminalProductivityReview(
companyId: string,
sourceIssueId: string,
thresholds: ProductivityReviewThresholds,
now: Date,
) {
const cutoff = new Date(now.getTime() - thresholds.resolvedSnoozeMs);
return db
.select({ id: issues.id, identifier: issues.identifier, status: issues.status, updatedAt: issues.updatedAt })
.from(issues)
.where(
and(
eq(issues.companyId, companyId),
eq(issues.originKind, PRODUCTIVITY_REVIEW_ORIGIN_KIND),
eq(issues.originId, sourceIssueId),
inArray(issues.status, ["done", "cancelled"]),
gt(issues.updatedAt, cutoff),
),
)
.orderBy(desc(issues.updatedAt))
.limit(1)
.then((rows) => rows[0] ?? null);
}
async function countRecentProductivityReviews(
companyId: string,
sourceIssueId: string,
thresholds: ProductivityReviewThresholds,
now: Date,
) {
const cutoff = new Date(now.getTime() - thresholds.creationWindowMs);
return db
.select({ count: sql<number>`count(*)::int` })
.from(issues)
.where(
and(
eq(issues.companyId, companyId),
eq(issues.originKind, PRODUCTIVITY_REVIEW_ORIGIN_KIND),
eq(issues.originId, sourceIssueId),
visibleIssueCondition(),
sql`${issues.status} <> 'cancelled'`,
sql`${issues.createdAt} >= ${cutoff.toISOString()}::timestamptz`,
),
)
.then((rows) => Number(rows[0]?.count ?? 0));
}
async function countConsecutiveNoActionProductivityReviews(
companyId: string,
sourceIssueId: string,
thresholds: ProductivityReviewThresholds,
) {
const completedReviews = await db
.select({
createdAt: issues.createdAt,
})
.from(issues)
.where(
and(
eq(issues.companyId, companyId),
eq(issues.originKind, PRODUCTIVITY_REVIEW_ORIGIN_KIND),
eq(issues.originId, sourceIssueId),
eq(issues.status, "done"),
visibleIssueCondition(),
),
)
.orderBy(desc(issues.createdAt), desc(issues.id))
.limit(thresholds.maxConsecutiveNoActionReviews);
const earliestReviewCreatedAt = completedReviews.at(-1)?.createdAt;
if (!earliestReviewCreatedAt) return 0;
const sourceActions = await db
.select({ createdAt: activityLog.createdAt })
.from(activityLog)
.where(
and(
eq(activityLog.companyId, companyId),
eq(activityLog.entityType, "issue"),
eq(activityLog.entityId, sourceIssueId),
gte(activityLog.createdAt, earliestReviewCreatedAt),
),
);
let streak = 0;
for (const [index, review] of completedReviews.entries()) {
const nextNewerReviewCreatedAt = completedReviews[index - 1]?.createdAt ?? null;
const sourceAction = sourceActions.some((activity) => {
if (activity.createdAt < review.createdAt) return false;
return !nextNewerReviewCreatedAt || activity.createdAt < nextNewerReviewCreatedAt;
});
if (sourceAction) break;
streak += 1;
}
return streak;
}
async function getRefreshCommentState(companyId: string, reviewIssueId: string) {
return db
.select({
count: sql<number>`count(*)::int`,
latestCreatedAt: sql<Date | null>`max(${issueComments.createdAt})`,
})
.from(issueComments)
.where(
and(
eq(issueComments.companyId, companyId),
eq(issueComments.issueId, reviewIssueId),
sql`${issueComments.body} like ${`${PRODUCTIVITY_REVIEW_REFRESH_COMMENT_PREFIX}%`}`,
),
)
.then((rows) => {
const row = rows[0];
return {
count: Number(row?.count ?? 0),
latestCreatedAt: coerceDate(row?.latestCreatedAt),
};
});
}
async function addRefreshComment(
reviewIssueId: string,
body: string,
generatedAt: Date,
) {
const comment = await issuesSvc.addComment(reviewIssueId, body, {});
await db
.update(issueComments)
.set({ createdAt: generatedAt, updatedAt: generatedAt })
.where(eq(issueComments.id, comment.id));
await db
.update(issues)
.set({ updatedAt: generatedAt })
.where(eq(issues.id, reviewIssueId));
return comment;
}
async function countIssueRunsSince(companyId: string, agentId: string, issueId: string, since: Date) {
return db
.select({ count: sql<number>`count(*)::int` })
.from(heartbeatRuns)
.where(
and(
eq(heartbeatRuns.companyId, companyId),
eq(heartbeatRuns.agentId, agentId),
issueRunScopeSql(issueId),
sql`coalesce(${heartbeatRuns.startedAt}, ${heartbeatRuns.createdAt}) >= ${since.toISOString()}::timestamptz`,
),
)
.then((rows) => rows[0]?.count ?? 0);
}
async function countIssueCommentsSince(companyId: string, issueId: string, agentId: string, since?: Date) {
return db
.select({ count: sql<number>`count(*)::int` })
.from(issueComments)
.innerJoin(heartbeatRuns, eq(heartbeatRuns.id, issueComments.createdByRunId))
.where(
and(
eq(issueComments.companyId, companyId),
eq(issueComments.issueId, issueId),
eq(issueComments.authorAgentId, agentId),
eq(heartbeatRuns.companyId, companyId),
eq(heartbeatRuns.agentId, agentId),
issueRunScopeSql(issueId),
since ? sql`${issueComments.createdAt} >= ${since.toISOString()}::timestamptz` : undefined,
),
)
.then((rows) => rows[0]?.count ?? 0);
}
async function collectEvidence(
sourceIssue: IssueRow,
sourceAgent: AgentRow,
thresholds: ProductivityReviewThresholds,
now: Date,
): Promise<ProductivityReviewEvidence | null> {
const oneHourAgo = new Date(now.getTime() - 60 * 60 * 1000);
const sixHoursAgo = new Date(now.getTime() - 6 * 60 * 60 * 1000);
const latestRuns = await db
.select({
id: heartbeatRuns.id,
agentId: heartbeatRuns.agentId,
status: heartbeatRuns.status,
livenessState: heartbeatRuns.livenessState,
createdAt: heartbeatRuns.createdAt,
nextAction: heartbeatRuns.nextAction,
usageJson: heartbeatRuns.usageJson,
})
.from(heartbeatRuns)
.where(
and(
eq(heartbeatRuns.companyId, sourceIssue.companyId),
eq(heartbeatRuns.agentId, sourceAgent.id),
issueRunScopeSql(sourceIssue.id),
),
)
.orderBy(desc(heartbeatRuns.createdAt), desc(heartbeatRuns.id))
.limit(MAX_RUNS_FOR_STREAK);
const runIds = latestRuns.map((run) => run.id);
const commentRunIds = new Set<string>();
if (runIds.length > 0) {
const commentRows = await db
.select({ createdByRunId: issueComments.createdByRunId })
.from(issueComments)
.where(
and(
eq(issueComments.companyId, sourceIssue.companyId),
eq(issueComments.issueId, sourceIssue.id),
inArray(issueComments.createdByRunId, runIds),
),
);
for (const row of commentRows) {
if (row.createdByRunId) commentRunIds.add(row.createdByRunId);
}
}
const terminalRuns = latestRuns.filter((run) =>
TERMINAL_RUN_STATUSES.includes(run.status as (typeof TERMINAL_RUN_STATUSES)[number]),
);
let noCommentStreak = 0;
for (const run of terminalRuns) {
if (commentRunIds.has(run.id)) break;
noCommentStreak += 1;
}
const [
runCountLastHour,
runCountLastSixHours,
assigneeRunCommentCount,
assigneeRunCommentCountLastHour,
assigneeRunCommentCountLastSixHours,
latestComments,
costRow,
] = await Promise.all([
countIssueRunsSince(sourceIssue.companyId, sourceAgent.id, sourceIssue.id, oneHourAgo),
countIssueRunsSince(sourceIssue.companyId, sourceAgent.id, sourceIssue.id, sixHoursAgo),
countIssueCommentsSince(sourceIssue.companyId, sourceIssue.id, sourceAgent.id),
countIssueCommentsSince(sourceIssue.companyId, sourceIssue.id, sourceAgent.id, oneHourAgo),
countIssueCommentsSince(sourceIssue.companyId, sourceIssue.id, sourceAgent.id, sixHoursAgo),
db
.select({ comment: issueComments })
.from(issueComments)
.innerJoin(heartbeatRuns, eq(heartbeatRuns.id, issueComments.createdByRunId))
.where(
and(
eq(issueComments.companyId, sourceIssue.companyId),
eq(issueComments.issueId, sourceIssue.id),
eq(issueComments.authorAgentId, sourceAgent.id),
eq(heartbeatRuns.companyId, sourceIssue.companyId),
eq(heartbeatRuns.agentId, sourceAgent.id),
issueRunScopeSql(sourceIssue.id),
),
)
.orderBy(desc(issueComments.createdAt), desc(issueComments.id))
.limit(5)
.then((rows) => rows.map((row) => row.comment)),
db
.select({ costCents: sql<number>`coalesce(sum(${costEvents.costCents}), 0)::int` })
.from(costEvents)
.where(and(eq(costEvents.companyId, sourceIssue.companyId), eq(costEvents.issueId, sourceIssue.id)))
.then((rows) => rows[0] ?? { costCents: 0 }),
]);
const activeRunCount = latestRuns.filter((run) =>
ACTIVE_RUN_STATUSES.includes(run.status as (typeof ACTIVE_RUN_STATUSES)[number]),
).length;
const activeStartedAt = sourceIssue.startedAt ?? sourceIssue.executionLockedAt ?? null;
const elapsedMs = sourceIssue.status === "in_progress" && activeStartedAt
? Math.max(0, now.getTime() - activeStartedAt.getTime())
: null;
const noComment = noCommentStreak >= thresholds.noCommentStreakRuns;
const longActive = elapsedMs !== null && elapsedMs >= thresholds.longActiveMs;
const highChurn =
runCountLastHour >= thresholds.highChurnHourly ||
assigneeRunCommentCountLastHour >= thresholds.highChurnHourly ||
runCountLastSixHours >= thresholds.highChurnSixHours ||
assigneeRunCommentCountLastSixHours >= thresholds.highChurnSixHours;
const trigger = choosePrimaryTrigger({ noComment, longActive, highChurn });
if (!trigger) return null;
const triggerReasons: string[] = [];
if (noComment) triggerReasons.push(`${noCommentStreak} consecutive completed issue-linked runs had no run-created issue comment`);
if (longActive) triggerReasons.push(`current active episode has lasted ${msToHuman(elapsedMs)}`);
if (highChurn) {
triggerReasons.push(
`${runCountLastHour} runs/${assigneeRunCommentCountLastHour} assignee-run comments in 1h; ${runCountLastSixHours} runs/${assigneeRunCommentCountLastSixHours} assignee-run comments in 6h`,
);
}
return {
trigger,
triggerReasons,
sourceIssue,
sourceAgent,
noCommentStreak,
totalRunCount: latestRuns.length,
terminalRunCount: terminalRuns.length,
activeRunCount,
runCountLastHour,
runCountLastSixHours,
commentCount: assigneeRunCommentCount,
commentCountLastHour: assigneeRunCommentCountLastHour,
commentCountLastSixHours: assigneeRunCommentCountLastSixHours,
elapsedMs,
latestRuns: latestRuns.slice(0, 5),
latestComments,
costCents: costRow.costCents,
usageSamples: latestRuns
.filter((run) => run.usageJson)
.slice(0, 3)
.map((run) => ({ runId: run.id, usageJson: run.usageJson ?? null })),
nextAction: latestRuns.find((run) => run.nextAction)?.nextAction ?? null,
thresholds,
generatedAt: now,
};
}
async function resolveReviewOwnerAgentId(sourceIssue: IssueRow, sourceAgent: AgentRow) {
const candidateIds: string[] = [];
if (sourceAgent.reportsTo) candidateIds.push(sourceAgent.reportsTo);
if (sourceIssue.createdByAgentId) candidateIds.push(sourceIssue.createdByAgentId);
if (sourceIssue.projectId) {
const project = await db
.select({ leadAgentId: projects.leadAgentId })
.from(projects)
.where(and(eq(projects.companyId, sourceIssue.companyId), eq(projects.id, sourceIssue.projectId)))
.then((rows) => rows[0] ?? null);
if (project?.leadAgentId) candidateIds.push(project.leadAgentId);
}
const roleCandidates = await db
.select({ id: agents.id })
.from(agents)
.where(and(eq(agents.companyId, sourceIssue.companyId), inArray(agents.role, ["cto", "ceo"])))
.orderBy(sql`case when ${agents.role} = 'cto' then 0 else 1 end`, asc(agents.createdAt), asc(agents.id));
candidateIds.push(...roleCandidates.map((agent) => agent.id));
const seen = new Set<string>();
for (const agentId of candidateIds) {
if (seen.has(agentId)) continue;
seen.add(agentId);
const candidate = await getAgent(agentId);
if (!candidate || candidate.companyId !== sourceIssue.companyId || !isAgentInvokable(candidate)) continue;
const budgetBlock = await budgets.getInvocationBlock(sourceIssue.companyId, candidate.id, {
issueId: sourceIssue.id,
projectId: sourceIssue.projectId ?? null,
});
if (!budgetBlock) return candidate.id;
}
return null;
}
function buildReviewMarkdown(evidence: ProductivityReviewEvidence, prefix: string) {
const latestRuns = evidence.latestRuns.length > 0
? evidence.latestRuns.map((run) =>
`- ${runUiLink(run, prefix)} \`${run.status}\` liveness \`${run.livenessState ?? "unknown"}\`, created ${run.createdAt.toISOString()}${run.nextAction ? `, next action: ${truncateInline(run.nextAction, 160)}` : ""}`,
).join("\n")
: "- none";
const latestComments = evidence.latestComments.length > 0
? evidence.latestComments.map((comment) =>
`- ${comment.createdAt.toISOString()}${comment.createdByRunId ? ` run \`${comment.createdByRunId}\`` : ""}: ${truncateInline(comment.body)}`,
).join("\n")
: "- none";
const usage = evidence.usageSamples.length > 0
? evidence.usageSamples.map((sample) => `- \`${sample.runId}\`: \`${JSON.stringify(sample.usageJson).slice(0, 500)}\``).join("\n")
: "- no usage payloads on sampled runs";
return [
"Paperclip detected an unusual productivity/progression pattern on an assigned issue.",
"",
"## Source",
"",
`- Source issue: ${issueUiLink(evidence.sourceIssue, prefix)}`,
`- Assigned agent: ${evidence.sourceAgent.name} (${evidence.sourceAgent.role})`,
`- Primary trigger: \`${evidence.trigger}\` (${formatTrigger(evidence.trigger)})`,
`- Trigger reasons: ${evidence.triggerReasons.join("; ")}`,
`- Generated at: ${evidence.generatedAt.toISOString()}`,
"",
"## Evidence",
"",
`- Total sampled issue-linked runs: ${evidence.totalRunCount}`,
`- Terminal sampled runs: ${evidence.terminalRunCount}`,
`- Active queued/running/scheduled runs: ${evidence.activeRunCount}`,
`- No-comment completed-run streak: ${evidence.noCommentStreak}`,
`- Current active elapsed time: ${msToHuman(evidence.elapsedMs)}`,
`- Runs in rolling windows: ${evidence.runCountLastHour}/1h, ${evidence.runCountLastSixHours}/6h`,
`- Assignee run-linked comments total/window: ${evidence.commentCount} total, ${evidence.commentCountLastHour}/1h, ${evidence.commentCountLastSixHours}/6h`,
`- Cost events total: ${evidence.costCents} cents`,
`- Current next action: ${evidence.nextAction ? truncateInline(evidence.nextAction, 500) : "none recorded"}`,
"",
"## Thresholds",
"",
`- No-comment streak: ${evidence.thresholds.noCommentStreakRuns} completed runs`,
`- Long active duration: ${msToHuman(evidence.thresholds.longActiveMs)}`,
`- High churn: ${evidence.thresholds.highChurnHourly}/1h or ${evidence.thresholds.highChurnSixHours}/6h runs/assignee-run comments`,
`- Resolved-review snooze: ${msToHuman(evidence.thresholds.resolvedSnoozeMs)}`,
"",
"## Latest Runs",
"",
latestRuns,
"",
"## Latest Assignee Run Comments",
"",
latestComments,
"",
"## Usage Samples",
"",
usage,
"",
"## Manager Decision",
"",
"- Close as productive if this pattern is expected.",
"- Continue with a snooze window if the current work should keep running without repeat review spam.",
"- Request decomposition, reroute, block with an unblock owner, or stop/cancel the source work if the work is inefficient.",
].join("\n");
}
function buildRefreshComment(evidence: ProductivityReviewEvidence, prefix: string) {
return [
"Productivity review evidence refreshed.",
"",
`- Source issue: ${issueUiLink(evidence.sourceIssue, prefix)}`,
`- Trigger: \`${evidence.trigger}\` (${formatTrigger(evidence.trigger)})`,
`- Reasons: ${evidence.triggerReasons.join("; ")}`,
`- No-comment streak: ${evidence.noCommentStreak}`,
`- Runs/assignee comments: ${evidence.runCountLastHour}/${evidence.commentCountLastHour} in 1h, ${evidence.runCountLastSixHours}/${evidence.commentCountLastSixHours} in 6h`,
`- Next action: ${evidence.nextAction ? truncateInline(evidence.nextAction, 300) : "none recorded"}`,
].join("\n");
}
async function createOrUpdateReview(
evidence: ProductivityReviewEvidence,
opts: { prefix: string; thresholds: ProductivityReviewThresholds },
) {
const existing = await findOpenProductivityReview(evidence.sourceIssue.companyId, evidence.sourceIssue.id);
if (existing) {
const refreshState = await getRefreshCommentState(evidence.sourceIssue.companyId, existing.id);
const lastRefreshOrCreationAt = refreshState.latestCreatedAt ?? existing.createdAt;
if (
refreshState.count >= opts.thresholds.maxRefreshComments ||
evidence.generatedAt.getTime() - lastRefreshOrCreationAt.getTime() < opts.thresholds.refreshIntervalMs
) {
return { kind: "existing" as const, reviewIssueId: existing.id };
}
await addRefreshComment(existing.id, buildRefreshComment(evidence, opts.prefix), evidence.generatedAt);
await logActivity(db, {
companyId: evidence.sourceIssue.companyId,
actorType: "system",
actorId: "system",
action: "issue.productivity_review_updated",
entityType: "issue",
entityId: existing.id,
agentId: existing.assigneeAgentId,
details: {
source: "productivity_review.reconcile",
sourceIssueId: evidence.sourceIssue.id,
trigger: evidence.trigger,
noCommentStreak: evidence.noCommentStreak,
runCountLastHour: evidence.runCountLastHour,
commentCountLastHour: evidence.commentCountLastHour,
},
});
return { kind: "updated" as const, reviewIssueId: existing.id };
}
const recentCreationCount = await countRecentProductivityReviews(
evidence.sourceIssue.companyId,
evidence.sourceIssue.id,
opts.thresholds,
evidence.generatedAt,
);
if (recentCreationCount >= opts.thresholds.maxCreationsPerWindow) {
return { kind: "creation_capped" as const, reviewIssueId: null };
}
const consecutiveNoActionReviews = await countConsecutiveNoActionProductivityReviews(
evidence.sourceIssue.companyId,
evidence.sourceIssue.id,
opts.thresholds,
);
if (consecutiveNoActionReviews >= opts.thresholds.maxConsecutiveNoActionReviews) {
return { kind: "no_action_suppressed" as const, reviewIssueId: null };
}
const ownerAgentId = await resolveReviewOwnerAgentId(evidence.sourceIssue, evidence.sourceAgent);
let review: Awaited<ReturnType<typeof issuesSvc.create>>;
try {
review = await issuesSvc.create(evidence.sourceIssue.companyId, {
title: `Review productivity for ${evidence.sourceIssue.identifier ?? evidence.sourceIssue.title}`,
description: buildReviewMarkdown(evidence, opts.prefix),
status: "todo",
priority: evidence.trigger === "long_active_duration" ? "medium" : "high",
parentId: evidence.sourceIssue.id,
projectId: evidence.sourceIssue.projectId,
goalId: evidence.sourceIssue.goalId,
billingCode: evidence.sourceIssue.billingCode,
assigneeAgentId: ownerAgentId,
originKind: PRODUCTIVITY_REVIEW_ORIGIN_KIND,
originId: evidence.sourceIssue.id,
originFingerprint: productivityReviewFingerprint(evidence.sourceIssue.id),
requestDepth: clampIssueRequestDepth(evidence.sourceIssue.requestDepth + 1),
});
} catch (error) {
const maybe = error as { code?: string; constraint?: string; message?: string };
const uniqueConflict = maybe.code === "23505" &&
(
maybe.constraint === "issues_active_productivity_review_uq" ||
typeof maybe.message === "string" && maybe.message.includes("issues_active_productivity_review_uq")
);
if (!uniqueConflict) throw error;
const raced = await findOpenProductivityReview(evidence.sourceIssue.companyId, evidence.sourceIssue.id);
if (!raced) throw error;
return { kind: "existing" as const, reviewIssueId: raced.id };
}
await db
.update(issues)
.set({ createdAt: evidence.generatedAt, updatedAt: evidence.generatedAt })
.where(eq(issues.id, review.id));
await logActivity(db, {
companyId: evidence.sourceIssue.companyId,
actorType: "system",
actorId: "system",
action: "issue.productivity_review_created",
entityType: "issue",
entityId: review.id,
agentId: ownerAgentId,
details: {
source: "productivity_review.reconcile",
sourceIssueId: evidence.sourceIssue.id,
trigger: evidence.trigger,
noCommentStreak: evidence.noCommentStreak,
runCountLastHour: evidence.runCountLastHour,
commentCountLastHour: evidence.commentCountLastHour,
},
});
if (ownerAgentId && deps?.enqueueWakeup) {
await deps.enqueueWakeup(ownerAgentId, {
source: "assignment",
triggerDetail: "system",
reason: "issue_assigned",
payload: withRecoveryContext({
issueId: review.id,
sourceIssueId: evidence.sourceIssue.id,
trigger: evidence.trigger,
}, "status_only"),
requestedByActorType: "system",
requestedByActorId: "productivity_review",
contextSnapshot: withRecoveryContext({
issueId: review.id,
taskId: review.id,
wakeReason: "issue_assigned",
source: PRODUCTIVITY_REVIEW_ORIGIN_KIND,
sourceIssueId: evidence.sourceIssue.id,
productivityReviewTrigger: evidence.trigger,
}, "status_only"),
});
}
return { kind: "created" as const, reviewIssueId: review.id };
}
async function reconcileProductivityReviews(opts?: {
now?: Date;
companyId?: string;
thresholds?: Partial<ProductivityReviewThresholds>;
issueCreatedAtGte?: Date | null;
}) {
const now = opts?.now ?? new Date();
const thresholds = buildThresholds(opts?.thresholds);
const candidates = await db
.select()
.from(issues)
.where(
and(
opts?.companyId ? eq(issues.companyId, opts.companyId) : undefined,
visibleIssueCondition(),
isNull(issues.assigneeUserId),
inArray(issues.status, ["todo", "in_progress"]),
sql`${issues.assigneeAgentId} is not null`,
sql`${issues.originKind} <> ${PRODUCTIVITY_REVIEW_ORIGIN_KIND}`,
opts?.issueCreatedAtGte ? gte(issues.createdAt, opts.issueCreatedAtGte) : undefined,
),
)
.orderBy(asc(issues.updatedAt), asc(issues.id))
.limit(MAX_CANDIDATE_ISSUES);
const result = {
scanned: candidates.length,
created: 0,
updated: 0,
existing: 0,
snoozed: 0,
creationCapped: 0,
noActionSuppressed: 0,
skipped: 0,
failed: 0,
reviewIssueIds: [] as string[],
failedIssueIds: [] as string[],
};
const prefixCache = new Map<string, string>();
for (const candidate of candidates) {
if (!candidate.assigneeAgentId) {
result.skipped += 1;
continue;
}
if (await isProductivityReviewDescendant(candidate)) {
result.skipped += 1;
continue;
}
if (await findRecentTerminalProductivityReview(candidate.companyId, candidate.id, thresholds, now)) {
result.snoozed += 1;
continue;
}
const sourceAgent = await getAgent(candidate.assigneeAgentId);
if (!sourceAgent || sourceAgent.companyId !== candidate.companyId) {
result.skipped += 1;
continue;
}
// A paused assignee cannot act on a review, so raising one only creates noise.
if (sourceAgent.status === "paused") {
result.skipped += 1;
continue;
}
const evidence = await collectEvidence(candidate, sourceAgent, thresholds, now);
if (!evidence) {
result.skipped += 1;
continue;
}
let prefix = prefixCache.get(candidate.companyId);
if (!prefix) {
prefix = await getCompanyIssuePrefix(candidate.companyId);
prefixCache.set(candidate.companyId, prefix);
}
try {
const outcome = await createOrUpdateReview(evidence, { prefix, thresholds });
if (outcome.kind === "created") result.created += 1;
else if (outcome.kind === "updated") result.updated += 1;
else if (outcome.kind === "creation_capped") result.creationCapped += 1;
else if (outcome.kind === "no_action_suppressed") result.noActionSuppressed += 1;
else result.existing += 1;
if (outcome.reviewIssueId) result.reviewIssueIds.push(outcome.reviewIssueId);
} catch (err) {
result.failed += 1;
result.failedIssueIds.push(candidate.id);
logger.warn(
{
err,
companyId: candidate.companyId,
issueId: candidate.id,
requestDepth: candidate.requestDepth,
},
"productivity review reconciliation skipped malformed candidate",
);
}
}
return result;
}
async function isProductivityReviewContinuationHoldActive(input: {
companyId: string;
issueId: string;
agentId: string;
now?: Date;
thresholds?: Partial<ProductivityReviewThresholds>;
}) {
const now = input.now ?? new Date();
const thresholds = buildThresholds(input.thresholds);
const [sourceIssue, sourceAgent, openReview] = await Promise.all([
db
.select()
.from(issues)
.where(and(eq(issues.companyId, input.companyId), eq(issues.id, input.issueId)))
.then((rows) => rows[0] ?? null),
getAgent(input.agentId),
findOpenProductivityReview(input.companyId, input.issueId),
]);
if (!sourceIssue || !sourceAgent || !openReview) return { held: false as const };
if (sourceAgent.companyId !== input.companyId) return { held: false as const };
const evidence = await collectEvidence(sourceIssue, sourceAgent, thresholds, now);
if (!evidence || !isSoftStopTrigger(evidence.trigger)) return { held: false as const };
return {
held: true as const,
reviewIssueId: openReview.id,
reviewIdentifier: openReview.identifier,
trigger: evidence.trigger,
reason: evidence.triggerReasons.join("; "),
};
}
async function recordContinuationHold(input: {
companyId: string;
issueId: string;
runId: string;
agentId: string;
reviewIssueId: string;
trigger: ProductivityReviewTrigger;
reason: string;
}) {
await logActivity(db, {
companyId: input.companyId,
actorType: "system",
actorId: "system",
agentId: input.agentId,
runId: input.runId,
action: "issue.productivity_review_continuation_held",
entityType: "issue",
entityId: input.issueId,
details: {
source: "productivity_review.continuation_hold",
reviewIssueId: input.reviewIssueId,
trigger: input.trigger,
reason: input.reason,
},
});
}
return {
reconcileProductivityReviews,
isProductivityReviewContinuationHoldActive,
recordContinuationHold,
};
}
+1
View File
@@ -1,5 +1,6 @@
export const RECOVERY_ORIGIN_KINDS = {
issueGraphLivenessEscalation: "harness_liveness_escalation",
// Historical tasks retain their origin and recovery-recursion exclusion.
issueProductivityReview: "issue_productivity_review",
strandedIssueRecovery: "stranded_issue_recovery",
staleActiveRunEvaluation: "stale_active_run_evaluation",
@@ -0,0 +1,52 @@
import { and, eq } from "drizzle-orm";
import { environmentLeases, type Db } from "@paperclipai/db";
type LeaseIdentity = {
id: string; companyId: string; heartbeatRunId: string | null;
provider: string | null; providerLeaseId: string | null;
};
/** Bind a provider receipt to the exact host-owned lease and run. Old plugins
* return void; that remains supported but grants no continuation authority. */
export function remoteTerminationReceipt(lease: LeaseIdentity, value: unknown) {
const receipt = value as { providerLeaseId?: unknown; state?: unknown } | null;
if (!lease.heartbeatRunId || !lease.provider || lease.provider === "local" ||
!lease.providerLeaseId || receipt?.providerLeaseId !== lease.providerLeaseId ||
!["stopped", "destroyed"].includes(String(receipt?.state))) return undefined;
return {
schema: "paperclip.remote-termination.v1", companyId: lease.companyId,
runId: lease.heartbeatRunId, leaseId: lease.id, provider: lease.provider,
providerLeaseId: lease.providerLeaseId, state: receipt!.state,
confirmedAt: new Date().toISOString(),
};
}
export function hasRemoteTerminationReceipt(lease: LeaseIdentity & {
releasedAt: unknown; cleanupStatus: string | null; status: string;
metadata: Record<string, unknown> | null;
}): boolean {
const receipt = lease.metadata?.remoteExecutionTermination as Record<string, unknown> | undefined;
return Boolean(lease.releasedAt && lease.cleanupStatus === "success" &&
["released", "expired", "failed"].includes(lease.status) && receipt &&
receipt.schema === "paperclip.remote-termination.v1" &&
receipt.companyId === lease.companyId && receipt.runId === lease.heartbeatRunId &&
receipt.leaseId === lease.id && receipt.provider === lease.provider &&
remoteTerminationReceipt(lease, receipt));
}
export function remoteLeaseCleanupScope(lease: Pick<LeaseIdentity, "provider" | "providerLeaseId">) {
return lease.provider && lease.provider !== "local" && lease.providerLeaseId
? JSON.stringify([lease.provider, lease.providerLeaseId]) : undefined;
}
export async function stoppedRemoteCleanupScopes(db: Db, companyId: string, runId: string) {
const leases = await db.select().from(environmentLeases).where(and(
eq(environmentLeases.companyId, companyId), eq(environmentLeases.heartbeatRunId, runId),
));
if (leases.length === 0 || !leases.every(hasRemoteTerminationReceipt)) return null;
return [...new Set(leases.map(lease => remoteLeaseCleanupScope(lease)!))];
}
export async function remoteExecutionHasStopped(db: Db, companyId: string, runId: string) {
return await stoppedRemoteCleanupScopes(db, companyId, runId) !== null;
}
+50 -6
View File
@@ -205,6 +205,7 @@ import {
splitRemoteUrlCredential,
} from "./remote-url-credentials.js";
import { secretService } from "./secrets.js";
import { agentmailApi } from "./agentmail-api.js";
import { toolAccessPolicyService } from "./tool-access-policy.js";
import {
readSignedToolArgumentsPayload,
@@ -7363,11 +7364,48 @@ export function toolAccessService(
};
}
function isAgentMailConnection(connection: typeof toolConnections.$inferSelect) {
return connection.transport === "rest_api" && connection.config.provider === "agentmail";
}
async function validateAgentMailConnection(
connection: typeof toolConnections.$inferSelect,
) {
const configPath = connection.config.emailCredential
? "credentials.controlKey"
: "credentials.apiKey";
const ref = connection.credentialSecretRefs.find(
(candidate) => candidate.configPath === configPath,
);
if (!ref) {
throw unprocessable("Reconnect AgentMail to restore its API key", {
code: "missing_secret",
});
}
const key = await secrets.resolveSecretValue(
connection.companyId,
ref.secretId,
ref.versionSelector ?? "latest",
{
consumerType: "tool_connection",
consumerId: connection.id,
configPath,
actorType: "system",
actorId: null,
},
);
await agentmailApi(key).whoami();
}
async function discoverTools(
connection: typeof toolConnections.$inferSelect,
credentialHeaders?: Record<string, string>,
actor?: ActorInfo,
): Promise<McpToolDescriptor[]> {
if (isAgentMailConnection(connection)) {
await validateAgentMailConnection(connection);
return [];
}
if (connection.transport === "mcp_remote")
return remoteTools(connection, credentialHeaders, actor);
if (isComposioConnection(connection)) {
@@ -7495,6 +7533,8 @@ export function toolAccessService(
});
for (const grant of grantsToCheck)
await refreshManagedGitHubGrantAccess(connection, grant, actor);
} else if (isAgentMailConnection(connection)) {
await validateAgentMailConnection(connection);
} else if (connection.transport === "mcp_remote") {
await assertComposioConnectedAccountActive(connection);
const credentialHeaders =
@@ -7516,11 +7556,13 @@ export function toolAccessService(
config.sourceTemplateKey === "github" &&
oauth.connectorProfile === "github.code"
? "GitHub account, installation, and repository access are available."
: isComposioConnection(connection)
? "Composio accepted the API key and returned its toolkits."
: connection.transport === "local_stdio"
? "Approved stdio template is ready."
: "Remote MCP server responded to tools/list.",
: isAgentMailConnection(connection)
? "AgentMail API key is connected."
: isComposioConnection(connection)
? "Composio accepted the API key and returned its toolkits."
: connection.transport === "local_stdio"
? "Approved stdio template is ready."
: "Remote MCP server responded to tools/list.",
);
const runtimeSlot = await ensureRuntimeSlot(updated);
await audit({
@@ -7759,7 +7801,9 @@ export function toolAccessService(
config: normalizedConfig,
transportConfig: normalizedTransportConfig,
healthStatus: "ok",
healthMessage: "Tool catalog refreshed.",
healthMessage: isAgentMailConnection(connection)
? "AgentMail API key is connected."
: "Tool catalog refreshed.",
healthCheckedAt: refreshedAt,
lastHealthAt: refreshedAt,
lastCatalogRefreshAt: refreshedAt,
@@ -356,7 +356,9 @@ function startInput(options?: {
command: options?.command ?? serviceCommand(),
env: { PAPERCLIP_PUBLIC_URL: "http://127.0.0.1:3100" },
port: options?.port ?? { type: "auto", envKey: "PORT" },
readiness: { type: "http", urlTemplate: "http://127.0.0.1:{{port}}", timeoutSec: 5 },
// These lifecycle tests spawn real servers; cold CI startup can take
// five seconds before the first listener is ready.
readiness: { type: "http", urlTemplate: "http://127.0.0.1:{{port}}", timeoutSec: 10 },
...(expose ? { expose } : {}),
}],
},