Jannik/PaperClipAI
1
0 Fork 0
mirror of https://github.com/paperclipai/paperclip.git synced 2026-11-03 09:37:19 +01:00
Code Issues Packages Projects Releases Wiki Activity

merge: preserve status-card and recovery producers in the review stack

Browse Source 
Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
DottaandPaperclip committed 2026-10-30 20:24:40 -05:00
2 parents 6469daec07 290d12cf4b
commit 9b4beff72b
126 files changed
+4453 -581
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

server/src/services/access.ts
+2
View File
@@ -1,4 +1,5 @@
import { notifyStatusCardIssueChange } from "./status-card-notifications.js";
import { notifyDeferredExecutionEligibility } from "./deferred-execution-notifications.js";
import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js";
import { notifyNativeStatusReassessment } from "./native-status-reassessment-notifications.js";
import { notifyUnsafeWorkspaceExportRecovery } from "./unsafe-workspace-export-notifications.js";
@@ -744,6 +745,7 @@ export function accessService(db: Db) {
sql`${issues.status} not in ('done', 'cancelled')`,
);
await notifyNativeStatusReassessment(tx, { companyId });
await notifyDeferredExecutionEligibility(tx, { companyId });
await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors);
await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers);
await notifyStatusCardIssueChange(tx, companyId);
server/src/services/agent-configuration.ts
+4
View File
@@ -1,3 +1,5 @@
import { notifyDeferredExecutionEligibility } from "./deferred-execution-notifications.js";
import { notifyRunDispatch } from "./run-dispatch-notifications.js";
import { notifySkillVersionPinChange } from "./skill-version-retention-notifications.js";
import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js";
import { notifyExternalConversationSettlement } from "./external-conversation-settlement-notifications.js";
@@ -83,6 +85,8 @@ export async function prepareAgentConfiguration(db: Db, existing: AgentConfigura
existing.adapterConfig,
);
}
await notifyRunDispatch(db, { companyId: existing.companyId });
await notifyDeferredExecutionEligibility(db, { companyId: existing.companyId });
if (normalizedPatch.runtimeConfig !== undefined || normalizedPatch.reportsTo !== undefined) {
await notifyDeliveryWork(db, DELIVERY_QUEUES.agentTimers);
}
server/src/services/agent-conversations.ts
+2
View File
@@ -1,4 +1,5 @@
import { notifyStatusCardIssueChange } from "./status-card-notifications.js";
import { notifyIssueRecovery } from "./issue-recovery-notifications.js";
import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js";
import { notifyUnsafeWorkspaceExportRecovery } from "./unsafe-workspace-export-notifications.js";
import { FAST_RESPONSE_AGENT_GUIDANCE, fastResponseHistoryBody } from "@paperclipai/shared";
@@ -547,6 +548,7 @@ export async function resumeConversationForReset(db: Db, comment: typeof issueCo
eq(issues.id, comment.issueId), eq(issues.companyId, comment.companyId),
)).for("update");
if (!isConversation(issue) || comment.conversationSessionGeneration != null) return;
await notifyIssueRecovery(tx, { companyId: issue.companyId });
const released = await tx.update(issueTreeHolds).set({
status: "released", releasedAt: new Date(), updatedAt: new Date(),
releasedByActorType: "user", releasedByUserId: comment.authorUserId,
server/src/services/agent-lifecycle.ts
+7 -1
View File
@@ -1,3 +1,5 @@
import { notifyDeferredExecutionEligibility } from "./deferred-execution-notifications.js";
import { notifyRunDispatch } from "./run-dispatch-notifications.js";
import { notifyNativeReviewResponse } from "./native-review-response-notifications.js";
import { isAgentStatusInvokable } from "@paperclipai/shared";
import { notifyNativeResponsePresentation } from "./native-response-presentation-notifications.js";
@@ -38,7 +40,11 @@ export function createAgentLifecycleEffects(_hooks: BudgetServiceHooks = {}): li
},
notifyWork: tx => notifyDeliveryWork(tx, DELIVERY_QUEUES.agentLifecycle),
async beforeStatusChange(tx, companyId, agentId, before, after) {
if (before !== after) await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers);
if (before !== after) {
await notifyRunDispatch(tx, { companyId });
await notifyDeferredExecutionEligibility(tx, { companyId });
await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers);
}
if (!isAgentStatusInvokable(before) && isAgentStatusInvokable(after)) {
await notifyNativeResponsePresentation(tx, { companyId, agentId });
await notifyNativeReviewResponse(tx, { companyId, agentId });
server/src/services/budgets.ts
+32 -2
View File
@@ -1,4 +1,6 @@
import { notifyStatusCardIssueChange } from "./status-card-notifications.js";
import { notifyDeferredExecutionEligibility } from "./deferred-execution-notifications.js";
import { notifyRunDispatch } from "./run-dispatch-notifications.js";
import { notifyApprovalWork } from "./approval-work-notifications.js";
import { notifyCompanyRetentionWork } from "./retention-work-notifications.js";
import { addCents, compareCents, normalizeCents } from "@paperclipai/shared";
@@ -164,6 +166,19 @@ async function resolveScopeRecord(db: Db, scopeType: BudgetScopeType, scopeId: s
};
}
async function resolveSummaryScopeRecord(db: Db, companyId: string, scopeType: BudgetScopeType, scopeId: string): Promise<ScopeRecord> {
let scope: ScopeRecord;
try {
scope = await resolveScopeRecord(db, scopeType, scopeId);
} catch (error) {
if (!(error instanceof HttpError) || error.status !== 404 || (scopeType !== "agent" && scopeType !== "project")) throw error;
return { companyId, name: scopeType === "agent" ? "Deleted agent" : "Deleted project",
paused: false, pauseReason: null, budgetHeld: false };
}
if (scope.companyId !== companyId) throw notFound("Budget scope not found");
return scope;
}
// Keep a large dashboard from filling the database pool with queued reads.
async function mapBudgetReads<T, R>(rows: T[], read: (row: T) => Promise<R>): Promise<R[]> {
const result: R[] = [];
@@ -288,6 +303,8 @@ export function budgetServiceInTransaction(db: Db, publications: ActivityPublica
}
if (policy.scopeType === "project") {
await notifyRunDispatch(db, { companyId: policy.companyId });
await notifyDeferredExecutionEligibility(db, { companyId: policy.companyId });
await db
.update(projects)
.set({
@@ -301,6 +318,8 @@ export function budgetServiceInTransaction(db: Db, publications: ActivityPublica
const scope = await resolveScopeRecord(db, "company", policy.scopeId);
if (scope.paused) return;
await notifyRunDispatch(db, { companyId: policy.companyId });
await notifyDeferredExecutionEligibility(db, { companyId: policy.companyId });
await notifyCompanyRetentionWork(db, policy.companyId);
await notifyStatusCardIssueChange(db, policy.companyId);
await notifyDeliveryWork(db, DELIVERY_QUEUES.agentTimers);
@@ -348,6 +367,8 @@ export function budgetServiceInTransaction(db: Db, publications: ActivityPublica
}
if (policy.scopeType === "project") {
await notifyRunDispatch(db, { companyId: policy.companyId });
await notifyDeferredExecutionEligibility(db, { companyId: policy.companyId });
await db
.update(projects)
.set({
@@ -362,6 +383,8 @@ export function budgetServiceInTransaction(db: Db, publications: ActivityPublica
await notifyDeliveryWork(db, DELIVERY_QUEUES.skillSourceSync);
const scope = await resolveScopeRecord(db, "company", policy.scopeId);
if (scope.pauseReason !== "budget") return;
await notifyRunDispatch(db, { companyId: policy.companyId });
await notifyDeferredExecutionEligibility(db, { companyId: policy.companyId });
await notifyCompanyRetentionWork(db, policy.companyId);
await notifyStatusCardIssueChange(db, policy.companyId);
await notifyDeliveryWork(db, DELIVERY_QUEUES.agentTimers);
@@ -398,7 +421,7 @@ export function budgetServiceInTransaction(db: Db, publications: ActivityPublica
}
async function buildPolicySummary(policy: PolicyRow): Promise<BudgetPolicySummary> {
const scope = await resolveScopeRecord(db, policy.scopeType as BudgetScopeType, policy.scopeId);
const scope = await resolveSummaryScopeRecord(db, policy.companyId, policy.scopeType as BudgetScopeType, policy.scopeId);
const observed = await computeObservedSpend(db, policy);
const { total: observedAmount, totalExact: observedAmountExact, unpricedEventCount, pendingRunCount } = observed;
const { windowStart: start, windowEnd: end } = observed;
@@ -563,7 +586,7 @@ export function budgetServiceInTransaction(db: Db, publications: ActivityPublica
const approvalStatusById = new Map(approvalRows.map((row) => [row.id, row.status]));
return mapBudgetReads(rows, async (row) => {
const scope = await resolveScopeRecord(db, row.scopeType as BudgetScopeType, row.scopeId);
const scope = await resolveSummaryScopeRecord(db, row.companyId, row.scopeType as BudgetScopeType, row.scopeId);
return {
id: row.id,
companyId: row.companyId,
@@ -928,6 +951,13 @@ export function budgetServiceInTransaction(db: Db, publications: ActivityPublica
if (incident.companyId !== companyId) throw notFound("Budget incident not found");
const policy = await getPolicyRow(incident.policyId);
if (policy.companyId !== companyId || policy.scopeType !== incident.scopeType || policy.scopeId !== incident.scopeId) {
throw notFound("Budget scope not found");
}
const scope = input.action === "keep_paused"
? await resolveSummaryScopeRecord(db, companyId, policy.scopeType as BudgetScopeType, policy.scopeId)
: await resolveScopeRecord(db, policy.scopeType as BudgetScopeType, policy.scopeId);
if (scope.companyId !== companyId) throw notFound("Budget scope not found");
if (input.action === "raise_budget_and_resume") {
const nextAmount = Math.max(0, Math.floor(input.amount ?? 0));
const observed = await computeObservedSpend(db, policy);
server/src/services/chat-channels/failed-run-retry.ts
+5
View File
@@ -1,3 +1,4 @@
import { notifyDeferredExecutionWork, notifyTaskRetrySettlement } from "../deferred-execution-notifications.js";
import { notifyNativeResponsePresentation } from "../native-response-presentation-notifications.js";
import { notifyChatInboundWakeNotices } from "../chat-inbound-wake-notice-notifications.js";
import { createFailedRetryNotices } from "./failed-retry-notices.js";
@@ -1424,6 +1425,7 @@ export function createChatFailedRunRetry(db: Db, options: ChatFailedRunRetryOpti
// Source authorization above checked every external receipt and its owner.
// Reset only this certified unstarted batch under the task's admission lock.
const skippedIds = source.taskRetryInput?.skippedActionIds ?? [];
await notifyDeferredExecutionWork(tx, { companyId: input.companyId });
await tx.update(agentWakeupRequests).set({ status: "deferred_issue_execution", runId: null, finishedAt: null,
payload: sql`${agentWakeupRequests.payload} - 'coalescedIntoWakeupRequestId'`, updatedAt: new Date(),
}).where(and(eq(agentWakeupRequests.companyId, input.companyId), eq(agentWakeupRequests.agentId, input.agentId),
@@ -1784,6 +1786,9 @@ export function createChatFailedRunRetry(db: Db, options: ChatFailedRunRetryOpti
const attemptCount = Number(candidate.result?.attemptCount ?? 0) + 1;
const settle = async (status: string, result: Record<string, unknown>) => {
const [current] = await db.transaction(async (tx) => {
if (typeof claimed.payload.taskRetryIntentId === "string") await notifyTaskRetrySettlement(tx, {
companyId: claimed.companyId, requestId: claimed.payload.taskRetryIntentId,
});
await notifyChatActionWork(tx, "failed_run_retry");
if (status === "processed") await notifyNativeResponsePresentation(tx, {
companyId: claimed.companyId, issueId: String(claimed.payload.issueId),
server/src/services/chat-channels/task-retry-input.ts
+2
View File
@@ -1,3 +1,4 @@
import { notifyDeferredExecutionWork } from "../deferred-execution-notifications.js";
import { z } from "zod";
import { and, asc, eq, gt, inArray, isNull, or, sql } from "drizzle-orm";
import { agentWakeupRequests, chatActions, chatConversations, chatEndpoints, heartbeatRuns, issues, type Db } from "@paperclipai/db";
@@ -91,6 +92,7 @@ export async function releaseUnadmittedTaskRetryInput(db: Db, input: { companyId
)).for("update");
if (receipt?.runId) return;
await notifyChatInboundWakeNotices(tx, { companyId: input.companyId, issueId: issue.id });
await notifyDeferredExecutionWork(tx, { companyId: input.companyId });
await tx.update(agentWakeupRequests).set({ status: "deferred_issue_execution", finishedAt: null,
payload: sql`${agentWakeupRequests.payload} - 'coalescedIntoWakeupRequestId'`, updatedAt: new Date(),
}).where(and(eq(agentWakeupRequests.companyId, input.companyId), eq(agentWakeupRequests.status, "coalesced"),
server/src/services/chat-channels/telegram/attachment-failure.ts
+2
View File
@@ -1,4 +1,5 @@
import { notifyStatusCardIssueChange } from "../../status-card-notifications.js";
import { notifyIssueRecovery } from "../../issue-recovery-notifications.js";
import { createProviderEffects, type ProviderEffectTarget } from "../provider-effects.js";
import type { EndpointRow, RuntimeContext, InboundRuntimeContext } from "../runtime-types.js";
@@ -79,6 +80,7 @@ export function createTelegramAttachmentFailure(db: Db, options: TelegramAttachm
})(),
);
const effect = await db.transaction(async (tx) => {
await notifyIssueRecovery(tx, { companyId: input.endpoint.companyId });
// Replace the optimistic attachment placeholder with an explicit safe
// Paperclip-visible failure. The message link stays authoritative for
// exact provider redelivery, while no agent wakeup is enqueued for an
server/src/services/chat-run-milestone-notifications.ts
+2
View File
@@ -1,3 +1,4 @@
import { notifyRunDispatch } from "./run-dispatch-notifications.js";
import { notifyIssueReviewState } from "./issue-review-state-notifications.js";
import { notifyNativeResponsePresentation } from "./native-response-presentation-notifications.js";
import { notifyChatResponseAvailability } from "./chat-response-availability.js";
@@ -11,6 +12,7 @@ import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notificatio
export type ChatRunMilestoneScope = { companyId: string; issueId?: string; runId?: string };
export async function notifyChatRunMilestones(tx: object, scope: ChatRunMilestoneScope) {
if (!scope.issueId && !scope.runId) await notifyRunDispatch(tx, scope);
if (scope.issueId || scope.runId) {
await notifyIssueReviewState(tx, { companyId: scope.companyId, issueId: scope.issueId,
runId: scope.issueId ? undefined : scope.runId });
server/src/services/companies.ts
+6
View File
@@ -1,4 +1,6 @@
import { notifyStatusCardIssueChange } from "./status-card-notifications.js";
import { notifyDeferredExecutionEligibility } from "./deferred-execution-notifications.js";
import { notifyRunDispatch } from "./run-dispatch-notifications.js";
import { notifyChatInboundWakeNotices } from "./chat-inbound-wake-notice-notifications.js";
import { notifyCompanyRetentionWork } from "./retention-work-notifications.js";
import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js";
@@ -333,6 +335,8 @@ export function companyService(db: Db, _budgetHooks: BudgetServiceHooks = {}) {
const renamedPrefix = await resolveRenamedIssuePrefix(tx, id, companyPatch);
if (companyPatch.status !== undefined && companyPatch.status !== existing.status) {
await notifyRunDispatch(tx, { companyId: id });
await notifyDeferredExecutionEligibility(tx, { companyId: id });
await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers);
await notifyDeliveryWork(tx, DELIVERY_QUEUES.skillSourceSync);
await notifyDeliveryWork(tx, DELIVERY_QUEUES.routineSchedules);
@@ -490,6 +494,8 @@ export function companyService(db: Db, _budgetHooks: BudgetServiceHooks = {}) {
const wasAlreadyArchived = existing.status === "archived";
if (!wasAlreadyArchived) {
await notifyRunDispatch(tx, { companyId: id });
await notifyDeferredExecutionEligibility(tx, { companyId: id });
await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers);
await notifyDeliveryWork(tx, DELIVERY_QUEUES.skillSourceSync);
await notifyDeliveryWork(tx, DELIVERY_QUEUES.routineSchedules);
server/src/services/company-deletion.ts
+2
View File
@@ -1,4 +1,5 @@
import { notifyStatusCardEvaluation } from "./status-card-notifications.js";
import { notifyIssueRecovery } from "./issue-recovery-notifications.js";
import { notifyUnsafeWorkspaceExportRecovery } from "./unsafe-workspace-export-notifications.js";
import { notifyCancelledWorkspacePreservation } from "./cancelled-workspace-preservation-notifications.js";
import { notifyInstructionCollection } from "./instruction-collection-notifications.js";
@@ -72,6 +73,7 @@ export async function deleteCompany(db: Db, id: string) {
const [existing] = await tx.select({ id: companies.id }).from(companies)
.where(eq(companies.id, id)).for("no key update");
if (!existing) return null;
await notifyIssueRecovery(tx, { companyId: id });
await notifyChatCompanyConfiguration(tx, id);
await chatCompanyDeletion.deleteCompanyData(tx, id);
await notifyDeliveryWork(tx, DELIVERY_QUEUES.skillSourceSync);
server/src/services/company-portability.ts
+5 -1
View File
@@ -1,3 +1,4 @@
import { notifyIssueRecovery } from "./issue-recovery-notifications.js";
import { createAgentLifecycle } from "./agent-lifecycle.js";
import { agentAppearanceSchema, agentHarnessType, paperclipRunnerProfileForHarness, agentRunner, type AgentRunnerChoice } from "@paperclipai/shared";
import { resolveNewAgentRunnerForCompany } from "./agent-runner-selection.js";
@@ -6465,7 +6466,9 @@ export function companyPortabilityService(db: Db, storage?: StorageService) {
}
if (relationRows.length > 0) {
const relationCompanyId = targetCompany.id;
await db
await db.transaction(async tx => {
await notifyIssueRecovery(tx, { companyId: relationCompanyId });
await tx
.insert(issueRelations)
.values(relationRows.map((row) => ({
companyId: relationCompanyId,
@@ -6476,6 +6479,7 @@ export function companyPortabilityService(db: Db, storage?: StorageService) {
createdByUserId: actorUserId ?? null,
})))
.onConflictDoNothing();
});
}
}
server/src/services/deferred-execution-notifications.ts
+34
View File
@@ -0,0 +1,34 @@
import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js";
export type DeferredExecutionScope = { companyId: string; requestId?: string };
export type DeferredReceiptRecoveryOptions = {
companyId?: string;
requestId?: string;
onError?: (error: unknown) => void;
};
function hint(scope: DeferredExecutionScope) {
return JSON.stringify({ companyId: scope.companyId, requestId: scope.requestId });
}
export async function notifyDeferredExecutionWork(tx: object, scope: DeferredExecutionScope) {
await notifyDeliveryWork(tx, DELIVERY_QUEUES.deferredExecutionReceipts, hint(scope));
}
export async function notifyNativeStatusWakeWork(tx: object, scope: DeferredExecutionScope) {
await notifyDeliveryWork(tx, DELIVERY_QUEUES.nativeStatusWakeIntents, hint(scope));
}
export async function notifyTaskRetrySettlement(tx: object, scope: DeferredExecutionScope) {
await notifyDeliveryWork(tx, DELIVERY_QUEUES.taskRetryReceiptSettlement, hint(scope));
}
export async function notifyDeferredExecutionEligibility(tx: object, scope?: { companyId: string }) {
if (scope) {
await notifyDeferredExecutionWork(tx, scope);
await notifyNativeStatusWakeWork(tx, scope);
} else {
await notifyDeliveryWork(tx, DELIVERY_QUEUES.deferredExecutionReceipts, "*");
await notifyDeliveryWork(tx, DELIVERY_QUEUES.nativeStatusWakeIntents, "*");
}
}
server/src/services/deferred-execution-wait.ts
+24
View File
@@ -0,0 +1,24 @@
import { and, eq, sql } from "drizzle-orm";
import { agentWakeupRequests, type Db } from "@paperclipai/db";
import { notifyDeferredExecutionWork } from "./deferred-execution-notifications.js";
export async function writeDeferredExecutionWait(db: Db, scope: { companyId: string; requestId: string },
condition: Record<string, unknown>, options: { replace?: boolean } = {}) {
return db.transaction(async tx => {
const [current] = await tx.select({ payload: agentWakeupRequests.payload }).from(agentWakeupRequests)
.where(and(eq(agentWakeupRequests.companyId, scope.companyId), eq(agentWakeupRequests.id, scope.requestId),
eq(agentWakeupRequests.status, "deferred_issue_execution"))).for("update");
if (!current) return;
const previous = current.payload?.executionWait;
const previousReason = previous && typeof previous === "object" && !Array.isArray(previous)
? (previous as Record<string, unknown>).reason : undefined;
if (!previous || previousReason !== condition.reason) await notifyDeferredExecutionWork(tx, scope);
await tx.update(agentWakeupRequests).set({
payload: sql`jsonb_set(coalesce(${agentWakeupRequests.payload}, '{}'::jsonb), '{executionWait}',
${options.replace ? sql`'{}'::jsonb` : sql`coalesce(${agentWakeupRequests.payload}->'executionWait', '{}'::jsonb)`}
|| ${JSON.stringify(condition)}::jsonb)`,
updatedAt: new Date(),
}).where(and(eq(agentWakeupRequests.companyId, scope.companyId), eq(agentWakeupRequests.id, scope.requestId),
eq(agentWakeupRequests.status, "deferred_issue_execution")));
});
}
server/src/services/delivery-work-notifications.ts
+11
View File
@@ -2,6 +2,17 @@ import { signalDatabaseWork, subscribeDatabaseWork } from "@paperclipai/db";
/** Topics for existing durable queues; register intent before their writes. */
export const DELIVERY_QUEUES = {
deferredExecutionReceipts: "deferred-execution-receipts",
taskRetryReceiptSettlement: "task-retry-receipt-settlement",
nativeStatusWakeIntents: "native-status-wake-intents",
orphanBlockers: "orphan-blockers",
activeRecoveryActions: "active-recovery-actions",
strandedAssignments: "stranded-assignments",
resolvedDependencies: "resolved-dependencies",
taskWatchdogs: "task-watchdogs",
staleIssueLocks: "stale-issue-locks",
scheduledRetryDispatch: "scheduled-retry-dispatch",
queuedRunDispatch: "queued-run-dispatch",
statusCardEvaluation: "status-card-evaluation",
runEvidenceMigration: "run-evidence-migration",
providerTraceRetention: "provider-trace-retention",
server/src/services/documents.ts
+12 -1
View File
@@ -1,4 +1,5 @@
import { notifyStatusCardIssueChange } from "./status-card-notifications.js";
import { notifyIssueRecovery } from "./issue-recovery-notifications.js";
import { randomUUID } from "node:crypto";
import { and, asc, desc, eq, sql } from "drizzle-orm";
import type { Db } from "@paperclipai/db";
@@ -277,6 +278,7 @@ export function documentService(db: Db) {
.where(eq(issueDocuments.issueId, issue.id));
const fallbackKey = nextAvailableDocumentKey(key, issueDocumentKeys.map((row) => row.key));
await notifyIssueRecovery(tx, { companyId: issue.companyId });
const [document] = await tx
.insert(documents)
.values({
@@ -378,6 +380,7 @@ export function documentService(db: Db) {
});
}
await notifyIssueRecovery(tx, { companyId: issue.companyId });
const nextRevisionNumber = existing.latestRevisionNumber + 1;
const [revision] = await tx
.insert(documentRevisions)
@@ -440,6 +443,7 @@ export function documentService(db: Db) {
throw conflict("Document does not exist yet", { key });
}
await notifyIssueRecovery(tx, { companyId: issue.companyId });
const [document] = await tx
.insert(documents)
.values({
@@ -594,7 +598,10 @@ export function documentService(db: Db) {
});
}
await db.transaction(async (tx) => {
for (const companyId of new Set(rows.map(row => row.companyId))) await notifyStatusCardIssueChange(tx, companyId);
for (const companyId of new Set(rows.map(row => row.companyId))) {
await notifyStatusCardIssueChange(tx, companyId);
await notifyIssueRecovery(tx, { companyId });
}
await insertRowsInChunks(tx, documents, documentRows);
await insertRowsInChunks(tx, documentRevisions, revisionRows);
await insertRowsInChunks(tx, issueDocuments, linkRows);
@@ -647,6 +654,7 @@ export function documentService(db: Db) {
});
}
await notifyIssueRecovery(tx, { companyId: existing.companyId });
const now = new Date();
const nextRevisionNumber = existing.latestRevisionNumber + 1;
const [restoredRevision] = await tx
@@ -726,6 +734,7 @@ export function documentService(db: Db) {
};
}
await notifyIssueRecovery(tx, { companyId: existing.companyId });
const now = new Date();
await tx
.update(documents)
@@ -773,6 +782,7 @@ export function documentService(db: Db) {
};
}
await notifyIssueRecovery(tx, { companyId: existing.companyId });
const now = new Date();
await tx
.update(documents)
@@ -821,6 +831,7 @@ export function documentService(db: Db) {
});
}
await notifyIssueRecovery(tx, { companyId: existing.companyId });
await tx.delete(issueDocuments).where(eq(issueDocuments.documentId, existing.id));
await notifyStatusCardIssueChange(tx, existing.companyId);
await tx.delete(documents).where(eq(documents.id, existing.id));
server/src/services/environment-runtime.ts
+7 -1
View File
@@ -15,6 +15,7 @@ import { hasStopOnlyCleanup, prepareSandboxStopAndRetain, readStopOnlyCleanup, s
import { JsonRpcCallError, readEnvironmentAcquisitionDiagnostic, readEnvironmentCreationCleanupError } from "@paperclipai/plugin-sdk";
import { preserveEnvironmentSyncOutErrorDiagnostic } from "./environment-sync-out-error.js";
import { remoteTerminationReceipt } from "./remote-execution-termination.js";
import { inspectCloudSandboxCleanupAbsence } from "./sandbox-absence-proof.js";
import { hasNativeWorkspaceExportResume, releaseCompletedNativeWorkspaceExportRetention } from "./native-runtime/native-workspace-export-resume.js";
import { createHash, randomUUID } from "node:crypto";
import { and, eq, inArray, ne, or, sql } from "drizzle-orm";
@@ -4021,7 +4022,12 @@ export function environmentRuntimeService(
const lease = await environmentsSvc.getLeaseById(leaseId);
if (!lease || lease.status !== "failed" || lease.leasePolicy !== "ephemeral" || !lease.providerLeaseId) return undefined;
const environment = lease.environmentId ? await environmentsSvc.getById(lease.environmentId) : null;
return await getDriver(getLeaseDriverKey(lease, environment))?.inspectLeaseResource?.({ environment, lease });
// Legacy Cloud leases can lack a credential version that permits a
// provider-local observation. Cloud's manager verifies the historical
// organization and signs only exact absence; no replacement key or
// cleanup label is treated as proof. Explicit provider answers win.
return await getDriver(getLeaseDriverKey(lease, environment))?.inspectLeaseResource?.({ environment, lease })
?? await inspectCloudSandboxCleanupAbsence(lease);
},
async retryPendingSandboxTeardown(input: {
server/src/services/execution-control-notifications.ts
+9 -1
View File
@@ -1,11 +1,17 @@
import { notifyStatusCardIssueChange } from "./status-card-notifications.js";
import { notifyDeferredExecutionEligibility } from "./deferred-execution-notifications.js";
import { notifyRunDispatch } from "./run-dispatch-notifications.js";
import type { Db } from "@paperclipai/db";
import { DELIVERY_QUEUES, notifyDeliveryWork, type DeliveryQueue } from "./delivery-work-notifications.js";
export type ExecutionWorkScope = { companyId: string; agentId?: string; runId?: string; issueId?: string };
export async function notifyExecutionRunWork(transaction: object, scope?: ExecutionWorkScope) {
if (scope) await notifyStatusCardIssueChange(transaction, scope.companyId);
if (scope) {
await notifyStatusCardIssueChange(transaction, scope.companyId);
await notifyRunDispatch(transaction, scope);
await notifyDeferredExecutionEligibility(transaction, scope);
}
for (const queue of [DELIVERY_QUEUES.executionStatus, DELIVERY_QUEUES.executionFinalization,
DELIVERY_QUEUES.executionReplacement, DELIVERY_QUEUES.executionDisposition, DELIVERY_QUEUES.issueMonitors]) {
await notifyDeliveryWork(transaction, queue);
@@ -20,6 +26,8 @@ export function writeExecutionControl<T>(
): Promise<T> {
return db.transaction(async transaction => {
if (scope) await notifyStatusCardIssueChange(transaction, scope.companyId);
await notifyRunDispatch(transaction, scope);
if (scope) await notifyDeferredExecutionEligibility(transaction, scope);
await notifyDeliveryWork(transaction, DELIVERY_QUEUES.issueMonitors);
for (const topic of typeof queue === "string" ? [queue] : queue) await notifyDeliveryWork(transaction, topic);
return write(transaction as unknown as Db);
server/src/services/external-conversation-settlement-notifications.ts
+3 -1
View File
@@ -1,7 +1,9 @@
import { notifyIssueRecovery } from "./issue-recovery-notifications.js";
import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js";
import type { ExternalConversationSettlementScope } from "./slack-conversation-lifecycle.js";
export function notifyExternalConversationSettlement(tx: object, scope?: ExternalConversationSettlementScope) {
export async function notifyExternalConversationSettlement(tx: object, scope?: ExternalConversationSettlementScope) {
await notifyIssueRecovery(tx, scope);
return notifyDeliveryWork(tx, DELIVERY_QUEUES.externalConversationSettlement,
scope ? JSON.stringify({ companyId: scope.companyId, issueId: scope.issueId }) : "*");
}
server/src/services/heartbeat.ts
+43 -55
View File
@@ -1,3 +1,7 @@
import { registerDeferredReceiptWork } from "./heartbeat/deferred-receipt-work.js";
import { reconcileTaskChatRetryReceipts } from "./task-retry.js";
import { registerIssueRecoveryWork } from "./recovery/work.js";
import { registerRunDispatchWork } from "./heartbeat/dispatch-work.js";
import { externalConversationSettlementWork } from "./external-conversation-settlement-work.js";
import { nativeRecoveryNoticeWork } from "./native-runtime/native-recovery-notice-work.js";
import { nativeRunProjectionWork } from "./native-runtime/native-run-projection-work.js";
@@ -999,6 +1003,7 @@ export function heartbeatService(
finalizeAiConnectionBusyDeferral,
finalizeWorkspaceBusyDeferral,
promoteDueScheduledRetries,
nextScheduledRetryAt,
retryScheduledRetryNow,
scheduleBoundedRetry,
} = createHeartbeatRetries(db, {
@@ -1333,8 +1338,8 @@ export function heartbeatService(
return heartbeatQueue.dispatchPendingNativeStatusWakeups(...args);
}
function resumeQueuedRunsAfterDrain() {
const promise = heartbeatQueue.resumeQueuedRunsAfterDrain();
function resumeQueuedRunsAfterDrain(...args: Parameters<typeof heartbeatQueue.resumeQueuedRunsAfterDrain>) {
const promise = heartbeatQueue.resumeQueuedRunsAfterDrain(...args);
activeWakeupPromises.add(promise);
void promise.then(
() => { activeWakeupPromises.delete(promise); },
@@ -1609,15 +1614,15 @@ export function heartbeatService(
void reportRunFailure(db, updated, failureReport);
}
async function reconcileStrandedAssignedIssues(opts?: { skipExternalConversationSettlement?: boolean }) {
async function reconcileStrandedAssignedIssues(opts?: Parameters<typeof recovery.reconcileStrandedAssignedIssues>[0]) {
return recovery.reconcileStrandedAssignedIssues({
skipExternalConversationSettlement: opts?.skipExternalConversationSettlement,
...opts,
issueCreatedAtGte: await getWorktreeExecutionCutoff(),
});
}
async function sweepStaleIssueLocks() {
return recovery.sweepStaleIssueLocks();
async function sweepStaleIssueLocks(opts?: Parameters<typeof recovery.sweepStaleIssueLocks>[0]) {
return recovery.sweepStaleIssueLocks(opts);
}
function issueIdFromRunContext(contextSnapshot: unknown) {
@@ -1647,10 +1652,7 @@ export function heartbeatService(
});
}
async function reconcileTaskWatchdogs(opts?: {
companyId?: string | null;
runId?: string | null;
}) {
async function reconcileTaskWatchdogs(opts?: Parameters<typeof taskWatchdogs.reconcileTaskWatchdogs>[0]) {
return taskWatchdogs.reconcileTaskWatchdogs({
...opts,
issueCreatedAtGte: await getWorktreeExecutionCutoff(),
@@ -1675,10 +1677,7 @@ export function heartbeatService(
return recovery.buildRunOutputSilence(run, now);
}
async function reconcileResolvedDependencyWakes(opts?: {
runId?: string | null;
companyId?: string | null;
}) {
async function reconcileResolvedDependencyWakes(opts?: Parameters<typeof recovery.reconcileResolvedDependencyWakeBackstop>[0]) {
return recovery.reconcileResolvedDependencyWakeBackstop(opts);
}
@@ -2007,8 +2006,11 @@ export function heartbeatService(
const dispatchIssueId = readNonEmptyString(parseObject(run.contextSnapshot).issueId);
const resumingAdmittedConversationTurn = !!runOptions.nativeLeaseOwner
&& typeof run.contextSnapshot?.conversationSessionGeneration === "number";
if (dispatchIssueId && isConversation(await getIssueExecutionContext(run.companyId, dispatchIssueId))
&& !resumingAdmittedConversationTurn && !(await instanceSettings.getExperimental()).enableAgentChat) {
const conversationIdentity = dispatchIssueId && !resumingAdmittedConversationTurn
? await db.select({ conversationAgentId: issues.conversationAgentId, conversationUserId: issues.conversationUserId })
.from(issues).where(and(eq(issues.companyId, run.companyId), eq(issues.id, dispatchIssueId))).then(rows => rows[0] ?? null)
: null;
if (isConversation(conversationIdentity) && !(await instanceSettings.getExperimental()).enableAgentChat) {
await setRunStatus(run.id, "cancelled", { finishedAt: new Date(), error: "Agent Chat is disabled", errorCode: "agent_chat_disabled" });
await setWakeupStatus(run.wakeupRequestId, "cancelled", { finishedAt: new Date() });
await releaseIssueExecutionAndPromote((await getRun(run.id))!, { suppressImmediateRecovery: true });
@@ -2204,44 +2206,6 @@ export function heartbeatService(
context[PAPERCLIP_EXTERNAL_CHAT_EXECUTION_BOUND_KEY] = true;
}
const wakeCommentId = deriveCommentId(context, null);
const wakeCommentContext =
issueContext && wakeCommentId
? await db
.select({
id: issueComments.id,
body: issueComments.body,
authorType: issueComments.authorType,
authorAgentId: issueComments.authorAgentId,
authorUserId: issueComments.authorUserId,
presentation: issueComments.presentation,
metadata: issueComments.metadata,
deletedAt: issueComments.deletedAt,
deletedByType: issueComments.deletedByType,
deletedByAgentId: issueComments.deletedByAgentId,
deletedByUserId: issueComments.deletedByUserId,
deletedByRunId: issueComments.deletedByRunId,
sourceTrust: issueComments.sourceTrust,
})
.from(issueComments)
.where(
and(
eq(issueComments.id, wakeCommentId),
eq(issueComments.issueId, issueContext.id),
eq(issueComments.companyId, agent.companyId),
),
)
.then((rows) => {
const row = rows[0] ?? null;
return row?.deletedAt
? {
...row,
body: "",
presentation: null,
metadata: null,
}
: row;
})
: null;
let issueAssigneeOverrides =
issueContext && issueContext.assigneeAgentId === agent.id
? parseIssueAssigneeAdapterOverrides(
@@ -2610,7 +2574,7 @@ export function heartbeatService(
continuationSummary, executionContinuation, paperclipWakePayload, safeWakeComments, safeWakeCommentContext, pinnedSkillTest,
} = await timePreparation("task_context", () => prepareHeartbeatTaskContext(db, {
run, agent,
task: { issueRef, issueContext, wakeCommentContext, context, taskSession },
task: { issueRef, issueContext, context, taskSession },
policy: { trustPreset, experimentalInstanceSettings, recovering: Boolean(runOptions.nativeRestartRecovery || runOptions.nativeLeaseOwner) },
services: { issuesSvc, getPinnedSkillTestContext },
}));
@@ -4246,6 +4210,30 @@ export function heartbeatService(
computeTaskDrain,
applyTaskDrain,
registerDeferredReceiptWork: (coordinator: Parameters<typeof registerDeferredReceiptWork>[1], canRun: () => boolean) =>
registerDeferredReceiptWork(db, coordinator, {
resume: scope => heartbeatQueue.resumeDeferredExecutionReceipts(scope),
settleTaskChat: scope => reconcileTaskChatRetryReceipts(db, { ...scope, settleOnly: true }),
dispatchNative: dispatchPendingNativeStatusWakeups,
schedulingSuppressed: async () => (await getSchedulingSuppression(true)).suppressed,
}, () => canRun() && readTaskDrain(new Date()) === null),
registerIssueStateRecoveryWork: (coordinator: Parameters<typeof registerIssueRecoveryWork>[1], canRun: () => boolean) =>
registerIssueRecoveryWork(db, coordinator, {
stranded: scope => reconcileStrandedAssignedIssues({ ...scope, skipExternalConversationSettlement: true, skipRelatedRecovery: true }),
orphanBlockers: recovery.reconcileUnassignedBlockingIssues,
recoveryActions: recovery.reconcileActiveRecoveryActions,
dependencies: reconcileResolvedDependencyWakes,
locks: sweepStaleIssueLocks,
watchdogs: reconcileTaskWatchdogs,
schedulingSuppressed: async () => (await getSchedulingSuppression(true)).suppressed,
}, () => canRun() && readTaskDrain(new Date()) === null),
registerRunDispatchWork: (coordinator: Parameters<typeof registerRunDispatchWork>[0], canRun: () => boolean) =>
registerRunDispatchWork(coordinator, {
promoteDueScheduledRetries,
nextScheduledRetryAt,
resumeQueuedRunsAfterDrain,
schedulingSuppressed: async () => (await getSchedulingSuppression(true)).suppressed,
}, () => canRun() && readTaskDrain(new Date()) === null),
promoteDueScheduledRetries,
retryScheduledRetryNow,
server/src/services/heartbeat/deferred-receipt-work.ts
+118
View File
@@ -0,0 +1,118 @@
import { and, asc, eq, gt, inArray, isNull, or, sql } from "drizzle-orm";
import { agentWakeupRequests, type Db } from "@paperclipai/db";
import type { createDeliveryWorkCoordinator } from "../delivery-work-coordinator.js";
import { DELIVERY_QUEUES } from "../delivery-work-notifications.js";
import type { DeferredExecutionScope, DeferredReceiptRecoveryOptions } from "../deferred-execution-notifications.js";
import { registerScopedDeadlineWork } from "../scoped-deadline-work.js";
type Entry = { id: string; companyId: string; dueAt: number };
type NativeIntentOptions = DeferredExecutionScope & {
onDeferredDeadline(at: number): void;
onError(error: unknown): void;
};
export function registerDeferredReceiptWork(
db: Db,
coordinator: ReturnType<typeof createDeliveryWorkCoordinator>,
owner: {
resume(scope: DeferredReceiptRecoveryOptions): Promise<unknown>;
settleTaskChat(scope: DeferredExecutionScope): Promise<unknown>;
dispatchNative(scope: NativeIntentOptions): Promise<unknown>;
schedulingSuppressed(): Promise<boolean>;
},
canRun: () => boolean,
) {
function register(kind: "deferred" | "task_chat" | "native") {
const cursors = new Map<string, string>();
let enabled = false;
const queue = kind === "deferred" ? DELIVERY_QUEUES.deferredExecutionReceipts
: kind === "task_chat" ? DELIVERY_QUEUES.taskRetryReceiptSettlement : DELIVERY_QUEUES.nativeStatusWakeIntents;
return registerScopedDeadlineWork<Entry, DeferredExecutionScope>(coordinator, {
queue,
reconstructionIntervalMs: 30_000,
beforePass: async () => { enabled = !(await owner.schedulingSuppressed()); },
decodeScope: hint => hint === "*" ? undefined : JSON.parse(hint),
matches: (entry, scope) => !scope || entry.companyId === scope.companyId && (!scope.requestId || entry.id === scope.requestId),
load: async scope => {
const key = scope ? JSON.stringify({ companyId: scope.companyId, requestId: scope.requestId }) : "*";
if (!enabled) { cursors.delete(key); return []; }
const cursor = cursors.get(key);
const rows = await db.select({ id: agentWakeupRequests.id, companyId: agentWakeupRequests.companyId,
status: agentWakeupRequests.status, updatedAt: agentWakeupRequests.updatedAt,
claimedAt: agentWakeupRequests.claimedAt, error: agentWakeupRequests.error,
delayed: sql<boolean>`${agentWakeupRequests.payload}->'queuedCommentInterrupt' is not null
or ${agentWakeupRequests.payload}->'executionWait' is not null`,
interruptedLegacy: sql<boolean>`exists (select 1 from heartbeat_runs r
where r.company_id = ${agentWakeupRequests.companyId} and r.agent_id = ${agentWakeupRequests.agentId}
and r.runtime_mode = 'legacy' and r.status = 'cancelled'
and r.result_json->>'queuedCommentInterruptQueueId' = ${agentWakeupRequests.id}::text)`,
}).from(agentWakeupRequests).where(and(
scope ? eq(agentWakeupRequests.companyId, scope.companyId) : undefined,
scope?.requestId ? eq(agentWakeupRequests.id, scope.requestId) : undefined,
cursor ? gt(agentWakeupRequests.id, cursor) : undefined,
kind === "native" ? and(
eq(agentWakeupRequests.requestedByActorType, "system"),
eq(agentWakeupRequests.requestedByActorId, "native-status-committer"),
inArray(agentWakeupRequests.status, ["queued", "claimed"]), isNull(agentWakeupRequests.runId),
) : and(eq(agentWakeupRequests.status, "deferred_issue_execution"),
kind === "task_chat" ? and(eq(agentWakeupRequests.reason, "task_retry"),
sql`${agentWakeupRequests.payload}#>>'{taskRetry,chatActionId}' is not null`) : or(
eq(agentWakeupRequests.reason, "task_retry"),
sql`${agentWakeupRequests.payload}->'queuedCommentInterrupt' is not null`,
sql`${agentWakeupRequests.payload}->'executionWait' is not null`,
sql`jsonb_typeof(${agentWakeupRequests.payload}#>'{_paperclipWakeContext,wakeCommentIds}') = 'array'
and ${agentWakeupRequests.payload}#>'{_paperclipWakeContext,wakeCommentIds}' <> '[]'::jsonb`,
sql`${agentWakeupRequests.payload}->>'mutation' = 'interaction'`,
sql`exists (select 1 from heartbeat_runs r where r.company_id = ${agentWakeupRequests.companyId}
and r.runtime_mode = 'legacy' and r.status = 'cancelled'
and r.result_json->>'queuedCommentInterruptQueueId' = ${agentWakeupRequests.id}::text)`,
)),
)).orderBy(asc(agentWakeupRequests.id)).limit(100);
const complete = rows.length < 100;
if (complete) cursors.delete(key); else cursors.set(key, rows.at(-1)!.id);
return { rows: rows.map(row => ({ id: row.id, companyId: row.companyId,
dueAt: kind === "native" && row.status === "claimed" && row.claimedAt ? row.claimedAt.getTime() + 60_000
: kind === "native" && row.error ? row.updatedAt.getTime() + 30_000
: kind === "deferred" && row.delayed && !row.interruptedLegacy ? row.updatedAt.getTime() + 30_000
: Date.now(),
})), complete };
},
check: async entry => {
if (!enabled) return null;
if (kind !== "task_chat") {
const [current] = await db.select({ status: agentWakeupRequests.status, payload: agentWakeupRequests.payload,
updatedAt: agentWakeupRequests.updatedAt, claimedAt: agentWakeupRequests.claimedAt, error: agentWakeupRequests.error,
interruptedLegacy: sql<boolean>`exists (select 1 from heartbeat_runs r
where r.company_id = ${agentWakeupRequests.companyId} and r.agent_id = ${agentWakeupRequests.agentId}
and r.runtime_mode = 'legacy' and r.status = 'cancelled'
and r.result_json->>'queuedCommentInterruptQueueId' = ${agentWakeupRequests.id}::text)`,
}).from(agentWakeupRequests).where(and(eq(agentWakeupRequests.companyId, entry.companyId),
eq(agentWakeupRequests.id, entry.id)));
if (!current) return null;
const due = kind === "native" && current.status === "claimed" && current.claimedAt
? current.claimedAt.getTime() + 60_000
: kind === "native" && current.status === "queued" && current.error ? current.updatedAt.getTime() + 30_000
: kind === "deferred" && current.status === "deferred_issue_execution" && !current.interruptedLegacy
&& (current.payload?.queuedCommentInterrupt || current.payload?.executionWait)
? current.updatedAt.getTime() + 30_000 : null;
if (due !== null && due > Date.now()) return { ...entry, dueAt: due };
}
const errors: unknown[] = [];
let next: number | null = null;
const scope = { companyId: entry.companyId, requestId: entry.id };
if (kind === "native") await owner.dispatchNative({ ...scope,
onDeferredDeadline: at => { next = next === null ? at : Math.min(next, at); },
onError: error => { errors.push(error); },
});
else if (kind === "task_chat") await owner.settleTaskChat(scope);
else await owner.resume({ ...scope, onError: error => { errors.push(error); } });
if (errors.length) throw new AggregateError(errors, `Deferred receipt ${entry.id} could not settle`);
return next === null ? null : { ...entry, dueAt: next };
},
}, canRun);
}
const workers = [register("deferred"), register("task_chat"), register("native")];
return { ready: Promise.all(workers.map(worker => worker.ready)),
wake() { for (const worker of workers) worker.invalidate(); },
async stop() { await Promise.all(workers.map(worker => worker.stop())); } };
}
server/src/services/heartbeat/dispatch-work.ts
+58
View File
@@ -0,0 +1,58 @@
import type { RunDispatchScope } from "../run-dispatch-notifications.js";
import type { createDeliveryWorkCoordinator } from "../delivery-work-coordinator.js";
import { registerDeadlineWork } from "../deadline-work.js";
import { DELIVERY_QUEUES } from "../delivery-work-notifications.js";
export function registerRunDispatchWork(
coordinator: ReturnType<typeof createDeliveryWorkCoordinator>,
owner: {
promoteDueScheduledRetries(): Promise<unknown>;
nextScheduledRetryAt(): Promise<number | null>;
resumeQueuedRunsAfterDrain(scope?: RunDispatchScope): Promise<unknown>;
schedulingSuppressed(): Promise<boolean>;
},
canRun: () => boolean,
) {
let suppressed = false;
const retries = registerDeadlineWork(coordinator, DELIVERY_QUEUES.scheduledRetryDispatch, {
retryMs: 30_000,
continuationMs: 1_000,
notificationCoalesceMs: 1_000,
canRun,
run: async signal => {
suppressed = await owner.schedulingSuppressed();
if (!signal.aborted && !suppressed && canRun()) await owner.promoteDueScheduledRetries();
},
next: () => suppressed ? Promise.resolve(null) : owner.nextScheduledRetryAt(),
});
const scopes = new Set<string>(["*"]);
const queue = registerDeadlineWork(coordinator, DELIVERY_QUEUES.queuedRunDispatch, {
retryMs: 30_000,
notificationCoalesceMs: 1_000,
canRun,
onSettled: hints => { for (const hint of hints ?? ["*"]) scopes.add(hint); },
run: async signal => {
const requested = scopes.has("*") ? ["*"] : [...scopes];
if (requested[0] === "*") scopes.clear();
else for (const hint of requested) scopes.delete(hint);
for (let index = 0; index < requested.length; index++) {
if (signal.aborted || !canRun()) {
for (const hint of requested.slice(index)) scopes.add(hint);
return;
}
try {
await owner.resumeQueuedRunsAfterDrain(requested[index] === "*" ? undefined : JSON.parse(requested[index]) as RunDispatchScope);
} catch (error) {
for (const hint of requested.slice(index)) scopes.add(hint);
throw error;
}
}
},
next: async () => null,
});
return {
ready: Promise.all([retries.ready, queue.ready]),
wake() { scopes.add("*"); retries.wake(); queue.wake(); },
async stop() { await Promise.all([retries.stop(), queue.stop()]); },
};
}
server/src/services/heartbeat/execution-configuration.ts
+29 -18
View File
@@ -139,28 +139,27 @@ export async function prepareHeartbeatExecutionConfiguration(db: Db, input: Hear
stripWorkspaceRuntimeFromExecutionRunConfig(mergedConfig);
const skillActor: AuthorizationActor = { type: "agent", agentId: agent.id, companyId: agent.companyId,
runId: run.id, source: "agent_jwt", onBehalfOfUserId: responsibleUserId === "local-board" ? null : responsibleUserId };
const runScopedMentionedSkillKeys =
preconfiguredHome || pinnedSkillTest ? [] : await time("configuration_mentions", () => resolveRunScopedMentionedSkillKeys({
const [mentionedSkills, githubIdentity] = await Promise.allSettled([
preconfiguredHome || pinnedSkillTest ? Promise.resolve([]) : time("configuration_mentions", () => resolveRunScopedMentionedSkillKeys({
db,
companyId: agent.companyId,
issueId,
commentIds: extractWakeCommentIds(context),
actor: skillActor,
}));
})),
time("configuration_github_identity", () => resolveManagedGitHubIdentitySelection(db, agent.companyId, {
agentId: agent.id, responsibleUserId, allowStandingDelegation: false,
})),
]);
if (mentionedSkills.status === "rejected") throw mentionedSkills.reason;
if (githubIdentity.status === "rejected") throw githubIdentity.reason;
const runScopedMentionedSkillKeys = mentionedSkills.value;
const runScopedSkillKeys =
!pinnedSkillTest && acceptedPlanContinuationWake &&
!acceptedPlanWakeRoutingDecision?.suppressAcceptedContinuation
? [...runScopedMentionedSkillKeys, ACCEPTED_PLAN_CONVERSION_SKILL_KEY]
: runScopedMentionedSkillKeys;
const githubSelection = await time("configuration_github_identity", () => resolveManagedGitHubIdentitySelection(
db,
agent.companyId,
{
agentId: agent.id,
responsibleUserId,
allowStandingDelegation: false,
},
));
const githubSelection = githubIdentity.value;
const useHostGitHub =
!githubSelection.configured &&
trustPreset.kind === "standard" &&
@@ -274,6 +273,7 @@ export async function prepareHeartbeatExecutionConfiguration(db: Db, input: Hear
);
const nativeRunnerPreparationSpans: NativeRunHistoricalSpan[] = [];
const skillsPrepareStartedAtMs = Date.now();
let skillsPrepareEndedAtMs = skillsPrepareStartedAtMs;
let skillDeliveryConfig = effectiveResolvedConfig;
const prepareSkills = async () => {
@@ -346,24 +346,35 @@ export async function prepareHeartbeatExecutionConfiguration(db: Db, input: Hear
throw error;
}
};
const runtimeSkillEntries = preconfiguredHome ? [] : await time("configuration_skills", prepareSkills);
const skillsPreparation = (preconfiguredHome ? Promise.resolve([]) : time("configuration_skills", prepareSkills))
.finally(() => { skillsPrepareEndedAtMs = Date.now(); });
const assignmentsPreparation = preconfiguredHome ? Promise.resolve([]) : time("configuration_connector_assignments", () => resolveConnectorAssignments(db, { companyId: agent.companyId, agentId: agent.id, runId: run.id, issueId: typeof context.issueId === "string" ? context.issueId : undefined }));
const [preparedSkills, preparedAssignments, preparedInstructions] = await Promise.allSettled([
skillsPreparation,
assignmentsPreparation,
assignmentsPreparation.then(() => time("configuration_connection_instructions", () => resolveAssignedConnectionInstructionsForRun(db, { companyId: agent.companyId, agentId: agent.id, runId: run.id }))),
]);
if (preparedSkills.status === "rejected") throw preparedSkills.reason;
if (preparedAssignments.status === "rejected") throw preparedAssignments.reason;
if (preparedInstructions.status === "rejected") throw preparedInstructions.reason;
const runtimeSkillEntries = preparedSkills.value;
nativeRunnerPreparationSpans.push({
name: "skills.prepare",
parentName: "task.prepare",
startedAtMs: skillsPrepareStartedAtMs,
endedAtMs: Date.now(),
endedAtMs: skillsPrepareEndedAtMs,
});
const connectorAssignments = preconfiguredHome ? [] : await time("configuration_connector_assignments", () => resolveConnectorAssignments(db, { companyId: agent.companyId, agentId: agent.id, runId: run.id, issueId: typeof context.issueId === "string" ? context.issueId : undefined }));
const connectorAssignments = preparedAssignments.value;
const connectorSkillConfig = await time("configuration_connector_skills", () => applyConnectorSkills(skillDeliveryConfig, runtimeSkillEntries, connectorAssignments));
// Both CLI adapters and native context materialization use the same resolved set.
runtimeSkillEntries.splice(0, runtimeSkillEntries.length, ...connectorSkillConfig.paperclipRuntimeSkills);
const connectorDelivery = await time("configuration_connector_delivery", () => prepareConnectorSkillDelivery(connectorSkillConfig, agent.adapterType));
// Always replace this runtime-only field; caller wake data cannot supply skills.
context.paperclipWake = { ...parseObject(context.paperclipWake), connectorSkillInstructions: connectorDelivery.instructions };
let runtimeConfig = await time("configuration_connection_instructions", () => prepareConnectionInstructionDelivery({
resolve: () => resolveAssignedConnectionInstructionsForRun(db, { companyId: agent.companyId, agentId: agent.id, runId: run.id }),
let runtimeConfig = await prepareConnectionInstructionDelivery({
resolve: async () => preparedInstructions.value,
context, config: connectorDelivery.config, native: agent.adapterType === "paperclip_runner",
}));
});
const agentIdentity = supportsManagedAgentIdentity(agent.adapterType, agent.adapterConfig, run.runtimeMode === "native" ? run.driverKind : undefined)
? await time("configuration_agent_identity", () => agentIdentityService(db).ensureAgentIdentity(agent.companyId, agent.id))
: undefined;
server/src/services/heartbeat/queue.ts
+44 -22
View File
@@ -1,4 +1,8 @@
import { notifyStatusCardIssueChange } from "../status-card-notifications.js";
import { writeDeferredExecutionWait } from "../deferred-execution-wait.js";
import { notifyDeferredExecutionWork } from "../deferred-execution-notifications.js";
import type { DeferredReceiptRecoveryOptions } from "../deferred-execution-notifications.js";
import { notifyRunDispatch } from "../run-dispatch-notifications.js";
import { DELIVERY_QUEUES, notifyDeliveryWork } from "../delivery-work-notifications.js";
import { notifyExternalAgentSettlement } from "../../modules/external-agents/index.js";
import { notifyNativeStatusReassessment } from "../native-status-reassessment-notifications.js";
@@ -222,7 +226,7 @@ export interface HeartbeatQueueDependencies extends Pick<HeartbeatRetryDependenc
publishRunLifecyclePluginEvent: (run: HeartbeatRun) => void;
cancelRunInternal: (runId: string, reason?: string, options?: { errorCode?: string }) => Promise<unknown>;
cancelActiveForAgentInternal: (agentId: string, reason: string) => Promise<unknown>;
resumeExecutionWaitComments: () => Promise<unknown>;
resumeExecutionWaitComments: (scope?: DeferredReceiptRecoveryOptions) => Promise<unknown>;
resumeQueuedCommentInterrupt: (companyId: string, queueId: string) => Promise<unknown>;
resumeSavedLegacyComments: (companyId: string, queueId: string) => Promise<unknown>;
formatIssueIdentifierLink: (identifier: string | null, fallback: string) => string;
@@ -1852,13 +1856,15 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
return cancelled;
}
async function resumeQueuedRuns() {
async function resumeDeferredExecutionReceipts(scope: DeferredReceiptRecoveryOptions = {}) {
if ((await getSchedulingSuppression()).suppressed) return;
await resumeExecutionWaitComments();
await resumeExecutionWaitComments(scope);
const cutoff = await getWorktreeExecutionCutoff();
const pendingInterrupts = await db.select({ id: agentWakeupRequests.id, companyId: agentWakeupRequests.companyId })
.from(agentWakeupRequests).innerJoin(companies, eq(companies.id, agentWakeupRequests.companyId))
.where(and(eq(agentWakeupRequests.status, "deferred_issue_execution"),
.where(and(scope.companyId ? eq(agentWakeupRequests.companyId, scope.companyId) : undefined,
scope.requestId ? eq(agentWakeupRequests.id, scope.requestId) : undefined,
eq(agentWakeupRequests.status, "deferred_issue_execution"),
eq(companies.status, "active"),
sql`${agentWakeupRequests.payload}->'queuedCommentInterrupt' is not null`,
lte(agentWakeupRequests.updatedAt, new Date(Date.now() - 30_000)),
@@ -1869,6 +1875,7 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
eq(agentWakeupRequests.id, wake.id), eq(agentWakeupRequests.status, "deferred_issue_execution"),
));
await resumeQueuedCommentInterrupt(wake.companyId, wake.id).catch(err => {
scope.onError?.(err);
logger.warn({ err, queueId: wake.id }, "failed to resume interrupted comment queue");
});
}
@@ -1882,7 +1889,9 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
or(eq(issues.assigneeAgentId, agentWakeupRequests.agentId),
sql`${agentWakeupRequests.payload} #>> '{_paperclipWakeContext,wakeReason}' = 'issue_comment_mentioned'`)))
.innerJoin(companies, and(eq(companies.id, issues.companyId), eq(companies.status, "active")))
.where(and(eq(agentWakeupRequests.status, "deferred_issue_execution"),
.where(and(scope.companyId ? eq(agentWakeupRequests.companyId, scope.companyId) : undefined,
scope.requestId ? eq(agentWakeupRequests.id, scope.requestId) : undefined,
eq(agentWakeupRequests.status, "deferred_issue_execution"),
isNull(issues.executionRunId),
or(and(
sql`jsonb_typeof(${agentWakeupRequests.payload} #> '{_paperclipWakeContext,wakeCommentIds}') = 'array'`,
@@ -1921,7 +1930,8 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
if (latest.runtimeMode !== "legacy" && latest.runtimeMode !== "native") continue;
if (wake.idempotencyKey?.startsWith("chat-inbound:")) {
await resumeDeferredChatWake(db, { companyId: wake.companyId, agentId: wake.agentId,
issueId: String(wake.payload?.issueId), requestId: wake.id }).catch(() => {
issueId: String(wake.payload?.issueId), requestId: wake.id }).catch(err => {
scope.onError?.(err);
logger.warn({ requestId: wake.id }, "saved chat source could not resume after recovery");
});
continue;
@@ -1933,6 +1943,7 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
latest.errorCode === "execution_reconciliation_required";
if ((latest.status !== "cancelled" || cancelledAdmission) && await getExecutionBlocker(db, wake.companyId, String(wake.payload?.issueId))) {
await resumeSavedLegacyComments(wake.companyId, wake.id).catch(err => {
scope.onError?.(err);
logger.warn({ err, queueId: wake.id }, "failed to deliver saved legacy comment after recovery stopped");
});
}
@@ -1940,6 +1951,7 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
await releaseIssueExecutionAndPromote(latest, { suppressImmediateRecovery: true, resumeStrandedQueue: true }).then(() => {
if (latest.runtimeMode === "native") releasedNativeSources.add(releaseSource);
}).catch(err => {
scope.onError?.(err);
logger.warn({ err, queueId: wake.id }, "failed to promote stranded comments");
});
}
@@ -1957,6 +1969,8 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
))
.innerJoin(companies, eq(companies.id, heartbeatRuns.companyId))
.where(and(
scope.companyId ? eq(agentWakeupRequests.companyId, scope.companyId) : undefined,
scope.requestId ? eq(agentWakeupRequests.id, scope.requestId) : undefined,
eq(agentWakeupRequests.status, "deferred_issue_execution"),
eq(heartbeatRuns.status, "cancelled"),
eq(heartbeatRuns.runtimeMode, "legacy"),
@@ -1965,16 +1979,21 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
));
for (const run of interruptedQueues) {
await releaseIssueExecutionAndPromote(run, { suppressImmediateRecovery: true }).catch((err) => {
scope.onError?.(err);
logger.error({ err, runId: run.id }, "failed to retry interrupted comment queue");
});
}
}
async function resumeQueuedRuns() {
await resumeDeferredExecutionReceipts();
await resumeQueuedRunsAfterDrain();
}
// Release admission promptly without first scanning unrelated recovery work.
// Every run still passes the ordinary per-agent claim and dispatch gates.
async function resumeQueuedRunsAfterDrain() {
async function resumeQueuedRunsAfterDrain(scope?: { companyId: string }) {
if ((await getSchedulingSuppression()).suppressed) return;
const cutoff = await getWorktreeExecutionCutoff();
const queuedRuns = await db
@@ -1984,6 +2003,7 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
.where(
and(
eq(heartbeatRuns.status, "queued"),
scope ? eq(heartbeatRuns.companyId, scope.companyId) : undefined,
eq(companies.status, "active"),
cutoff ? gte(heartbeatRuns.createdAt, cutoff) : undefined,
),
@@ -2506,17 +2526,11 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
// its one durable receipt ID on a skipped wake.
if (caseRequest) return { created: false };
if (executionWaitRequestId) {
await db.update(agentWakeupRequests).set({
payload: sql`jsonb_set(coalesce(${agentWakeupRequests.payload}, '{}'::jsonb), '{executionWait}',
coalesce(${agentWakeupRequests.payload}->'executionWait', '{}'::jsonb) || ${JSON.stringify({
reason: skipReason, message: patch.error ?? (skipReason === "issue_tree_hold_active"
? "This task is paused. Resume it to send your saved message."
: "Waiting for task execution to be enabled. Your message is saved."),
})}::jsonb)`,
updatedAt: new Date(),
}).where(and(eq(agentWakeupRequests.id, executionWaitRequestId),
eq(agentWakeupRequests.companyId, agent.companyId), eq(agentWakeupRequests.agentId, agentId),
eq(agentWakeupRequests.status, "deferred_issue_execution")));
await writeDeferredExecutionWait(db, { companyId: agent.companyId, requestId: executionWaitRequestId }, {
reason: skipReason, message: patch.error ?? (skipReason === "issue_tree_hold_active"
? "This task is paused. Resume it to send your saved message."
: "Waiting for task execution to be enabled. Your message is saved."),
});
return { created: false };
}
const request = {
@@ -3404,15 +3418,14 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
opts.issueStateGuard?.onExecutionDeferred?.(executionBlocker.cause);
const condition = { recoveryActionId: executionBlocker.recoveryActionId, ...continuationWait };
if (executionWaitRequestId) {
await tx.update(agentWakeupRequests).set({
payload: sql`jsonb_set(coalesce(${agentWakeupRequests.payload}, '{}'::jsonb), '{executionWait}', ${JSON.stringify(condition)}::jsonb)`,
updatedAt: new Date(),
}).where(eq(agentWakeupRequests.id, executionWaitRequestId));
await writeDeferredExecutionWait(tx as unknown as Db, { companyId: agent.companyId,
requestId: executionWaitRequestId }, condition, { replace: true });
return { kind: "deferred" as const, admitted: true };
}
if (durableRequest || wakeCommentId ||
hasInteractionContinuationWakeContext(enrichedContextSnapshot) ||
readNonEmptyString(enrichedContextSnapshot.nativeStatusWakeIntentId)) {
await notifyDeferredExecutionWork(tx, { companyId: agent.companyId });
await tx.insert(agentWakeupRequests).values({
...durableReceiptFields,
companyId: agent.companyId, agentId, source, triggerDetail, reason,
@@ -4031,6 +4044,7 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
}));
}
if (followUp) {
if (!followUp.consumed) await notifyDeferredExecutionWork(tx, { companyId: agent.companyId });
const assignmentId = followUp.assignmentId;
const pending = followUp.consumed ? undefined : pendingFollowUps.find(wake =>
wake.payload?.[followUpPayloadKey] === assignmentId &&
@@ -4635,6 +4649,7 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
}
const queueOutcome = await db.transaction(async (tx) => {
await notifyRunDispatch(tx, { companyId: agent.companyId });
await tx.execute(
sql`select id from agents where id = ${agentId} and company_id = ${agent.companyId} for update`,
);
@@ -4768,6 +4783,9 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
companyId?: string;
limit?: number;
staleClaimMs?: number;
requestId?: string;
onDeferredDeadline?: (at: number) => void;
onError?: (error: unknown) => void;
} = {},
) {
const now = new Date();
@@ -4780,6 +4798,7 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
input.companyId
? eq(agentWakeupRequests.companyId, input.companyId)
: undefined,
input.requestId ? eq(agentWakeupRequests.id, input.requestId) : undefined,
eq(agentWakeupRequests.requestedByActorType, "system"),
eq(agentWakeupRequests.requestedByActorId, "native-status-committer"),
inArray(agentWakeupRequests.status, ["queued", "claimed"]),
@@ -4856,6 +4875,7 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
candidate.claimedAt &&
now.getTime() - candidate.claimedAt.getTime() < staleClaimMs
) {
input.onDeferredDeadline?.(candidate.claimedAt.getTime() + staleClaimMs);
deferred += 1;
continue;
}
@@ -5030,6 +5050,7 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
if (wakeRun) dispatched += 1;
else deferred += 1;
} catch (error) {
input.onError?.(error);
await db
.update(agentWakeupRequests)
.set({
@@ -5073,6 +5094,7 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
withChatControlRecoveryGate,
trackWakeup,
resumeQueuedRuns,
resumeDeferredExecutionReceipts,
resumeQueuedRunsAfterDrain,
parseHeartbeatPolicy,
nextTimerHeartbeatAt,
server/src/services/heartbeat/recovery.ts
+2 -2
View File
@@ -1979,7 +1979,7 @@ export function createHeartbeatRecovery(db: Db, dependencies: HeartbeatRecoveryD
let settledProcessesDiscovered = false;
async function reapOrphanedRuns(opts?: { staleThresholdMs?: number; skipScheduledRetryCancellation?: boolean; skipCapturedInstructionCommit?: boolean; skipInstructionDirectoryCleanup?: boolean; skipStoppedInstructionCollection?: boolean; skipCancelledWorkspacePreservation?: boolean; skipObsoletePolicyReviews?: boolean; skipAutomaticCompletionReviews?: boolean; skipUnsafeWorkspaceExports?: boolean; skipNativePendingEffects?: boolean; skipNativeResponsePresentation?: boolean; skipNativeReviewResponse?: boolean; skipNativeStatusReassessment?: boolean; skipRetainedNativeSessionCleanup?: boolean; skipNativeWorkspaceFinalization?: boolean; skipNativeResultCommit?: boolean; skipNativeRunProjection?: boolean; skipNativeRecoveryNotices?: boolean; skipExternalConversationSettlement?: boolean; skipExternalAgentSettlement?: boolean; skipNativeEnvironmentSettlement?: boolean }) {
async function reapOrphanedRuns(opts?: { staleThresholdMs?: number; skipNativeStatusWakeIntents?: boolean; skipScheduledRetryCancellation?: boolean; skipCapturedInstructionCommit?: boolean; skipInstructionDirectoryCleanup?: boolean; skipStoppedInstructionCollection?: boolean; skipCancelledWorkspacePreservation?: boolean; skipObsoletePolicyReviews?: boolean; skipAutomaticCompletionReviews?: boolean; skipUnsafeWorkspaceExports?: boolean; skipNativePendingEffects?: boolean; skipNativeResponsePresentation?: boolean; skipNativeReviewResponse?: boolean; skipNativeStatusReassessment?: boolean; skipRetainedNativeSessionCleanup?: boolean; skipNativeWorkspaceFinalization?: boolean; skipNativeResultCommit?: boolean; skipNativeRunProjection?: boolean; skipNativeRecoveryNotices?: boolean; skipExternalConversationSettlement?: boolean; skipExternalAgentSettlement?: boolean; skipNativeEnvironmentSettlement?: boolean }) {
if (!opts?.skipCancelledWorkspacePreservation) await preserveCancelledNativeWorkspaces(db);
const staleThresholdMs = opts?.staleThresholdMs ?? 0;
const now = new Date();
@@ -2023,7 +2023,7 @@ export function createHeartbeatRecovery(db: Db, dependencies: HeartbeatRecoveryD
});
if (!opts?.skipRetainedNativeSessionCleanup) scheduleRetainedNativeSessionCleanup();
await reconcileRetainedWorkspaces(db, environmentRuntime, resumeRemoteStopComments);
await dispatchPendingNativeStatusWakeups().catch((error) => {
if (!opts?.skipNativeStatusWakeIntents) await dispatchPendingNativeStatusWakeups().catch((error) => {
logger.warn(
{ err: error },
"failed to dispatch persisted native status wake intents before orphan reaping",
server/src/services/heartbeat/retries.test.ts
+7 -4
View File
@@ -35,6 +35,7 @@ function callbacks() {
getWorktreeExecutionCutoff: vi.fn(async (): Promise<Date | null> => null),
applyRunDispatchPostCommitEffects: vi.fn<(effects: PostCommitEffect[]) => void>(),
runDispatch: {
nextScheduledRetryAt: vi.fn(async () => null),
evaluateScheduledRetryGate: vi.fn<HeartbeatRetryDependencies["runDispatch"]["evaluateScheduledRetryGate"]>(async () => ({ allowed: true })),
promoteDueScheduledRetries: vi.fn<HeartbeatRetryDependencies["runDispatch"]["promoteDueScheduledRetries"]>(async () => ({ promoted: 0, runIds: [], postCommitEffects: [] })),
promoteScheduledRetry: vi.fn<HeartbeatRetryDependencies["runDispatch"]["promoteScheduledRetry"]>(async () => ({ outcome: "not_promoted" })),
@@ -183,7 +184,9 @@ describe("heartbeat retry module callbacks", () => {
});
});
const support = await getEmbeddedPostgresTestSupport();
import { resolveExternalTestDatabaseUrl } from "../../__tests__/helpers/external-test-database.js";
const external = resolveExternalTestDatabaseUrl("PAPERCLIP_HEARTBEAT_RETRIES_TEST_DATABASE_URL");
const support = external ? { supported: true, reason: undefined } : await getEmbeddedPostgresTestSupport();
const describePostgres = support.supported ? describe : describe.skip;
if (!support.supported) console.warn(`Skipping retry PostgreSQL tests: ${support.reason ?? "unsupported host"}`);
@@ -193,11 +196,11 @@ describePostgres("heartbeat retry module database wiring", () => {
const now = new Date("2026-10-08T12:00:00Z");
beforeAll(async () => {
database = await startEmbeddedPostgresTestDatabase("heartbeat-retries-");
db = createDb(database.connectionString);
if (!external) database = await startEmbeddedPostgresTestDatabase("heartbeat-retries-");
db = createDb(external ?? database.connectionString, { maxConnections: 6 });
}, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS);
afterAll(async () => { await database?.cleanup(); });
afterAll(async () => { await db?.$client.end({ timeout: 1 }); await database?.cleanup(); });
afterEach(async () => {
await db.delete(issues);
server/src/services/heartbeat/retries.ts
+9 -1
View File
@@ -1,4 +1,5 @@
import { notifyStatusCardIssueChange } from "../status-card-notifications.js";
import { notifyRunDispatch } from "../run-dispatch-notifications.js";
import { deriveTaskKeyWithHeartbeatFallback } from "./run-state.js";
import { infrastructureRetryDelayMs } from "../system-recovery-state.js";
import {
@@ -78,7 +79,7 @@ type PlanApprovalResumeFailure = {
export interface HeartbeatRetryDependencies {
hasCaseHandoffForRun: (run: Pick<HeartbeatRun, "id" | "companyId">) => Promise<boolean>;
runDispatch: Pick<ReturnType<typeof createRunDispatch>,
"evaluateScheduledRetryGate" | "promoteDueScheduledRetries" | "promoteScheduledRetry">;
"evaluateScheduledRetryGate" | "promoteDueScheduledRetries" | "promoteScheduledRetry" | "nextScheduledRetryAt">;
getAgent: (agentId: string) => Promise<typeof agents.$inferSelect | null | undefined>;
getRun: (
runId: string,
@@ -998,6 +999,7 @@ export function createHeartbeatRetries(db: Db, dependencies: HeartbeatRetryDepen
retryContextSnapshot.previousRunId = continuation.previousRunId;
}
await notifyRunDispatch(tx, { companyId: run.companyId });
const wakeupRequest = await tx
.insert(agentWakeupRequests)
.values({
@@ -1645,6 +1647,10 @@ export function createHeartbeatRetries(db: Db, dependencies: HeartbeatRetryDepen
});
}
async function nextScheduledRetryAt() {
return runDispatch.nextScheduledRetryAt({ cutoff: await getWorktreeExecutionCutoff() });
}
async function promoteDueScheduledRetries(now = new Date()) {
const cutoff = await getWorktreeExecutionCutoff();
const result = await runDispatch.promoteDueScheduledRetries({
@@ -1752,6 +1758,7 @@ export function createHeartbeatRetries(db: Db, dependencies: HeartbeatRetryDepen
};
const updated = await db.transaction(async (tx) => {
await notifyRunDispatch(tx, { companyId: scheduled.run.companyId });
const row = await tx
.update(heartbeatRuns)
.set({
@@ -1903,6 +1910,7 @@ export function createHeartbeatRetries(db: Db, dependencies: HeartbeatRetryDepen
finalizeAiConnectionBusyDeferral,
finalizeWorkspaceBusyDeferral,
promoteDueScheduledRetries,
nextScheduledRetryAt,
retryScheduledRetryNow,
scheduleBoundedRetry: async (
server/src/services/heartbeat/run-control.ts
+16 -11
View File
@@ -1,3 +1,5 @@
import { writeDeferredExecutionWait } from "../deferred-execution-wait.js";
import type { DeferredReceiptRecoveryOptions } from "../deferred-execution-notifications.js";
import { observeStoppedWorkspaceRecovery } from "../workspace-recovery.js";
import { notifyScheduledRetryCancellation } from "../scheduled-retry-cancellation-notifications.js";
import { notifyChatInboundWakeNotices } from "../chat-inbound-wake-notice-notifications.js";
@@ -616,12 +618,7 @@ export function createHeartbeatRunControl(db: Db, dependencies: HeartbeatRunCont
onBlocked: (reason, message) => { wait = { reason, message }; },
});
if (!admitted) {
await db.update(agentWakeupRequests).set({
payload: sql`jsonb_set(coalesce(${agentWakeupRequests.payload}, '{}'::jsonb), '{executionWait}',
coalesce(${agentWakeupRequests.payload}->'executionWait', '{}'::jsonb) || ${JSON.stringify(wait)}::jsonb)`,
updatedAt: new Date(),
}).where(and(eq(agentWakeupRequests.id, wake.id), eq(agentWakeupRequests.companyId, run.companyId),
eq(agentWakeupRequests.status, "deferred_issue_execution")));
await writeDeferredExecutionWait(db, { companyId: run.companyId, requestId: wake.id }, wait);
continue;
}
}
@@ -777,21 +774,25 @@ export function createHeartbeatRunControl(db: Db, dependencies: HeartbeatRunCont
idempotencyKey: `task-retry-dispatch:${wake.id}` }, wake.id);
}
async function resumeExecutionWaitComments() {
async function resumeExecutionWaitComments(scope: DeferredReceiptRecoveryOptions = {}) {
if ((await getSchedulingSuppression()).suppressed) return;
await reconcileTaskChatRetryReceipts(db);
await reconcileTaskChatRetryReceipts(db, scope);
const retries = await db.select({ id: agentWakeupRequests.id, companyId: agentWakeupRequests.companyId })
.from(agentWakeupRequests).innerJoin(issues, and(eq(issues.companyId, agentWakeupRequests.companyId),
sql`${issues.id}::text = ${agentWakeupRequests.payload}->>'issueId'`,
eq(issues.assigneeAgentId, agentWakeupRequests.agentId)))
.innerJoin(companies, and(eq(companies.id, issues.companyId), eq(companies.status, "active")))
.where(and(eq(agentWakeupRequests.reason, "task_retry"), eq(agentWakeupRequests.status, "deferred_issue_execution"),
.where(and(scope.companyId ? eq(agentWakeupRequests.companyId, scope.companyId) : undefined,
scope.requestId ? eq(agentWakeupRequests.id, scope.requestId) : undefined,
eq(agentWakeupRequests.reason, "task_retry"), eq(agentWakeupRequests.status, "deferred_issue_execution"),
sql`${agentWakeupRequests.payload}->'taskRetry'->>'chatActionId' is null`,
notInArray(issues.status, ["done", "cancelled"])))
.orderBy(asc(agentWakeupRequests.updatedAt)).limit(50);
for (const retry of retries) {
await db.update(agentWakeupRequests).set({ updatedAt: new Date() }).where(eq(agentWakeupRequests.id, retry.id));
await resumeTaskRetry(retry.companyId, retry.id).catch(err => logger.warn({ err, requestId: retry.id }, "Task retry dispatch deferred"));
await resumeTaskRetry(retry.companyId, retry.id).catch(err => {
scope.onError?.(err); logger.warn({ err, requestId: retry.id }, "Task retry dispatch deferred");
});
}
const waits = await db.select({ wake: agentWakeupRequests })
.from(agentWakeupRequests)
@@ -799,7 +800,9 @@ export function createHeartbeatRunControl(db: Db, dependencies: HeartbeatRunCont
sql`${issues.id}::text = ${agentWakeupRequests.payload}->>'issueId'`,
eq(issues.assigneeAgentId, agentWakeupRequests.agentId)))
.innerJoin(companies, and(eq(companies.id, issues.companyId), eq(companies.status, "active")))
.where(and(exists(db.select({ id: issueRecoveryActions.id }).from(issueRecoveryActions).where(and(
.where(and(scope.companyId ? eq(agentWakeupRequests.companyId, scope.companyId) : undefined,
scope.requestId ? eq(agentWakeupRequests.id, scope.requestId) : undefined,
exists(db.select({ id: issueRecoveryActions.id }).from(issueRecoveryActions).where(and(
eq(issueRecoveryActions.companyId, issues.companyId), eq(issueRecoveryActions.sourceIssueId, issues.id),
executionBlockerPredicate(),
))), eq(agentWakeupRequests.status, "deferred_issue_execution"),
@@ -831,11 +834,13 @@ export function createHeartbeatRunControl(db: Db, dependencies: HeartbeatRunCont
if (!run || run.companyId !== wake.companyId || run.agentId !== wake.agentId) continue;
if (canContinueCancelledRun(run)) {
await resumeSavedComments(wake.companyId, wake.id).catch(err => {
scope.onError?.(err);
logger.warn({ err, runId: run.id }, "failed to resume saved input after provider cancellation");
});
continue;
}
await resumeRemoteStopComments(run, wake.id).catch(err => {
scope.onError?.(err);
logger.warn({ err, runId: run.id }, "failed to resume saved execution-wait message");
});
}
server/src/services/heartbeat/run-lifecycle.ts
+2
View File
@@ -1,3 +1,4 @@
import { notifyRunDispatch } from "../run-dispatch-notifications.js";
import { preserveAcceptedRunUsageProjectionSql } from "../usage-receipts.js";
import { notifyNativeEnvironmentSettlement } from "../native-environment-settlement-notifications.js";
import { notifyExternalAgentSettlement } from "../../modules/external-agents/index.js";
@@ -2447,6 +2448,7 @@ export function createHeartbeatLifecycle(db: Db, dependencies: HeartbeatLifecycl
.returning()
.then((rows) => rows[0]);
await notifyRunDispatch(tx, { companyId: run.companyId });
const queuedRun = await tx
.insert(heartbeatRuns)
.values({
server/src/services/heartbeat/run-preparation.test.ts
+43
View File
@@ -14,6 +14,7 @@ import {
ConfigurationIncompleteFailure,
createHeartbeatRunPreparation,
resolveExecutionRunAdapterConfig,
readHeartbeatWakeComments,
} from "./run-preparation.js";
// Existing integration suites exercise the real database and dispatch path.
@@ -38,6 +39,12 @@ function databaseWithResults(...results: unknown[][]) {
return { db: { select } as unknown as Db, select, scopes };
}
function deferred<T>() {
let resolve!: (value: T) => void;
const promise = new Promise<T>(complete => { resolve = complete; });
return { promise, resolve };
}
const emptyRoutineContext = { routineId: null, env: null, responsibleUserId: null };
describe("heartbeat run preparation module", () => {
@@ -120,6 +127,42 @@ describe("heartbeat run preparation module", () => {
expect(database.select).toHaveBeenCalledTimes(2);
});
it("scopes one selected-comment snapshot for both wake consumers", async () => {
const database = databaseWithResults([{ id: "comment-one", body: "selected wake" }]);
expect(await readHeartbeatWakeComments(database.db, "company-one", "issue-one", ["comment-one"]))
.toEqual([{ id: "comment-one", body: "selected wake" }]);
expect(database.scopes[0]).toEqual(expect.arrayContaining(["company-one", "issue-one", "comment-one"]));
expect(await readHeartbeatWakeComments(database.db, "company-one", "issue-one", [])).toEqual([]);
expect(database.select).toHaveBeenCalledOnce();
});
it("settles independent binding checks before surfacing a failure or resolving values", async () => {
const projectCheck = deferred<[]>();
const firstFailure = new Error("agent binding check failed");
const secretsSvc = {
resolveAdapterConfigForRuntime: vi.fn(), resolveEnvBindings: vi.fn(),
collectMissingRuntimeBindings: vi.fn(async (_company, _env, context) => {
if (context.consumerType === "agent") throw firstFailure;
return context.consumerType === "project" ? projectCheck.promise : [];
}),
collectMissingAdapterConfigRuntimeBindings: vi.fn(async () => []),
};
let settled = false;
const opening = resolveExecutionRunAdapterConfig({
companyId: "company-one", agentId: "agent-one", projectId: "project-one",
environmentId: "environment-one", environmentEnv: { SHARED: "environment" },
executionRunConfig: { env: { SHARED: "agent" } }, projectEnv: { SHARED: "project" }, secretsSvc,
});
void opening.then(() => { settled = true; }, () => { settled = true; });
await vi.waitFor(() => expect(secretsSvc.collectMissingRuntimeBindings).toHaveBeenCalledTimes(3));
expect(secretsSvc.collectMissingAdapterConfigRuntimeBindings).toHaveBeenCalledOnce();
expect(settled).toBe(false);
expect(secretsSvc.resolveAdapterConfigForRuntime).not.toHaveBeenCalled();
expect(secretsSvc.resolveEnvBindings).not.toHaveBeenCalled();
projectCheck.resolve([]);
await expect(opening).rejects.toBe(firstFailure);
});
it("keeps missing scoped credentials as a pre-dispatch failure", async () => {
const secretsSvc = {
resolveAdapterConfigForRuntime: vi.fn(),
server/src/services/heartbeat/run-preparation.ts
+58 -97
View File
@@ -394,76 +394,31 @@ export async function resolveExecutionRunAdapterConfig(input: {
// no binding before resolving any secret value. Missing bindings short-circuit
// to a configuration-incomplete blocker routed to a human owner instead of a
// dispatched-then-failed run (which previously surfaced as opaque setup_failed).
if (typeof input.secretsSvc.collectMissingRuntimeBindings === "function") {
const missingBindings: MissingRuntimeBinding[] = [];
if (environmentEnv && input.environmentId) {
missingBindings.push(
...(await input.secretsSvc.collectMissingRuntimeBindings(
input.companyId,
environmentEnv,
{
consumerType: "environment",
consumerId: input.environmentId,
responsibleUserId: input.responsibleUserId ?? null,
},
)),
);
}
const collectMissing = input.secretsSvc.collectMissingRuntimeBindings;
if (typeof collectMissing === "function") {
const checks: Array<() => Promise<MissingRuntimeBinding[]>> = [];
const checkEnv = (env: unknown, consumerType: "environment" | "agent" | "project" | "routine", consumerId: string) => {
checks.push(() => collectMissing(input.companyId, env, {
consumerType, consumerId, responsibleUserId: input.responsibleUserId ?? null,
}));
};
if (environmentEnv && input.environmentId) checkEnv(environmentEnv, "environment", input.environmentId);
if (input.agentId) {
missingBindings.push(
...(await input.secretsSvc.collectMissingRuntimeBindings(
input.companyId,
parseObject(executionRunConfig.env),
{
consumerType: "agent",
consumerId: input.agentId,
responsibleUserId: input.responsibleUserId ?? null,
},
)),
);
if (
typeof input.secretsSvc.collectMissingAdapterConfigRuntimeBindings ===
"function"
) {
missingBindings.push(
...(await input.secretsSvc.collectMissingAdapterConfigRuntimeBindings(
input.companyId,
executionRunConfig,
input.adapterType ?? null,
{
consumerType: "agent",
consumerId: input.agentId,
responsibleUserId: input.responsibleUserId ?? null,
},
)),
);
checkEnv(parseObject(executionRunConfig.env), "agent", input.agentId);
const collectAdapterMissing = input.secretsSvc.collectMissingAdapterConfigRuntimeBindings;
if (typeof collectAdapterMissing === "function") {
checks.push(() => collectAdapterMissing(input.companyId, executionRunConfig, input.adapterType ?? null, {
consumerType: "agent", consumerId: input.agentId!, responsibleUserId: input.responsibleUserId ?? null,
}));
}
}
if (projectEnv && input.projectId) {
missingBindings.push(
...(await input.secretsSvc.collectMissingRuntimeBindings(
input.companyId,
projectEnv,
{
consumerType: "project",
consumerId: input.projectId,
responsibleUserId: input.responsibleUserId ?? null,
},
)),
);
}
if (routineEnv && input.routineId) {
missingBindings.push(
...(await input.secretsSvc.collectMissingRuntimeBindings(
input.companyId,
routineEnv,
{
consumerType: "routine",
consumerId: input.routineId,
responsibleUserId: input.responsibleUserId ?? null,
},
)),
);
if (projectEnv && input.projectId) checkEnv(projectEnv, "project", input.projectId);
if (routineEnv && input.routineId) checkEnv(routineEnv, "routine", input.routineId);
const results = await Promise.allSettled(checks.map(check => Promise.resolve().then(check)));
const missingBindings: MissingRuntimeBinding[] = [];
for (const result of results) {
if (result.status === "rejected") throw result.reason;
missingBindings.push(...result.value);
}
if (requiredScopedEnvBinding) {
const requiredEnvKeys = new Set(requiredScopedEnvBinding.keys);
@@ -2244,12 +2199,45 @@ export async function attestReviewedExternalChatRun(input: {
throw new Error("reviewed_chat_execution_binding_not_ready");
}
export async function readHeartbeatWakeComments(db: Db, companyId: string, issueId: string | null, commentIds: string[]) {
if (commentIds.length === 0) return [];
return db
.select({
id: issueComments.id,
companyId: issueComments.companyId,
issueId: issueComments.issueId,
body: issueComments.body,
origin: issueComments.origin,
authorType: issueComments.authorType,
authorAgentId: issueComments.authorAgentId,
authorUserId: issueComments.authorUserId,
presentation: issueComments.presentation,
metadata: issueComments.metadata,
deletedAt: issueComments.deletedAt,
deletedByType: issueComments.deletedByType,
deletedByAgentId: issueComments.deletedByAgentId,
deletedByUserId: issueComments.deletedByUserId,
deletedByRunId: issueComments.deletedByRunId,
sourceTrust: issueComments.sourceTrust,
createdAt: issueComments.createdAt,
})
.from(issueComments)
.where(
and(
eq(issueComments.companyId, companyId),
issueId ? eq(issueComments.issueId, issueId) : undefined,
inArray(issueComments.id, commentIds),
),
);
}
export async function buildPaperclipWakePayload(input: {
db: Db;
companyId: string;
agentId?: string | null;
runId?: string | null;
contextSnapshot: Record<string, unknown>;
wakeCommentRows?: Awaited<ReturnType<typeof readHeartbeatWakeComments>>;
continuationSummary?: {
key: string;
title: string | null;
@@ -2314,36 +2302,9 @@ export async function buildPaperclipWakePayload(input: {
)
return null;
const commentRows =
commentIds.length === 0
? []
: await input.db
.select({
id: issueComments.id,
issueId: issueComments.issueId,
body: issueComments.body,
origin: issueComments.origin,
authorType: issueComments.authorType,
authorAgentId: issueComments.authorAgentId,
authorUserId: issueComments.authorUserId,
presentation: issueComments.presentation,
metadata: issueComments.metadata,
deletedAt: issueComments.deletedAt,
deletedByType: issueComments.deletedByType,
deletedByAgentId: issueComments.deletedByAgentId,
deletedByUserId: issueComments.deletedByUserId,
deletedByRunId: issueComments.deletedByRunId,
sourceTrust: issueComments.sourceTrust,
createdAt: issueComments.createdAt,
})
.from(issueComments)
.where(
and(
eq(issueComments.companyId, input.companyId),
issueId ? eq(issueComments.issueId, issueId) : undefined,
inArray(issueComments.id, commentIds),
),
);
const commentRows = input.wakeCommentRows
? input.wakeCommentRows.filter(row => row.companyId === input.companyId && (!issueId || row.issueId === issueId) && commentIds.includes(row.id))
: await readHeartbeatWakeComments(input.db, input.companyId, issueId, commentIds);
const commentsById = new Map(
commentRows.map((comment) => [comment.id, comment]),
server/src/services/heartbeat/task-context.ts
+57 -35
View File
@@ -1,5 +1,5 @@
import { createHash } from "node:crypto";
import type { Db, agents, agentTaskSessions, heartbeatRuns, issueComments } from "@paperclipai/db";
import type { Db, agents, agentTaskSessions, heartbeatRuns } from "@paperclipai/db";
import { FAST_RESPONSE_AGENT_GUIDANCE, fastResponseHistoryBody } from "@paperclipai/shared";
import type { PaperclipTurnContext } from "@paperclipai/adapter-utils/server-utils";
import { parseObject } from "../../adapters/utils.js";
@@ -15,8 +15,9 @@ import { getTaskPlanContext } from "../task-plan-context.js";
import { caseExecutionMarkdown, getCaseExecutionContext } from "../case-context.js";
import type { TrustPresetResolution } from "../trust-preset-resolver.js";
import { liveVoiceExecutionGuidance } from "../voice/voice-execution-guidance.js";
import { buildPaperclipWakePayload, ConfigurationIncompleteFailure, PAPERCLIP_WAKE_PAYLOAD_KEY, type createHeartbeatRunPreparation } from "./run-preparation.js";
import { buildPaperclipWakePayload, ConfigurationIncompleteFailure, readHeartbeatWakeComments, PAPERCLIP_WAKE_PAYLOAD_KEY, type createHeartbeatRunPreparation } from "./run-preparation.js";
import { buildPaperclipTaskMarkdown } from "./task-markdown.js";
import { deriveCommentId, extractWakeCommentIds } from "../../modules/run-dispatch/index.js";
import { workspacePreferenceService } from "../workspace-preferences.js";
type RunPreparation = ReturnType<typeof createHeartbeatRunPreparation>;
@@ -30,8 +31,6 @@ export interface HeartbeatTaskContextInput {
"id" | "identifier" | "title" | "status" | "priority" | "workMode" |
"conversationAgentId" | "description" | "projectId"> | null;
issueContext: IssueContext | null;
wakeCommentContext: Pick<typeof issueComments.$inferSelect,
"id" | "body" | "sourceTrust" | "presentation" | "metadata"> | null;
context: Record<string, unknown>;
taskSession: Pick<typeof agentTaskSessions.$inferSelect, "lastRunId"> | null;
};
@@ -53,35 +52,9 @@ function readNonEmptyString(value: unknown): string | null {
/** Rebuilds task prompt fields on the run's context; acquires no execution resources. */
export async function prepareHeartbeatTaskContext(db: Db, input: HeartbeatTaskContextInput) {
const { run, agent } = input;
const { issueRef, issueContext, wakeCommentContext, context, taskSession } = input.task;
const { issueRef, issueContext, context, taskSession } = input.task;
const { trustPreset, experimentalInstanceSettings } = input.policy;
const { issuesSvc, getPinnedSkillTestContext } = input.services;
const continuationSummary = issueRef && !isConversation(issueContext)
? await getIssueContinuationSummaryDocument(db, issueRef.id)
: null;
const exposeLowTrustRaw = trustPreset.kind === "low_trust_review";
const safeContinuationSummary =
continuationSummary && !exposeLowTrustRaw
? redactQuarantinedBodyForHigherTrust(continuationSummary)
: continuationSummary;
const safeWakeCommentContext =
wakeCommentContext && !exposeLowTrustRaw
? sanitizeQuarantinedCommentForHigherTrust(wakeCommentContext)
: wakeCommentContext;
const issueAncestors = issueRef
? await issuesSvc.getAncestors(issueRef.id)
: [];
if (continuationSummary) {
context.paperclipContinuationSummary = {
key: safeContinuationSummary!.key,
title: safeContinuationSummary!.title,
body: safeContinuationSummary!.body,
sourceTrust: safeContinuationSummary!.sourceTrust ?? null,
updatedAt: safeContinuationSummary!.updatedAt.toISOString(),
};
} else {
delete context.paperclipContinuationSummary;
}
const admittedScope = parseObject(context.paperclipAgentJwtScope);
if (!input.policy.recovering || (admittedScope.kind !== "standard" && admittedScope.kind !== "skill_test")) {
const savedIssue = parseObject(context.paperclipIssue);
@@ -93,10 +66,58 @@ export async function prepareHeartbeatTaskContext(db: Db, input: HeartbeatTaskCo
: { kind: "standard" };
}
const isSkillTest = parseObject(context.paperclipAgentJwtScope).kind === "skill_test";
const pinnedSkillTestContext =
isSkillTest && issueRef
? await getPinnedSkillTestContext(agent.companyId, issueRef.id)
: null;
const wakeCommentId = deriveCommentId(context, null);
const commentIds = [...new Set([...extractWakeCommentIds(context), ...(wakeCommentId ? [wakeCommentId] : [])])];
const [summary, ancestors, pinnedSkillTestRead, wakeComments] = await Promise.allSettled([
issueRef && !isConversation(issueContext) ? getIssueContinuationSummaryDocument(db, issueRef.id) : Promise.resolve(null),
issueRef ? issuesSvc.getAncestors(issueRef.id) : Promise.resolve([]),
isSkillTest && issueRef ? getPinnedSkillTestContext(agent.companyId, issueRef.id) : Promise.resolve(null),
readHeartbeatWakeComments(db, agent.companyId, readNonEmptyString(context.issueId), commentIds),
]);
if (summary.status === "rejected") throw summary.reason;
if (ancestors.status === "rejected") throw ancestors.reason;
if (pinnedSkillTestRead.status === "rejected") throw pinnedSkillTestRead.reason;
if (wakeComments.status === "rejected") throw wakeComments.reason;
const wakeCommentRows = wakeComments.value;
const selectedWakeComment = issueRef && wakeCommentId ? wakeCommentRows.find(comment => comment.id === wakeCommentId && comment.issueId === issueRef.id) ?? null : null;
const wakeCommentContext = selectedWakeComment ? {
id: selectedWakeComment.id,
body: selectedWakeComment.deletedAt ? "" : selectedWakeComment.body,
authorType: selectedWakeComment.authorType,
authorAgentId: selectedWakeComment.authorAgentId,
authorUserId: selectedWakeComment.authorUserId,
presentation: selectedWakeComment.deletedAt ? null : selectedWakeComment.presentation,
metadata: selectedWakeComment.deletedAt ? null : selectedWakeComment.metadata,
deletedAt: selectedWakeComment.deletedAt,
deletedByType: selectedWakeComment.deletedByType,
deletedByAgentId: selectedWakeComment.deletedByAgentId,
deletedByUserId: selectedWakeComment.deletedByUserId,
deletedByRunId: selectedWakeComment.deletedByRunId,
sourceTrust: selectedWakeComment.sourceTrust,
} : null;
const continuationSummary = summary.value;
const issueAncestors = ancestors.value;
const pinnedSkillTestContext = pinnedSkillTestRead.value;
const exposeLowTrustRaw = trustPreset.kind === "low_trust_review";
const safeContinuationSummary =
continuationSummary && !exposeLowTrustRaw
? redactQuarantinedBodyForHigherTrust(continuationSummary)
: continuationSummary;
const safeWakeCommentContext =
wakeCommentContext && !exposeLowTrustRaw
? sanitizeQuarantinedCommentForHigherTrust(wakeCommentContext)
: wakeCommentContext;
if (continuationSummary) {
context.paperclipContinuationSummary = {
key: safeContinuationSummary!.key,
title: safeContinuationSummary!.title,
body: safeContinuationSummary!.body,
sourceTrust: safeContinuationSummary!.sourceTrust ?? null,
updatedAt: safeContinuationSummary!.updatedAt.toISOString(),
};
} else {
delete context.paperclipContinuationSummary;
}
if (isSkillTest && !pinnedSkillTestContext) {
throw new ConfigurationIncompleteFailure("The skill test's saved version is unavailable.", {
configurationIncomplete: { reason: "skill_unavailable", actionUrl: "/skills" },
@@ -132,6 +153,7 @@ export async function prepareHeartbeatTaskContext(db: Db, input: HeartbeatTaskCo
agentId: agent.id,
runId: run.id,
contextSnapshot: context,
wakeCommentRows,
continuationSummary,
issueSummary: issueRef
? {
server/src/services/instance-settings.ts
+4
View File
@@ -1,4 +1,6 @@
import { notifyStatusCardEvaluation } from "./status-card-notifications.js";
import { notifyDeferredExecutionEligibility } from "./deferred-execution-notifications.js";
import { notifyRunDispatch } from "./run-dispatch-notifications.js";
import { notifySkillVersionRetentionReconstruction } from "./skill-version-retention-notifications.js";
import { notifyNativeEnvironmentSettlement } from "./native-environment-settlement-notifications.js";
import { notifyExternalAgentSettlement } from "../modules/external-agents/index.js";
@@ -632,6 +634,8 @@ export function instanceSettingsService(db: Db, options: InstanceSettingsService
await notifyDeliveryWork(tx, DELIVERY_QUEUES.grokDispatch);
if (patch.enableWorktreeRunExecution !== undefined) await notifyScheduledRetryCancellation(tx);
if (patch.enableWorktreeRunExecution !== undefined) {
await notifyRunDispatch(tx);
await notifyDeferredExecutionEligibility(tx);
await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers);
await notifyExternalAgentSettlement(tx);
await notifyInstructionCommit(tx);
server/src/services/issue-comment-writes.ts
+3
View File
@@ -1,4 +1,5 @@
import { notifyStatusCardIssueChange } from "./status-card-notifications.js";
import { notifyIssueRecovery } from "./issue-recovery-notifications.js";
import type { PgInsertValue } from "drizzle-orm/pg-core";
import { issueComments, type Db } from "@paperclipai/db";
import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js";
@@ -8,6 +9,8 @@ type WriteDb = Db | Parameters<Parameters<Db["transaction"]>[0]>[0];
export function insertIssueComment(db: WriteDb, input: PgInsertValue<typeof issueComments> & { companyId: string }) {
return db.transaction(async tx => {
await notifyStatusCardIssueChange(tx, input.companyId);
await notifyIssueRecovery(tx, { companyId: input.companyId,
issueId: typeof input.issueId === "string" ? input.issueId : undefined });
await notifyDeliveryWork(tx, DELIVERY_QUEUES.dotEvents, `company:${input.companyId}`);
return tx.insert(issueComments).values(input).returning();
});
server/src/services/issue-recovery-actions.ts
+6
View File
@@ -1,3 +1,4 @@
import { notifyIssueRecovery } from "./issue-recovery-notifications.js";
import { notifyNativeRecoveryNotices } from "./native-recovery-notice-notifications.js";
import { notifyUnsafeWorkspaceExportRecovery } from "./unsafe-workspace-export-notifications.js";
import { writeExecutionControl } from "./execution-control-notifications.js";
@@ -322,6 +323,7 @@ export function issueRecoveryActionService(db: DbOrTransaction) {
): Promise<IssueRecoveryAction> {
try {
const created = await db.transaction(async (tx) => {
await notifyIssueRecovery(tx, { companyId: input.companyId });
await notifyNativeRecoveryNotices(tx, { companyId: input.companyId, issueId: input.sourceIssueId });
await notifyDeliveryWork(tx, DELIVERY_QUEUES.executionDisposition);
await notifyDeliveryWork(tx, DELIVERY_QUEUES.executionContinuation);
@@ -413,6 +415,7 @@ export function issueRecoveryActionService(db: DbOrTransaction) {
Math.min(nextAttemptCount, effectiveMaxAttempts),
);
const [exhausted] = await writeExecutionControl(db, [DELIVERY_QUEUES.executionDisposition, DELIVERY_QUEUES.executionContinuation], async tx => {
await notifyIssueRecovery(tx, { companyId: input.companyId });
await notifyNativeRecoveryNotices(tx, { companyId: input.companyId, issueId: input.sourceIssueId });
if ((!input.preserveExistingOwner && existing.cause !== input.cause)
|| (input.evidence?.runId !== undefined && input.evidence.runId !== existing.evidence?.runId)) {
@@ -474,6 +477,7 @@ export function issueRecoveryActionService(db: DbOrTransaction) {
return toReadModel(exhausted);
}
const [updated] = await writeExecutionControl(db, [DELIVERY_QUEUES.executionDisposition, DELIVERY_QUEUES.executionContinuation], async tx => {
await notifyIssueRecovery(tx, { companyId: input.companyId });
await notifyNativeRecoveryNotices(tx, { companyId: input.companyId, issueId: input.sourceIssueId });
if ((!input.preserveExistingOwner && existing.cause !== input.cause)
|| (input.evidence?.runId !== undefined && input.evidence.runId !== existing.evidence?.runId)) {
@@ -548,6 +552,7 @@ export function issueRecoveryActionService(db: DbOrTransaction) {
try {
const [created] = await writeExecutionControl(db, [DELIVERY_QUEUES.executionDisposition, DELIVERY_QUEUES.executionContinuation], async tx => {
await notifyIssueRecovery(tx, { companyId: input.companyId });
await notifyNativeRecoveryNotices(tx, { companyId: input.companyId, issueId: input.sourceIssueId });
await notifyUnsafeWorkspaceExportRecovery(tx, { companyId: input.companyId, issueId: input.sourceIssueId });
return tx
@@ -595,6 +600,7 @@ export function issueRecoveryActionService(db: DbOrTransaction) {
const pending = await tx.select({ id: issueRecoveryActions.id }).from(issueRecoveryActions)
.where(and(...predicates)).limit(1);
if (!pending.length) return [];
await notifyIssueRecovery(tx, { companyId: input.companyId });
await notifyNativeRecoveryNotices(tx, { companyId: input.companyId, issueId: input.sourceIssueId });
await notifyUnsafeWorkspaceExportRecovery(tx, { companyId: input.companyId, issueId: input.sourceIssueId });
return tx
server/src/services/issue-recovery-notifications.ts
+16
View File
@@ -0,0 +1,16 @@
import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js";
/** An issue hint is reserved for local facts such as comments. Lifecycle,
* graph, run and eligibility changes retain company scope because their
* consequences can include ancestors and shared execution references. */
export type IssueRecoveryScope = { companyId: string; issueId?: string };
export async function notifyIssueRecovery(transaction: object, scope?: IssueRecoveryScope) {
const companyHint = scope ? JSON.stringify({ companyId: scope.companyId }) : "*";
const issueHint = scope ? JSON.stringify({ companyId: scope.companyId, issueId: scope.issueId }) : "*";
for (const queue of [DELIVERY_QUEUES.orphanBlockers, DELIVERY_QUEUES.activeRecoveryActions, DELIVERY_QUEUES.strandedAssignments, DELIVERY_QUEUES.resolvedDependencies,
DELIVERY_QUEUES.taskWatchdogs, DELIVERY_QUEUES.staleIssueLocks]) {
await notifyDeliveryWork(transaction, queue,
queue === DELIVERY_QUEUES.strandedAssignments || queue === DELIVERY_QUEUES.staleIssueLocks ? issueHint : companyHint);
}
}
server/src/services/issue-review-state-notifications.ts
+2
View File
@@ -1,4 +1,5 @@
import { notifyStatusCardIssueChange } from "./status-card-notifications.js";
import { notifyRunDispatch } from "./run-dispatch-notifications.js";
import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js";
import { notifyExternalConversationSettlement } from "./external-conversation-settlement-notifications.js";
import { notifyNativeReviewResponse } from "./native-review-response-notifications.js";
@@ -6,6 +7,7 @@ import { notifyNativeStatusReassessment } from "./native-status-reassessment-not
export async function notifyIssueReviewState(tx: object, scope: { companyId: string; issueId?: string; runId?: string; agentId?: string }) {
await notifyStatusCardIssueChange(tx, scope.companyId);
await notifyRunDispatch(tx, scope);
await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors);
await notifyExternalConversationSettlement(tx, scope);
await notifyNativeReviewResponse(tx, scope);
server/src/services/issue-tree-control.ts
+12 -1
View File
@@ -1,4 +1,6 @@
import { notifyStatusCardIssueChange } from "./status-card-notifications.js";
import { notifyDeferredExecutionEligibility } from "./deferred-execution-notifications.js";
import { notifyIssueRecovery } from "./issue-recovery-notifications.js";
import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js";
import { notifyNativeStatusReassessment } from "./native-status-reassessment-notifications.js";
import { notifyUnsafeWorkspaceExportRecovery } from "./unsafe-workspace-export-notifications.js";
@@ -759,6 +761,8 @@ export function issueTreeControlService(db: Db) {
const releaseReason = input.reason ?? "Subtree resume applied.";
const { hold: resumeHold } = await db.transaction(async (tx) => {
await notifyIssueRecovery(tx, { companyId });
await notifyDeferredExecutionEligibility(tx, { companyId });
const [createdHold] = await tx
.insert(issueTreeHolds)
.values({
@@ -837,6 +841,8 @@ export function issueTreeControlService(db: Db) {
}
const { hold, members } = await db.transaction(async (tx) => {
await notifyIssueRecovery(tx, { companyId });
await notifyDeferredExecutionEligibility(tx, { companyId });
const [createdHold] = await tx
.insert(issueTreeHolds)
.values({
@@ -1001,6 +1007,7 @@ export function issueTreeControlService(db: Db) {
const now = new Date();
const releasedCancelHoldIds = activeCancelHolds.map((hold) => hold.id);
const updatedIssues = await db.transaction(async (tx) => {
await notifyDeferredExecutionEligibility(tx, { companyId });
await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors);
await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers);
const restored: TreeStatusUpdateResult["updatedIssues"] = [];
@@ -1179,7 +1186,10 @@ export function issueTreeControlService(db: Db) {
throw conflict("Issue tree hold is already released");
}
const [updated] = await db
const [updated] = await db.transaction(async tx => {
await notifyIssueRecovery(tx, { companyId });
await notifyDeferredExecutionEligibility(tx, { companyId });
return tx
.update(issueTreeHolds)
.set({
status: "released",
@@ -1197,6 +1207,7 @@ export function issueTreeControlService(db: Db) {
})
.where(and(eq(issueTreeHolds.id, holdId), eq(issueTreeHolds.companyId, companyId)))
.returning();
});
const members = await db
.select()
server/src/services/issues.ts
+46 -21
View File
@@ -1,4 +1,7 @@
import { notifyStatusCardIssueChange } from "./status-card-notifications.js";
import { notifyDeferredExecutionEligibility } from "./deferred-execution-notifications.js";
import { notifyIssueRecovery } from "./issue-recovery-notifications.js";
import { notifyRunDispatch } from "./run-dispatch-notifications.js";
import { notifyExternalConversationSettlement } from "./external-conversation-settlement-notifications.js";
import { notifyNativeRecoveryNotices } from "./native-recovery-notice-notifications.js";
import { notifyNativeRunProjection } from "./native-run-projection-notifications.js";
@@ -7627,6 +7630,8 @@ export function issueService(db: Db) {
await assertNoBlockingCycles(companyId, issueId, deduped, dbOrTx);
}
await notifyIssueRecovery(dbOrTx, { companyId });
await notifyDeferredExecutionEligibility(dbOrTx, { companyId });
await dbOrTx
.delete(issueRelations)
.where(
@@ -9677,27 +9682,30 @@ export function issueService(db: Db) {
});
if (blockParentUntilDone) {
const existingBlockers = await db
.select({ blockerIssueId: issueRelations.issueId })
.from(issueRelations)
.where(
and(
eq(issueRelations.companyId, parent.companyId),
eq(issueRelations.relatedIssueId, parent.id),
eq(issueRelations.type, "blocks"),
),
await db.transaction(async tx => {
const existingBlockers = await tx
.select({ blockerIssueId: issueRelations.issueId })
.from(issueRelations)
.where(
and(
eq(issueRelations.companyId, parent.companyId),
eq(issueRelations.relatedIssueId, parent.id),
eq(issueRelations.type, "blocks"),
),
);
await syncBlockedByIssueIds(
parent.id,
parent.companyId,
[
...new Set([
...existingBlockers.map((row) => row.blockerIssueId),
child.id,
]),
],
{ agentId: actorAgentId ?? null, userId: actorUserId ?? null },
tx,
);
await syncBlockedByIssueIds(
parent.id,
parent.companyId,
[
...new Set([
...existingBlockers.map((row) => row.blockerIssueId),
child.id,
]),
],
{ agentId: actorAgentId ?? null, userId: actorUserId ?? null },
);
});
[child] = await withIssueRelationSummaries(
parent.companyId,
[child],
@@ -10608,6 +10616,8 @@ export function issueService(db: Db) {
: values.parentId ? `issue:${values.parentId}` : undefined);
}
if (values.monitorNextCheckAt) await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors);
await notifyRunDispatch(tx, { companyId: values.companyId });
await notifyDeferredExecutionEligibility(tx, { companyId: values.companyId });
await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers);
await notifyStatusCardIssueChange(tx, companyId);
const [issue] = await tx.insert(issues).values(values).returning();
@@ -11499,7 +11509,11 @@ export function issueService(db: Db) {
await notifyTerminalWorkspaceWork(tx, `issue:${id}`);
}
if (patch.status !== undefined || patch.assigneeAgentId !== undefined || patch.assigneeUserId !== undefined
|| patch.hiddenAt !== undefined || patch.conversationAgentId !== undefined) {
|| patch.hiddenAt !== undefined || patch.conversationAgentId !== undefined || patch.parentId !== undefined
|| patch.executionState !== undefined || patch.executionRunId !== undefined || patch.checkoutRunId !== undefined
|| blockedByIssueIds !== undefined) {
await notifyRunDispatch(tx, { companyId: receiptExisting.companyId });
await notifyDeferredExecutionEligibility(tx, { companyId: receiptExisting.companyId });
await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers);
}
if (patch.monitorNextCheckAt !== undefined || (receiptExisting.monitorNextCheckAt &&
@@ -11968,6 +11982,8 @@ export function issueService(db: Db) {
}
await notifyDecisionTargetIssueRemoval(tx, id);
await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors);
await notifyRunDispatch(tx, owner ? { companyId: owner.companyId } : undefined);
if (owner) await notifyDeferredExecutionEligibility(tx, owner);
await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers);
removedIssue = await tx
.delete(issues)
@@ -12097,6 +12113,8 @@ export function issueService(db: Db) {
const updated = await db.transaction(async tx => {
await notifyStatusCardIssueChange(tx, issueCompany.companyId);
await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors);
await notifyRunDispatch(tx, { companyId: issueCompany.companyId });
await notifyDeferredExecutionEligibility(tx, { companyId: issueCompany.companyId });
await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers);
return tx
.update(issues)
@@ -12230,6 +12248,8 @@ export function issueService(db: Db) {
const adopted = await db.transaction(async tx => {
await notifyStatusCardIssueChange(tx, issueCompany.companyId);
await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors);
await notifyRunDispatch(tx, { companyId: issueCompany.companyId });
await notifyDeferredExecutionEligibility(tx, { companyId: issueCompany.companyId });
await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers);
return tx
.update(issues)
@@ -12485,6 +12505,8 @@ export function issueService(db: Db) {
await notifyTerminalWorkspaceWork(tx, `issue:${id}`);
await notifyNativeStatusReassessment(tx, { companyId: existing.companyId, issueId: id });
await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors);
await notifyRunDispatch(tx, { companyId: existing.companyId });
await notifyDeferredExecutionEligibility(tx, { companyId: existing.companyId });
await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers);
const updated = await tx
.update(issues)
@@ -12546,6 +12568,8 @@ export function issueService(db: Db) {
await notifyUnsafeWorkspaceExportRecovery(tx, { companyId: existing.companyId, issueId: id });
}
await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors);
await notifyRunDispatch(tx, { companyId: existing.companyId });
await notifyDeferredExecutionEligibility(tx, { companyId: existing.companyId });
await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers);
const updated = await tx
.update(issues)
@@ -13809,6 +13833,7 @@ export function issueService(db: Db) {
.then((rows) => rows[0] ?? null);
if (!existing) return null;
await notifyIssueRecovery(tx, { companyId: existing.companyId });
await tx.delete(issueAttachments).where(eq(issueAttachments.id, id));
await tx.delete(assets).where(eq(assets.id, existing.assetId));
return existing;
server/src/services/legacy-controller-lease.ts
+8 -4
View File
@@ -77,14 +77,18 @@ async function renewOnLivenessPool(
});
}
export async function hasLiveLegacyController(db: Db, run: Run): Promise<boolean> {
if (run.runtimeMode === "native" || !run.controllerBootId) return false;
const [owner] = await db.select({ id: heartbeatRuns.id }).from(heartbeatRuns).where(and(
export async function liveLegacyControllerExpiresAt(db: Db, run: Run): Promise<Date | null> {
if (run.runtimeMode === "native" || !run.controllerBootId) return null;
const [owner] = await db.select({ expiresAt: heartbeatRuns.controllerLeaseExpiresAt }).from(heartbeatRuns).where(and(
eq(heartbeatRuns.id, run.id), eq(heartbeatRuns.companyId, run.companyId),
eq(heartbeatRuns.status, "running"),
gt(heartbeatRuns.controllerLeaseExpiresAt, sql`clock_timestamp()`),
));
return Boolean(owner);
return owner?.expiresAt ?? null;
}
export async function hasLiveLegacyController(db: Db, run: Run): Promise<boolean> {
return (await liveLegacyControllerExpiresAt(db, run)) !== null;
}
export type RevokedLegacyControllerState = Pick<Run, "executionStage" | "processPid" | "processGroupId">;
server/src/services/native-runtime/chat-attachment-reuse.test.ts
+1 -1
View File
@@ -73,7 +73,7 @@ describe("native same-conversation chat attachment reuse", () => {
beforeAll(async () => {
const external = resolveExternalTestDatabaseUrl("PAPERCLIP_CHAT_REUSE_TEST_DATABASE_URL");
if (!external) temporary = await startEmbeddedPostgresTestDatabase("native-chat-reuse-");
db = createDb(external ?? temporary!.connectionString);
db = createDb(external ?? temporary!.connectionString, { maxConnections: 6 });
const storageRoot = await mkdtemp(
path.join(tmpdir(), "paperclip-chat-reuse-"),
);
server/src/services/native-runtime/chat-attachment-reuse.ts
+5 -1
View File
@@ -1,3 +1,4 @@
import { notifyIssueRecovery } from "../issue-recovery-notifications.js";
import { createHash } from "node:crypto";
import { and, desc, eq, inArray, isNull, sql } from "drizzle-orm";
@@ -1582,7 +1583,9 @@ export async function prepareReusedChatAttachment(input: {
if (!workProduct) {
throw new Error("paperclip_runner_chat_attachment_origin_not_persisted");
}
await input.db
await input.db.transaction(async tx => {
await notifyIssueRecovery(tx, { companyId: input.binding.companyId });
await tx
.update(issueWorkProducts)
.set({
title: input.title,
@@ -1602,6 +1605,7 @@ export async function prepareReusedChatAttachment(input: {
eq(issueWorkProducts.createdByRunId, input.binding.runId),
),
);
});
const comment = await issueService(input.db).addComment(
input.binding.issueId,
`Prepared ${input.title} for this response.`,
server/src/services/native-runtime/native-provider-capacity-retry.ts
+2
View File
@@ -1,3 +1,4 @@
import { notifyRunDispatch } from "../run-dispatch-notifications.js";
import { randomUUID } from "node:crypto";
import { and, eq } from "drizzle-orm";
import { agentWakeupRequests, heartbeatRuns, type Db } from "@paperclipai/db";
@@ -49,6 +50,7 @@ export async function materializeNativeProviderCapacityRetry(input: {
scheduledRetryAt: dueAt.toISOString(),
executionRetryAccounting: accountingForScheduledRetry(run, NATIVE_PROVIDER_OVERLOADED_CODE, attempt),
};
await notifyRunDispatch(input.tx, { companyId: run.companyId });
const [wake] = await input.tx.insert(agentWakeupRequests).values({
companyId: run.companyId, agentId: run.agentId, source: "automation", triggerDetail: "system",
reason: NATIVE_PROVIDER_OVERLOADED_CODE, status: "queued", runId: retryId,
server/src/services/native-runtime/native-runner-file-handoff.test.ts
+4 -4
View File
@@ -1,3 +1,4 @@
import { resolveExternalTestDatabaseUrl } from "../../__tests__/helpers/external-test-database.js";
import type { PrpStructuredRunResult } from "../../vendor/paperclip-runner/index.js";
import { nativeCompletionFeedback } from "./native-completion-feedback.js";
import { createHash, randomUUID } from "node:crypto";
@@ -61,10 +62,9 @@ describe("native runner file handoff", () => {
const runId = "00000000-0000-4000-8000-000000009104";
beforeAll(async () => {
temporary = await startEmbeddedPostgresTestDatabase(
"native-runner-file-handoff-",
);
db = createDb(temporary.connectionString);
const external = resolveExternalTestDatabaseUrl("PAPERCLIP_NATIVE_HANDOFF_TEST_DATABASE_URL");
if (!external) temporary = await startEmbeddedPostgresTestDatabase("native-runner-file-handoff-");
db = createDb(external ?? temporary!.connectionString, { maxConnections: 6 });
temporaryRoot = await mkdtemp(
path.join(tmpdir(), "paperclip-native-file-handoff-"),
);
server/src/services/native-runtime/native-runner-file-handoff.ts
+5 -1
View File
@@ -1,3 +1,4 @@
import { notifyIssueRecovery } from "../issue-recovery-notifications.js";
import { createHash, randomUUID } from "node:crypto";
import { execFile } from "node:child_process";
import { constants } from "node:fs";
@@ -1265,7 +1266,9 @@ export async function prepareNativeRunnerFileHandoff(input: {
) {
throw new Error("paperclip_runner_file_handoff_origin_not_persisted");
}
await input.db
await input.db.transaction(async tx => {
await notifyIssueRecovery(tx, { companyId: input.binding.companyId });
await tx
.update(issueWorkProducts)
.set({ title: verified.title, updatedAt: new Date() })
.where(
@@ -1276,6 +1279,7 @@ export async function prepareNativeRunnerFileHandoff(input: {
eq(issueWorkProducts.createdByRunId, input.binding.runId),
),
);
});
const comment = await issueService(input.db).addComment(
input.binding.issueId,
`Prepared ${verified.title} for this response.`,
server/src/services/native-runtime/native-workspace-finalizer.test.ts
+5 -5
View File
@@ -131,7 +131,7 @@ describe("native workspace finalization recovery", () => {
const external = resolveExternalTestDatabaseUrl("PAPERCLIP_NATIVE_WORKSPACE_FINALIZER_TEST_DATABASE_URL");
if (!external) temporary = await startEmbeddedPostgresTestDatabase("paperclip-native-workspace-finalizer-");
connectionString = external ?? temporary!.connectionString;
db = createDb(connectionString);
db = createDb(connectionString, { maxConnections: 6 });
await db.insert(companies).values([
{ id: companyId, name: "Native workspace owner", issuePrefix: "NWO" },
@@ -452,7 +452,7 @@ describe("native workspace finalization recovery", () => {
const seeded = await seedRun({ executionWorkspaceId: randomUUID(), title: "Live copyback" });
const live = await withNativeWorkspaceFinalizationOwnership({ db, companyId, runId: seeded.runId }, async (ownership) => {
await ownership.assertHeld();
expect(await resumeNativeWorkspaceFinalization({ db: createDb(connectionString), runId: seeded.runId })).toBeNull();
expect(await resumeNativeWorkspaceFinalization({ db: createDb(connectionString, { maxConnections: 6 }), runId: seeded.runId })).toBeNull();
expect(await db.select().from(workspaceOperations).where(eq(workspaceOperations.heartbeatRunId, seeded.runId))).toEqual([]);
return "copied";
});
@@ -476,7 +476,7 @@ describe("native workspace finalization recovery", () => {
try {
await entered;
await db.execute(sql`select pg_terminate_backend(pid) from pg_locks where locktype = 'advisory' and database = (select oid from pg_database where datname = current_database()) and pid <> pg_backend_pid()`);
const second = await withNativeWorkspaceFinalizationOwnership({ db: createDb(connectionString), companyId, runId: seeded.runId }, async () => "must not run");
const second = await withNativeWorkspaceFinalizationOwnership({ db: createDb(connectionString, { maxConnections: 6 }), companyId, runId: seeded.runId }, async () => "must not run");
expect(second).toEqual({ acquired: false, reason: "busy" });
const [held] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, seeded.runId));
expect(held.runnerProfileJson?.nativeWorkspaceFinalizationOwner).toBeTruthy();
@@ -489,7 +489,7 @@ describe("native workspace finalization recovery", () => {
it("recovers its own joined copyback receipt after the application pool disconnects during cleanup", async () => {
const seeded = await seedRun({ executionWorkspaceId: randomUUID(), title: "Reconnect joined copyback" });
const interruptedDb = createDb(connectionString);
const interruptedDb = createDb(connectionString, { maxConnections: 6 });
let joined = false;
await expect(withNativeWorkspaceFinalizationOwnership({ db: interruptedDb, companyId, runId: seeded.runId }, async () => {
await fs.writeFile(path.join(workspaceRoot, "joined-copyback.txt"), "durable work");
@@ -507,7 +507,7 @@ describe("native workspace finalization recovery", () => {
.toEqual({ acquired: false, reason: "owner_unverified" });
}
await db.update(heartbeatRuns).set({ runnerProfileJson: beforeReconnect.runnerProfileJson }).where(eq(heartbeatRuns.id, seeded.runId));
expect(await withNativeWorkspaceFinalizationOwnership({ db: createDb(connectionString), companyId, runId: seeded.runId }, async () => "resumed"))
expect(await withNativeWorkspaceFinalizationOwnership({ db: createDb(connectionString, { maxConnections: 6 }), companyId, runId: seeded.runId }, async () => "resumed"))
.toEqual({ acquired: true, value: "resumed" });
});
server/src/services/native-runtime/runner-api.integration.test.ts
+1 -1
View File
@@ -497,7 +497,7 @@ else if(m.id!==undefined) send({id:m.id,result:{}});
const [agent] = await server.db.select().from(agents).where(eq(agents.id, fixture.agentId));
const context = structuredClone(run.contextSnapshot ?? {});
await prepareHeartbeatTaskContext(server.db, { run, agent,
task: { issueRef: issue, issueContext: null, wakeCommentContext: null, context, taskSession: null },
task: { issueRef: issue, issueContext: null, context, taskSession: null },
policy: { recovering: true, trustPreset: { kind: "standard" }, experimentalInstanceSettings: {} },
services: { issuesSvc: { getAncestors: async () => [] },
getPinnedSkillTestContext: createHeartbeatRunPreparation(server.db).getPinnedSkillTestContext },
server/src/services/native-runtime/status-decision-committer.ts
+2
View File
@@ -1,3 +1,4 @@
import { notifyNativeStatusWakeWork } from "../deferred-execution-notifications.js";
import { notifyExternalAgentSettlement } from "../../modules/external-agents/index.js";
import { lockCurrentNativeWorkAssessmentEvidence } from "./work-assessments.js";
import { hasNativeWorkspaceFinalizationOwner } from "./native-workspace-finalization-recovery-state.js";
@@ -478,6 +479,7 @@ async function enqueueWake(input: {
.limit(1)
.then((rows) => rows[0] ?? null);
if (existing) return existing.id;
await notifyNativeStatusWakeWork(input.tx, { companyId: input.companyId });
const inserted = await input.tx
.insert(agentWakeupRequests)
.values({
server/src/services/native-status-reassessment-notifications.ts
+2
View File
@@ -1,10 +1,12 @@
import { notifyStatusCardIssueChange } from "./status-card-notifications.js";
import { notifyIssueRecovery } from "./issue-recovery-notifications.js";
import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js";
export type NativeStatusReassessmentScope = { companyId: string; issueId?: string; runId?: string; agentId?: string };
export async function notifyNativeStatusReassessment(tx: object, scope: NativeStatusReassessmentScope) {
await notifyStatusCardIssueChange(tx, scope.companyId);
await notifyIssueRecovery(tx, scope ? { companyId: scope.companyId } : undefined);
return notifyDeliveryWork(tx, DELIVERY_QUEUES.nativeStatusReassessment,
JSON.stringify({ companyId: scope.companyId, issueId: scope.issueId, runId: scope.runId, agentId: scope.agentId }));
}
server/src/services/recovery/service.ts
+53 -19
View File
@@ -1,3 +1,4 @@
import { notifyRunDispatch } from "../run-dispatch-notifications.js";
import { notifyNativeEnvironmentSettlement } from "../native-environment-settlement-notifications.js";
import { notifyExternalAgentSettlement } from "../../modules/external-agents/index.js";
import { notifyNativeStatusReassessment } from "../native-status-reassessment-notifications.js";
@@ -26,7 +27,7 @@ import {
decideLegacyContinuation, legacyDispositionEpisode, legacyDispositionFingerprint,
LEGACY_DISPOSITION_REPAIR_INSTRUCTION, type LegacyDispositionEpisode,
} from "./legacy-continuation.js";
import { CONTROLLER_LEASE_LOST_FAILURE_CODE, hasLiveLegacyController } from "../legacy-controller-lease.js";
import { CONTROLLER_LEASE_LOST_FAILURE_CODE, hasLiveLegacyController, liveLegacyControllerExpiresAt } from "../legacy-controller-lease.js";
import { isCompletedOnboardingHandoffWake } from "../chat-completion-delivery.js";
import { instanceSettingsService } from "../instance-settings.js";
import { isWaitingConversation, settleConversationTurnForRun, deliverConversationComments } from "../agent-conversations.js";
@@ -244,6 +245,7 @@ type ResolvedDependencyWakeBackstopOptions = {
runId?: string | null;
companyId?: string | null;
blockerIssueId?: string | null;
issueId?: string;
source?: ResolvedDependencyWakeBackstopSource;
};
@@ -2201,7 +2203,7 @@ export function recoveryService(
return Boolean(budgetBlock);
}
async function reconcileUnassignedBlockingIssues() {
async function reconcileUnassignedBlockingIssues(opts?: { companyId?: string }) {
const candidates = await db
.select({
id: issues.id,
@@ -2215,6 +2217,8 @@ export function recoveryService(
.where(
and(
eq(issueRelations.type, "blocks"),
eq(issueRelations.companyId, issues.companyId),
opts?.companyId ? eq(issues.companyId, opts.companyId) : undefined,
inArray(issues.status, ["todo", "blocked"]),
isNull(issues.assigneeAgentId),
isNull(issues.assigneeUserId),
@@ -2716,6 +2720,7 @@ export function recoveryService(
const now = new Date();
const retryAt = readProviderQuotaRetryAt(input.latestRun, now);
return database.transaction(async (tx) => {
await notifyRunDispatch(tx, { companyId: input.issue.companyId });
const wakeup = await tx
.insert(agentWakeupRequests)
.values({
@@ -3316,6 +3321,7 @@ export function recoveryService(
created = Boolean(enqueuedRun);
} else {
scheduledRun = await db.transaction(async (tx) => {
await notifyRunDispatch(tx, { companyId: input.issue.companyId });
const wakeup = await tx
.insert(agentWakeupRequests)
.values({
@@ -3497,7 +3503,7 @@ export function recoveryService(
return Boolean(run || wake);
}
async function reconcileActiveRecoveryActions() {
async function reconcileActiveRecoveryActions(opts?: { companyId?: string }) {
const rows = await db
.select({
action: issueRecoveryActions,
@@ -3511,7 +3517,8 @@ export function recoveryService(
eq(issues.companyId, issueRecoveryActions.companyId),
),
)
.where(inArray(issueRecoveryActions.status, ["active", "escalated"]));
.where(and(inArray(issueRecoveryActions.status, ["active", "escalated"]),
opts?.companyId ? eq(issueRecoveryActions.companyId, opts.companyId) : undefined));
const result = {
requeued: 0,
@@ -4523,14 +4530,20 @@ export function recoveryService(
}
async function reconcileStrandedAssignedIssues(opts?: {
companyId?: string;
issueId?: string;
onDeferredDeadline?: (at: number) => void;
issueCreatedAtGte?: Date | null;
skipExternalConversationSettlement?: boolean;
skipRelatedRecovery?: boolean;
}) {
const candidates = await db
.select(recoveryIssueFields)
.from(issues)
.where(
and(
opts?.companyId ? eq(issues.companyId, opts.companyId) : undefined,
opts?.issueId ? eq(issues.id, opts.issueId) : undefined,
isNull(issues.assigneeUserId),
inArray(issues.status, ["todo", "in_progress", "in_review"]),
or(
@@ -5711,6 +5724,7 @@ export function recoveryService(
classification.baseBackoffMs *
Math.pow(2, Math.max(0, consecutive - 1));
if (elapsed < requiredDelay) {
opts?.onDeferredDeadline?.(latestFinishedAt.getTime() + requiredDelay);
result.skipped += 1;
continue;
}
@@ -5770,16 +5784,18 @@ export function recoveryService(
}
}
const orphanBlockerRecovery = await reconcileUnassignedBlockingIssues();
result.orphanBlockersAssigned = orphanBlockerRecovery.assigned;
result.skipped += orphanBlockerRecovery.skipped;
result.issueIds.push(...orphanBlockerRecovery.issueIds);
if (!opts?.skipRelatedRecovery) {
const orphanBlockerRecovery = await reconcileUnassignedBlockingIssues(opts);
result.orphanBlockersAssigned = orphanBlockerRecovery.assigned;
result.skipped += orphanBlockerRecovery.skipped;
result.issueIds.push(...orphanBlockerRecovery.issueIds);
const activeRecovery = await reconcileActiveRecoveryActions();
result.continuationRequeued += activeRecovery.requeued;
result.escalated += activeRecovery.escalated;
result.skipped += activeRecovery.skipped;
result.issueIds.push(...activeRecovery.issueIds);
const activeRecovery = await reconcileActiveRecoveryActions(opts);
result.continuationRequeued += activeRecovery.requeued;
result.escalated += activeRecovery.escalated;
result.skipped += activeRecovery.skipped;
result.issueIds.push(...activeRecovery.issueIds);
}
result.issueIds = [...new Set(result.issueIds)];
return result;
@@ -5812,7 +5828,7 @@ export function recoveryService(
source === "workspace.finalize"
? "workspace_finalize_reconciliation"
: "issue_graph_liveness_reconciliation";
const useCursor = !opts?.blockerIssueId;
const useCursor = !opts?.blockerIssueId && !opts?.issueId;
const queryCandidates = (afterIssueId: string | null) => {
const filters = [
@@ -5822,6 +5838,7 @@ export function recoveryService(
sql`${issues.assigneeAgentId} is not null`,
];
if (opts?.companyId) filters.push(eq(issues.companyId, opts.companyId));
if (opts?.issueId) filters.push(eq(issues.id, opts.issueId));
if (afterIssueId) filters.push(gt(issues.id, afterIssueId));
if (opts?.blockerIssueId) {
@@ -6112,6 +6129,7 @@ export function recoveryService(
// fallback below. Without it, a terminal issue named in the run context
// snapshot would still terminalize the shared run and defeat the guard.
runReferencedByActiveIssue?: boolean;
onDeferredDeadline?: (at: number) => void;
},
): Promise<{ terminalized: boolean; status: string }> {
// Act only on a run in "running" status. A "queued" run has no process yet,
@@ -6127,8 +6145,11 @@ export function recoveryService(
// Another controller may own a sandbox run whose PID has no meaning on
// this host. Its live lease owns both execution and finalization, even if
// the issue is already terminal or this process has no in-memory handle.
if (await hasLiveLegacyController(db, run))
const controllerDeadline = await liveLegacyControllerExpiresAt(db, run);
if (controllerDeadline) {
options?.onDeferredDeadline?.(controllerDeadline.getTime());
return { terminalized: false, status: run.status };
}
const pid = run.processPid ?? null;
const processGroupId = run.processGroupId ?? null;
@@ -6349,7 +6370,7 @@ export function recoveryService(
// still claims to be live but can no longer reach a terminal status on its
// own, so a stuck "running" run can no longer block the sweep. Idempotent and
// safe: clears at most one row's worth of lock columns per candidate.
async function sweepStaleIssueLocks() {
async function sweepStaleIssueLocks(opts?: { companyId?: string; issueId?: string; onDeferredDeadline?: (at: number) => void }) {
const result = {
cleared: 0,
issueIds: [] as string[],
@@ -6366,7 +6387,9 @@ export function recoveryService(
})
.from(issues)
.where(
sql`(${issues.checkoutRunId} is not null or ${issues.executionRunId} is not null)`,
and(sql`(${issues.checkoutRunId} is not null or ${issues.executionRunId} is not null)`,
opts?.companyId ? eq(issues.companyId, opts.companyId) : undefined,
opts?.issueId ? eq(issues.id, opts.issueId) : undefined),
);
const referencedRunIds = [
@@ -6381,7 +6404,8 @@ export function recoveryService(
? await db
.select()
.from(heartbeatRuns)
.where(inArray(heartbeatRuns.id, referencedRunIds))
.where(and(inArray(heartbeatRuns.id, referencedRunIds),
opts?.companyId ? eq(heartbeatRuns.companyId, opts.companyId) : undefined))
: [];
const runStatusById = new Map<string, string>();
for (const row of runRows) runStatusById.set(row.id, row.status);
@@ -6392,8 +6416,15 @@ export function recoveryService(
// the same run id in a stale lock column. The terminal reference alone must
// not terminalize a run that an active issue still owns, so exclude these
// runs from the issue-terminal authority below.
const ownershipReferences = opts?.issueId && referencedRunIds.length > 0
? await db.select({ companyId: issues.companyId, status: issues.status,
checkoutRunId: issues.checkoutRunId, executionRunId: issues.executionRunId }).from(issues).where(and(
opts.companyId ? eq(issues.companyId, opts.companyId) : undefined,
or(inArray(issues.checkoutRunId, referencedRunIds), inArray(issues.executionRunId, referencedRunIds)),
))
: candidates;
const runIdsReferencedByActiveIssue = new Set<string>();
for (const issue of candidates) {
for (const issue of ownershipReferences) {
if (issue.status === "done" || issue.status === "cancelled") continue;
for (const runId of [issue.checkoutRunId, issue.executionRunId]) {
if (runId) runIdsReferencedByActiveIssue.add(runId);
@@ -6434,6 +6465,7 @@ export function recoveryService(
referencingIssueTerminalStatus:
issueTerminalStatusByRunId.get(row.id) ?? null,
runReferencedByActiveIssue: runIdsReferencedByActiveIssue.has(row.id),
onDeferredDeadline: opts?.onDeferredDeadline,
});
runStatusById.set(row.id, outcome.status);
if (outcome.terminalized) result.terminalizedRunIds.push(row.id);
@@ -6537,6 +6569,8 @@ export function recoveryService(
recordWatchdogDecision,
scanSilentActiveRuns,
reconcileStrandedAssignedIssues,
reconcileUnassignedBlockingIssues,
reconcileActiveRecoveryActions,
reconcileLegacyContinuation,
legacyRepairDispatchBlock,
sweepStaleIssueLocks,
server/src/services/recovery/work.ts
+113
View File
@@ -0,0 +1,113 @@
import { registerTaskWatchdogWork } from "../task-watchdog-work.js";
import { and, asc, eq, gt, inArray, isNotNull, isNull, or, sql } from "drizzle-orm";
import { issueRecoveryActions, issueRelations, issues, type Db } from "@paperclipai/db";
import type { createDeliveryWorkCoordinator } from "../delivery-work-coordinator.js";
import { DELIVERY_QUEUES, type DeliveryQueue } from "../delivery-work-notifications.js";
import type { IssueRecoveryScope } from "../issue-recovery-notifications.js";
import { registerScopedDeadlineWork } from "../scoped-deadline-work.js";
type Entry = { id: string; companyId: string; dueAt: number };
type Scope = { companyId: string; issueId: string; onDeferredDeadline: (at: number) => void };
const PAGE_SIZE = 100;
export function registerIssueRecoveryWork(
db: Db,
coordinator: ReturnType<typeof createDeliveryWorkCoordinator>,
owner: {
stranded(scope: Scope): Promise<unknown>;
orphanBlockers(scope: { companyId: string }): Promise<unknown>;
recoveryActions(scope: { companyId: string }): Promise<unknown>;
dependencies(scope: Scope): Promise<{ enqueueFailed: number }>;
locks(scope: Scope): Promise<unknown>;
watchdogs(scope: { companyId: string; watchdogId: string; onDeferredDeadline: (at: number) => void }): Promise<{ incomplete: number }>;
schedulingSuppressed(): Promise<boolean>;
},
canRun: () => boolean,
) {
function register(queue: DeliveryQueue, statuses: string[] | null, check: (scope: Scope) => Promise<unknown>) {
const cursors = new Map<string, string>();
let enabled = false;
return registerScopedDeadlineWork<Entry, IssueRecoveryScope>(coordinator, {
queue,
reconstructionIntervalMs: 30_000,
beforePass: async () => { enabled = !(await owner.schedulingSuppressed()); },
decodeScope: hint => hint === "*" ? undefined : JSON.parse(hint) as IssueRecoveryScope,
matches: (entry, scope) => !scope || (entry.companyId === scope.companyId && (!scope.issueId || entry.id === scope.issueId)),
load: async scope => {
const key = scope ? JSON.stringify([scope.companyId, scope.issueId]) : "*";
if (!enabled) { cursors.delete(key); return []; }
const cursor = cursors.get(key);
const rows = await db.select({ id: issues.id, companyId: issues.companyId }).from(issues).where(and(
scope ? eq(issues.companyId, scope.companyId) : undefined,
scope?.issueId ? eq(issues.id, scope.issueId) : undefined,
cursor ? gt(issues.id, cursor) : undefined,
statuses ? inArray(issues.status, statuses) : or(isNotNull(issues.checkoutRunId), isNotNull(issues.executionRunId)),
statuses ? isNull(issues.hiddenAt) : undefined,
statuses ? isNull(issues.assigneeUserId) : undefined,
statuses ? or(isNotNull(issues.assigneeAgentId), eq(issues.status, "in_review")) : undefined,
)).orderBy(asc(issues.id)).limit(PAGE_SIZE);
const complete = rows.length < PAGE_SIZE;
if (complete) cursors.delete(key); else cursors.set(key, rows.at(-1)!.id);
return { rows: rows.map(row => ({ ...row, dueAt: Date.now() })), complete };
},
check: async entry => {
if (!enabled) return null;
let next: number | null = null;
await check({ companyId: entry.companyId, issueId: entry.id,
onDeferredDeadline: at => { next = next === null ? at : Math.min(next, at); } });
return next === null ? null : { ...entry, dueAt: next };
},
}, canRun);
}
function registerCompany(queue: DeliveryQueue, kind: "blockers" | "actions", check: (scope: { companyId: string }) => Promise<unknown>) {
const cursors = new Map<string, string>();
let enabled = false;
return registerScopedDeadlineWork<Entry, IssueRecoveryScope>(coordinator, {
queue,
reconstructionIntervalMs: 30_000,
beforePass: async () => { enabled = !(await owner.schedulingSuppressed()); },
decodeScope: hint => hint === "*" ? undefined : JSON.parse(hint) as IssueRecoveryScope,
matches: (entry, scope) => !scope || entry.companyId === scope.companyId,
load: async scope => {
const key = scope?.companyId ?? "*";
if (!enabled) { cursors.delete(key); return []; }
const cursor = cursors.get(key);
const rows = kind === "actions"
? await db.selectDistinct({ id: issueRecoveryActions.companyId, companyId: issueRecoveryActions.companyId })
.from(issueRecoveryActions).where(and(inArray(issueRecoveryActions.status, ["active", "escalated"]),
scope ? eq(issueRecoveryActions.companyId, scope.companyId) : undefined,
cursor ? gt(issueRecoveryActions.companyId, cursor) : undefined))
.orderBy(asc(issueRecoveryActions.companyId)).limit(PAGE_SIZE)
: await db.selectDistinct({ id: issues.companyId, companyId: issues.companyId }).from(issues).where(and(
inArray(issues.status, ["todo", "blocked"]), isNull(issues.assigneeAgentId), isNull(issues.assigneeUserId),
isNotNull(issues.createdByAgentId),
scope ? eq(issues.companyId, scope.companyId) : undefined,
cursor ? gt(issues.companyId, cursor) : undefined,
sql`exists (select 1 from ${issueRelations} r inner join ${issues} dependent
on dependent.id = r.related_issue_id and dependent.company_id = r.company_id
where r.company_id = ${issues.companyId} and r.issue_id = ${issues.id}
and r.type = 'blocks' and dependent.status not in ('done', 'cancelled'))`,
)).orderBy(asc(issues.companyId)).limit(PAGE_SIZE);
const complete = rows.length < PAGE_SIZE;
if (complete) cursors.delete(key); else cursors.set(key, rows.at(-1)!.id);
return { rows: rows.map(row => ({ ...row, dueAt: Date.now() })), complete };
},
check: async entry => { if (enabled) await check({ companyId: entry.companyId }); return null; },
}, canRun);
}
const assignments = register(DELIVERY_QUEUES.strandedAssignments, ["todo", "in_progress", "in_review"], owner.stranded);
const orphanBlockers = registerCompany(DELIVERY_QUEUES.orphanBlockers, "blockers", owner.orphanBlockers);
const recoveryActions = registerCompany(DELIVERY_QUEUES.activeRecoveryActions, "actions", owner.recoveryActions);
const dependencies = register(DELIVERY_QUEUES.resolvedDependencies, ["blocked"], async scope => {
const result = await owner.dependencies(scope);
if (result.enqueueFailed > 0) throw new Error("Resolved dependency wake remains undelivered");
});
const locks = register(DELIVERY_QUEUES.staleIssueLocks, null, owner.locks);
const watchdogs = registerTaskWatchdogWork(db, coordinator, {
reconcile: owner.watchdogs, enabled: async () => !(await owner.schedulingSuppressed()),
}, canRun);
const workers = [assignments, orphanBlockers, recoveryActions, dependencies, locks, watchdogs];
return { ready: Promise.all(workers.map(worker => worker.ready)),
wake() { for (const worker of workers) worker.invalidate(); },
async stop() { await Promise.all(workers.map(worker => worker.stop())); } };
}
server/src/services/run-dispatch-notifications.ts
+11
View File
@@ -0,0 +1,11 @@
import { notifyIssueRecovery } from "./issue-recovery-notifications.js";
import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js";
export type RunDispatchScope = { companyId: string };
export async function notifyRunDispatch(transaction: object, scope?: RunDispatchScope) {
await notifyIssueRecovery(transaction, scope ? { companyId: scope.companyId } : undefined);
const hint = scope ? JSON.stringify({ companyId: scope.companyId }) : "*";
await notifyDeliveryWork(transaction, DELIVERY_QUEUES.scheduledRetryDispatch, hint);
await notifyDeliveryWork(transaction, DELIVERY_QUEUES.queuedRunDispatch, hint);
}
server/src/services/sandbox-absence-proof.test.ts
+47 -5
View File
@@ -1,6 +1,7 @@
import { createHmac, generateKeyPairSync, randomUUID, sign } from "node:crypto";
import { describe, expect, it, vi } from "vitest";
import { requestSandboxAbsenceProof, verifySandboxAbsenceReceipt } from "./sandbox-absence-proof.js";
import { inspectCloudSandboxCleanupAbsence, requestSandboxAbsenceProof, verifySandboxAbsenceReceipt,
verifySandboxCleanupAbsenceReceipt } from "./sandbox-absence-proof.js";
const { privateKey, publicKey } = generateKeyPairSync("ed25519");
const binding = { companyId: randomUUID(), issueId: randomUUID(), runId: randomUUID(), leaseId: randomUUID(),
@@ -9,10 +10,10 @@ const env = { PAPERCLIP_CLOUD_STACK_ID: "fixture-stack", PAPERCLIP_CLOUD_API_ORI
PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN: "fleet-token", PAPERCLIP_PRIMARY_HOST: "fixture.paperclip.test", PAPERCLIP_AUTH_SECRET: "fixture-secret",
PAPERCLIP_CLOUD_RUNTIME_IDENTITY_JWKS: JSON.stringify({ keys: [{ ...publicKey.export({ format: "jwk" }), kid: "fixture", alg: "EdDSA", use: "sig" }] }) };
const now = new Date("2099-01-01T00:00:00Z"), seconds = now.getTime() / 1000;
function receipt(overrides: Record<string, unknown> = {}) {
const header = Buffer.from(JSON.stringify({ alg: "EdDSA", kid: "fixture", typ: "paperclip-sandbox-absence+jwt" })).toString("base64url");
const payload = Buffer.from(JSON.stringify({ v: 1, iss: "paperclip-cloud", aud: "paperclip-sandbox-absence/v1", sub: "fixture-stack",
...binding, absent: true, iat: seconds, exp: seconds + 60, ...overrides })).toString("base64url");
function receipt(overrides: Record<string, unknown> = {}, purpose = "sandbox-absence", source: Record<string, unknown> = binding) {
const header = Buffer.from(JSON.stringify({ alg: "EdDSA", kid: "fixture", typ: `paperclip-${purpose}+jwt` })).toString("base64url");
const payload = Buffer.from(JSON.stringify({ v: 1, iss: "paperclip-cloud", aud: `paperclip-${purpose}/v1`, sub: "fixture-stack",
...source, absent: true, iat: seconds, exp: seconds + 60, ...overrides })).toString("base64url");
return `${header}.${payload}.${sign(null, Buffer.from(`${header}.${payload}`), privateKey).toString("base64url")}`;
}
describe("sandbox absence proof", () => {
@@ -45,3 +46,44 @@ describe("sandbox absence proof", () => {
expect(fetchImpl).not.toHaveBeenCalled();
});
});
describe("cleanup-only sandbox absence", () => {
const cleanup = { purpose: "cleanup" as const, companyId: binding.companyId, environmentId: randomUUID(),
leaseId: binding.leaseId, heartbeatRunId: null as string | null, provider: binding.provider, providerLeaseId: binding.providerLeaseId };
const lease = { id: cleanup.leaseId, ...cleanup };
const signed = (overrides: Record<string, unknown> = {}) => receipt(overrides, "sandbox-cleanup-absence", cleanup);
it.each([null, binding.runId])("accepts exact absence with nullable run %s without issuing a run receipt", async heartbeatRunId => {
const source = { ...cleanup, heartbeatRunId };
const fetchImpl = vi.fn().mockResolvedValue(Response.json({ receipt: receipt({}, "sandbox-cleanup-absence", source) }));
expect(await inspectCloudSandboxCleanupAbsence({ ...lease, heartbeatRunId }, { env, now, fetchImpl }))
.toEqual({ providerLeaseId: cleanup.providerLeaseId, state: "destroyed" });
const [url, init] = fetchImpl.mock.calls[0];
expect(url).toBe("https://cloud.example.test/v1/tenant/sandbox-absence");
expect(JSON.parse(init.body)).toEqual(source);
expect(init.headers.authorization).not.toContain(env.PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN);
expect(init.redirect).toBe("error");
});
it("requires every binding, current stack, signature and freshness, and rejects recovery receipts", () => {
expect(verifySandboxCleanupAbsenceReceipt(signed(), cleanup, env, now)).toMatchObject(cleanup);
for (const overrides of [
...Object.keys(cleanup).map(key => ({ [key]: randomUUID() })),
{ sub: "foreign-stack" }, { absent: false }, { aud: "paperclip-sandbox-absence/v1" }, { iss: "other" },
{ iat: seconds + 31 }, { exp: seconds }, { exp: seconds + 61 },
]) expect(verifySandboxCleanupAbsenceReceipt(signed(overrides), cleanup, env, now)).toBeNull();
expect(verifySandboxCleanupAbsenceReceipt(signed().slice(0, -8) + "tampered", cleanup, env, now)).toBeNull();
expect(verifySandboxCleanupAbsenceReceipt(receipt(), cleanup, env, now)).toBeNull();
expect(verifySandboxAbsenceReceipt(signed(), binding, env, now)).toBeNull();
});
it("keeps old Clouds, outages, invalid replies and unconfigured instances inconclusive", async () => {
for (const answer of [{ absent: false }, { absent: true }, { receipt: "forged" }, { receipt: signed({ heartbeatRunId: binding.runId }) }])
expect(await inspectCloudSandboxCleanupAbsence(lease, { env, now, fetchImpl: vi.fn().mockResolvedValue(Response.json(answer)) })).toBeUndefined();
for (const status of [400, 401, 403, 503]) expect(await inspectCloudSandboxCleanupAbsence(lease,
{ env, now, fetchImpl: vi.fn().mockResolvedValue(new Response(null, { status })) })).toBeUndefined();
expect(await inspectCloudSandboxCleanupAbsence(lease, { env, now, fetchImpl: vi.fn().mockRejectedValue(new Error("offline")) })).toBeUndefined();
const fetchImpl = vi.fn();
expect(await inspectCloudSandboxCleanupAbsence(lease, { env: {}, now, fetchImpl })).toBeUndefined();
for (const change of [{ provider: "other" }, { environmentId: null }, { providerLeaseId: "invalid" }])
expect(await inspectCloudSandboxCleanupAbsence({ ...lease, ...change }, { env, now, fetchImpl })).toBeUndefined();
expect(fetchImpl).not.toHaveBeenCalled();
});
});
server/src/services/sandbox-absence-proof.ts
+56 -12
View File
@@ -7,8 +7,18 @@ const bindingSchema = z.object({
companyId: z.string().uuid(), issueId: z.string().uuid(), runId: z.string().uuid(),
leaseId: z.string().uuid(), ownerToken: z.string().uuid(), provider: z.literal("daytona"),
providerLeaseId: z.string().uuid(),
});
}).strict();
const cleanupBindingSchema = z.object({
purpose: z.literal("cleanup"), companyId: z.string().uuid(), environmentId: z.string().uuid(),
leaseId: z.string().uuid(), heartbeatRunId: z.string().uuid().nullable(),
provider: z.literal("daytona"), providerLeaseId: z.string().uuid(),
}).strict();
export type SandboxAbsenceBinding = z.infer<typeof bindingSchema>;
export type SandboxCleanupAbsenceBinding = z.infer<typeof cleanupBindingSchema>;
export type SandboxCleanupAbsenceProof = SandboxCleanupAbsenceBinding & {
v: 1; iss: "paperclip-cloud"; aud: "paperclip-sandbox-cleanup-absence/v1"; sub: string;
absent: true; iat: number; exp: number;
};
export type SandboxAbsenceProof = SandboxAbsenceBinding & {
v: 1; iss: "paperclip-cloud"; aud: "paperclip-sandbox-absence/v1"; sub: string;
absent: true; iat: number; exp: number;
@@ -16,19 +26,30 @@ export type SandboxAbsenceProof = SandboxAbsenceBinding & {
export function verifySandboxAbsenceReceipt(receipt: string, binding: SandboxAbsenceBinding,
env: NodeJS.ProcessEnv = process.env, now = new Date()): SandboxAbsenceProof | null {
const parsed = bindingSchema.safeParse(binding);
return parsed.success ? verifyReceipt(receipt, parsed.data, "sandbox-absence", env, now) as SandboxAbsenceProof | null : null;
}
export function verifySandboxCleanupAbsenceReceipt(receipt: string, binding: SandboxCleanupAbsenceBinding,
env: NodeJS.ProcessEnv = process.env, now = new Date()): SandboxCleanupAbsenceProof | null {
const parsed = cleanupBindingSchema.safeParse(binding);
return parsed.success ? verifyReceipt(receipt, parsed.data, "sandbox-cleanup-absence", env, now) as SandboxCleanupAbsenceProof | null : null;
}
function verifyReceipt(receipt: string, binding: Record<string, unknown>, purpose: string,
env: NodeJS.ProcessEnv, now: Date): Record<string, unknown> | null {
try {
const parsed = bindingSchema.safeParse(binding);
if (receipt.length > 16_000 || !parsed.success) return null;
if (receipt.length > 16_000) return null;
const parts = receipt.split(".");
if (parts.length !== 3 || parts.some(part => !/^[A-Za-z0-9_-]+$/.test(part))) return null;
const header = JSON.parse(Buffer.from(parts[0], "base64url").toString("utf8"));
if (header.alg !== "EdDSA" || header.typ !== "paperclip-sandbox-absence+jwt" || typeof header.kid !== "string") return null;
if (header.alg !== "EdDSA" || header.typ !== `paperclip-${purpose}+jwt` || typeof header.kid !== "string") return null;
if (!verify(null, Buffer.from(`${parts[0]}.${parts[1]}`, "ascii"), cloudRuntimePublicKeyForKid(env, header.kid), Buffer.from(parts[2], "base64url"))) return null;
const proof = JSON.parse(Buffer.from(parts[1], "base64url").toString("utf8")) as SandboxAbsenceProof;
const proof = JSON.parse(Buffer.from(parts[1], "base64url").toString("utf8"));
const seconds = Math.floor(now.getTime() / 1000);
if (proof.v !== 1 || proof.iss !== "paperclip-cloud" || proof.aud !== "paperclip-sandbox-absence/v1"
if (proof.v !== 1 || proof.iss !== "paperclip-cloud" || proof.aud !== `paperclip-${purpose}/v1`
|| !env.PAPERCLIP_CLOUD_STACK_ID || proof.sub !== env.PAPERCLIP_CLOUD_STACK_ID || proof.absent !== true
|| Object.keys(parsed.data).some(key => proof[key as keyof SandboxAbsenceBinding] !== parsed.data[key as keyof SandboxAbsenceBinding])
|| Object.keys(binding).some(key => proof[key] !== binding[key])
|| !Number.isInteger(proof.iat) || !Number.isInteger(proof.exp) || proof.iat > seconds + 30
|| proof.exp <= seconds || proof.exp <= proof.iat || proof.exp - proof.iat > 60) return null;
return proof;
@@ -38,11 +59,35 @@ export function verifySandboxAbsenceReceipt(receipt: string, binding: SandboxAbs
export async function requestSandboxAbsenceProof(binding: SandboxAbsenceBinding, options: {
env?: NodeJS.ProcessEnv; fetchImpl?: typeof fetch; now?: Date;
} = {}): Promise<{ proof: SandboxAbsenceProof; receipt: string } | null> {
const parsed = bindingSchema.safeParse(binding);
if (!parsed.success) return null;
const receipt = await requestReceipt(parsed.data, options);
if (!receipt) return null;
const proof = verifySandboxAbsenceReceipt(receipt, parsed.data, options.env, options.now);
return proof ? { proof, receipt } : null;
}
/** Cleanup-only observation: never grants continuation authority to a run. */
export async function inspectCloudSandboxCleanupAbsence(lease: {
id: string; companyId: string; environmentId: string | null; heartbeatRunId: string | null;
provider: string | null; providerLeaseId: string | null;
}, options: { env?: NodeJS.ProcessEnv; fetchImpl?: typeof fetch; now?: Date } = {}) {
const parsed = cleanupBindingSchema.safeParse({ purpose: "cleanup", companyId: lease.companyId,
environmentId: lease.environmentId, leaseId: lease.id, heartbeatRunId: lease.heartbeatRunId,
provider: lease.provider, providerLeaseId: lease.providerLeaseId });
if (!parsed.success) return undefined;
const receipt = await requestReceipt(parsed.data, options);
if (!receipt || !verifySandboxCleanupAbsenceReceipt(receipt, parsed.data, options.env, options.now)) return undefined;
return { providerLeaseId: parsed.data.providerLeaseId, state: "destroyed" as const };
}
async function requestReceipt(binding: Record<string, unknown>, options: {
env?: NodeJS.ProcessEnv; fetchImpl?: typeof fetch;
}): Promise<string | null> {
const env = options.env ?? process.env;
const context = getCloudStackContext(env);
const secret = env.PAPERCLIP_AUTH_SECRET ?? env.BETTER_AUTH_SECRET;
const parsed = bindingSchema.safeParse(binding);
if (!context?.stackId || !context.primaryHost || !context.cloudOrigin || !secret?.trim() || !parsed.success) return null;
if (!context?.stackId || !context.primaryHost || !context.cloudOrigin || !secret?.trim()) return null;
try {
const origin = new URL(context.cloudOrigin);
const host = new URL(`https://${context.primaryHost}`);
@@ -51,12 +96,11 @@ export async function requestSandboxAbsenceProof(binding: SandboxAbsenceBinding,
const response = await (options.fetchImpl ?? fetch)(`${origin.origin}/v1/tenant/sandbox-absence`, {
method: "POST", redirect: "error", signal: AbortSignal.timeout(10_000),
headers: { authorization: `Bearer ${capability}`, "x-paperclip-cloud-tenant-host": host.hostname, "content-type": "application/json" },
body: JSON.stringify(parsed.data),
body: JSON.stringify(binding),
});
if (!response.ok) return null;
const payload = await response.json() as { receipt?: unknown };
if (typeof payload.receipt !== "string") return null;
const proof = verifySandboxAbsenceReceipt(payload.receipt, binding, env, options.now);
return proof ? { proof, receipt: payload.receipt } : null;
return payload.receipt;
} catch { return null; }
}
server/src/services/scheduled-retry-cancellation-notifications.ts
+2
View File
@@ -1,7 +1,9 @@
import { notifyRunDispatch } from "./run-dispatch-notifications.js";
import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js";
export type ScheduledRetryCancellationScope = { companyId: string; runId?: string };
export async function notifyScheduledRetryCancellation(transaction: object, scope?: ScheduledRetryCancellationScope) {
await notifyRunDispatch(transaction, scope);
await notifyDeliveryWork(transaction, DELIVERY_QUEUES.scheduledRetryCancellation, scope ? JSON.stringify(scope) : "*");
}
server/src/services/scoped-deadline-work.ts
+18 -2
View File
@@ -12,6 +12,8 @@ export function registerScopedDeadlineWork<Entry extends ScheduledEntry, Scope>(
queue: DeliveryQueue;
batchSize?: number;
minimumPassIntervalMs?: number;
reconstructionIntervalMs?: number;
beforePass?(): Promise<void>;
decodeScope(hint: string): Scope | undefined;
matches(entry: Entry, scope?: Scope): boolean;
load(scope?: Scope, signal?: AbortSignal): Promise<Entry[] | { rows: Entry[] | null; complete: boolean }>;
@@ -25,6 +27,7 @@ export function registerScopedDeadlineWork<Entry extends ScheduledEntry, Scope>(
const entries = new Map<string, Entry>();
const failures = new Map<string, { count: number; retryAt: number }>();
const scopeFailures = new Map<string, { count: number; retryAt: number }>();
const reconstructAfter = new Map<string, number>();
let earliest = 0;
let lastCompany: string | null = null;
function failed(map: typeof failures, id: string) {
@@ -34,13 +37,14 @@ export function registerScopedDeadlineWork<Entry extends ScheduledEntry, Scope>(
function dirtyScopes() {
return [...dirty].filter(hint => hint !== "*" || !scopeFailures.has(hint))
.filter(hint => !scans.has(hint) || scopeFailures.has(hint))
.filter(hint => (reconstructAfter.get(hint) ?? 0) <= Date.now() || scopeFailures.has(hint))
.map(hint => { try { return options.decodeScope(hint); } catch { return undefined; } });
}
function next() {
if (!canRun()) return dirty.size > 0 || entries.size > 0 ? Date.now() + 30_000 : null;
let nextAt: number | null = null;
const take = (at: number) => { nextAt = nextAt === null ? at : Math.min(nextAt, at); };
for (const hint of dirty) take(Math.max(earliest, scopeFailures.get(hint)?.retryAt ?? 0));
for (const hint of dirty) take(Math.max(earliest, reconstructAfter.get(hint) ?? 0, scopeFailures.get(hint)?.retryAt ?? 0));
const unavailable = dirtyScopes();
for (const entry of entries.values()) if (!unavailable.some(scope => options.matches(entry, scope)))
take(Math.max(earliest, entry.dueAt, failures.get(entry.id)?.retryAt ?? 0));
@@ -54,9 +58,18 @@ export function registerScopedDeadlineWork<Entry extends ScheduledEntry, Scope>(
} },
run: async signal => {
if (!canRun() || Date.now() < earliest) return;
if ((options.reconstructionIntervalMs ?? 0) > 0) for (const [hint, until] of reconstructAfter) {
if (until <= Date.now()) reconstructAfter.delete(hint);
}
const ready = [...dirty].filter(hint => Math.max(reconstructAfter.get(hint) ?? 0, scopeFailures.get(hint)?.retryAt ?? 0) <= Date.now());
if (ready.length === 0 && ![...entries.values()].some(entry => Math.max(entry.dueAt, failures.get(entry.id)?.retryAt ?? 0) <= Date.now())) return;
earliest = Date.now() + (options.minimumPassIntervalMs ?? 1000);
try { await options.beforePass?.(); } catch (error) {
for (const hint of ready) failed(scopeFailures, hint);
for (const entry of entries.values()) if (entry.dueAt <= Date.now()) failed(failures, entry.id);
throw error;
}
const errors: unknown[] = [];
const ready = [...dirty].filter(hint => (scopeFailures.get(hint)?.retryAt ?? 0) <= Date.now());
const requested = ready.includes("*")
? ["*", ...ready.filter(hint => hint !== "*")].slice(0, 25)
: ready.slice(0, 25);
@@ -81,6 +94,9 @@ export function registerScopedDeadlineWork<Entry extends ScheduledEntry, Scope>(
entries.delete(entry.id); failures.delete(entry.id);
}
scans.delete(hint);
if ((options.reconstructionIntervalMs ?? 0) > 0) {
reconstructAfter.set(hint, Date.now() + options.reconstructionIntervalMs!);
}
if ((versions.get(hint) ?? 0) !== scan.version) dirty.add(hint); else versions.delete(hint);
if (!scope) scopeFailures.clear(); else scopeFailures.delete(hint);
} catch (error) {
server/src/services/system-recovery.ts
+2
View File
@@ -1,3 +1,4 @@
import { notifyIssueRecovery } from "./issue-recovery-notifications.js";
import { and, eq, sql } from "drizzle-orm";
import { heartbeatRuns, issueComments, issueRecoveryActions, type Db } from "@paperclipai/db";
import { writeExecutionControl } from "./execution-control-notifications.js";
@@ -11,6 +12,7 @@ export async function adoptSystemRecovery(db: Db, source: WorkspaceRecoverySourc
.where(and(eq(issueRecoveryActions.companyId, source.companyId), eq(issueRecoveryActions.id, source.actionId),
eq(issueRecoveryActions.sourceIssueId, source.issueId))).for("update");
if (!existing || existing.evidence.systemRecovery) return;
await notifyIssueRecovery(tx, { companyId: source.companyId });
await notifyDeliveryWork(tx, DELIVERY_QUEUES.workspaceRecovery);
const state = createSystemRecovery(source.runId);
// Atomic, once per source. An exhausted previous execution budget is history;
server/src/services/task-retry.ts
+4 -2
View File
@@ -339,8 +339,10 @@ export async function requestTaskRetry(input: {
/** Chat owns admission and authorization. The task receipt follows that
* durable outcome; it never launches a parallel generic board execution. */
export async function reconcileTaskChatRetryReceipts(db: Db) {
export async function reconcileTaskChatRetryReceipts(db: Db, scope: { companyId?: string; requestId?: string; settleOnly?: boolean } = {}) {
const pending = await db.select().from(agentWakeupRequests).where(and(
scope.companyId ? eq(agentWakeupRequests.companyId, scope.companyId) : undefined,
scope.requestId ? eq(agentWakeupRequests.id, scope.requestId) : undefined,
eq(agentWakeupRequests.reason, "task_retry"), eq(agentWakeupRequests.status, "deferred_issue_execution"),
sql`${agentWakeupRequests.payload}#>>'{taskRetry,chatActionId}' is not null`,
)).orderBy(agentWakeupRequests.updatedAt).limit(50);
@@ -356,7 +358,7 @@ export async function reconcileTaskChatRetryReceipts(db: Db) {
// A deferred chat receipt is admitted input, not a completed dispatch.
// Re-stage that exact action through chat's owner; it rechecks authority
// and normal queue policy before updating the existing receipt.
if (chat?.status === "processed" && chat.payload.taskRetryIntentId === wake.id &&
if (!scope.settleOnly && chat?.status === "processed" && chat.payload.taskRetryIntentId === wake.id &&
chat.payload.failedRunId === intent.sourceRunId && receipt?.status === "deferred_issue_execution" &&
!receipt.runId && receipt.payload?.taskRetryRequestId === wake.id) {
await db.transaction(async tx => {
server/src/services/task-watchdog-work.ts
+49
View File
@@ -0,0 +1,49 @@
import { and, asc, eq, gt } from "drizzle-orm";
import { issueWatchdogs, type Db } from "@paperclipai/db";
import type { createDeliveryWorkCoordinator } from "./delivery-work-coordinator.js";
import { DELIVERY_QUEUES } from "./delivery-work-notifications.js";
import type { IssueRecoveryScope } from "./issue-recovery-notifications.js";
import { registerScopedDeadlineWork } from "./scoped-deadline-work.js";
type Entry = { id: string; companyId: string; dueAt: number };
export function registerTaskWatchdogWork(
db: Db,
coordinator: ReturnType<typeof createDeliveryWorkCoordinator>,
owner: {
enabled(): Promise<boolean>;
reconcile(scope: { companyId: string; watchdogId: string; onDeferredDeadline: (at: number) => void }): Promise<{ incomplete: number }>;
},
canRun: () => boolean,
) {
const cursors = new Map<string, string>();
let enabled = false;
return registerScopedDeadlineWork<Entry, IssueRecoveryScope>(coordinator, {
queue: DELIVERY_QUEUES.taskWatchdogs,
reconstructionIntervalMs: 30_000,
beforePass: async () => { enabled = await owner.enabled(); },
decodeScope: hint => hint === "*" ? undefined : JSON.parse(hint) as IssueRecoveryScope,
matches: (entry, scope) => !scope || entry.companyId === scope.companyId,
load: async scope => {
const key = scope?.companyId ?? "*";
if (!enabled) { cursors.delete(key); return []; }
const cursor = cursors.get(key);
const rows = await db.select({ id: issueWatchdogs.id, companyId: issueWatchdogs.companyId })
.from(issueWatchdogs).where(and(eq(issueWatchdogs.status, "active"),
scope ? eq(issueWatchdogs.companyId, scope.companyId) : undefined,
cursor ? gt(issueWatchdogs.id, cursor) : undefined))
.orderBy(asc(issueWatchdogs.id)).limit(100);
const complete = rows.length < 100;
if (complete) cursors.delete(key); else cursors.set(key, rows.at(-1)!.id);
return { rows: rows.map(row => ({ ...row, dueAt: Date.now() })), complete };
},
check: async entry => {
if (!enabled) return null;
let next: number | null = null;
const result = await owner.reconcile({ companyId: entry.companyId, watchdogId: entry.id,
onDeferredDeadline: at => { next = next === null ? at : Math.min(next, at); } });
if (result.incomplete > 0) throw new Error(`Task watchdog ${entry.id} in company ${entry.companyId} has an incomplete subtree`);
return next === null ? null : { ...entry, dueAt: next };
},
}, canRun);
}
server/src/services/task-watchdogs.ts
+34 -10
View File
@@ -1,4 +1,5 @@
import { notifyStatusCardIssueChange } from "./status-card-notifications.js";
import { notifyIssueRecovery } from "./issue-recovery-notifications.js";
import { createHash } from "node:crypto";
import { and, asc, eq, gte, inArray, isNull, or, sql } from "drizzle-orm";
import type { Db } from "@paperclipai/db";
@@ -832,7 +833,7 @@ async function updateIssueWatchdogRow(
return updated;
}
export async function upsertIssueWatchdogForIssue(
async function upsertIssueWatchdogInTransaction(
dbOrTx: any,
companyId: string,
issueId: string,
@@ -848,12 +849,13 @@ export async function upsertIssueWatchdogForIssue(
.where(and(eq(issueWatchdogs.companyId, companyId), eq(issueWatchdogs.issueId, issueId)))
.then((rows: IssueWatchdogRow[]) => rows[0] ?? null);
await notifyIssueRecovery(dbOrTx, { companyId });
if (existing) {
const updated = await updateIssueWatchdogRow(dbOrTx, existing, input, now);
return { watchdog: toIssueWatchdog(updated), created: false };
}
const insertResult: { row: IssueWatchdogRow; created: boolean } = await dbOrTx
const insertResult: { row: IssueWatchdogRow; created: boolean } = await dbOrTx.transaction(async (tx: Db) => tx
.insert(issueWatchdogs)
.values({
companyId,
@@ -871,7 +873,7 @@ export async function upsertIssueWatchdogForIssue(
updatedAt: now,
})
.returning()
.then((rows: IssueWatchdogRow[]) => ({ row: rows[0], created: true }))
.then((rows: IssueWatchdogRow[]) => ({ row: rows[0], created: true })))
.catch(async (error: unknown) => {
if (!isIssueWatchdogUniqueConflict(error)) throw error;
const winner = await dbOrTx
@@ -886,6 +888,15 @@ export async function upsertIssueWatchdogForIssue(
return { watchdog: toIssueWatchdog(insertResult.row), created: insertResult.created };
}
export function upsertIssueWatchdogForIssue(
dbOrTx: Db | Parameters<Parameters<Db["transaction"]>[0]>[0],
companyId: string,
issueId: string,
input: IssueWatchdogUpsertInput,
): Promise<{ watchdog: IssueWatchdog; created: boolean }> {
return dbOrTx.transaction(tx => upsertIssueWatchdogInTransaction(tx, companyId, issueId, input));
}
export function taskWatchdogService(db: Db, deps: TaskWatchdogServiceDeps = {}) {
const issuesSvc = issueService(db);
@@ -1482,7 +1493,7 @@ export function taskWatchdogService(db: Db, deps: TaskWatchdogServiceDeps = {})
return created;
}
async function evaluateWatchdog(row: IssueWatchdogRow, opts: { runId?: string | null } = {}) {
async function evaluateWatchdog(row: IssueWatchdogRow, opts: { runId?: string | null; onDeferredDeadline?: (at: number) => void } = {}) {
const watchdog = await markTerminalWatchdogIssueReviewed(row, opts);
const sourceIssue = await db
.select()
@@ -1495,6 +1506,13 @@ export function taskWatchdogService(db: Db, deps: TaskWatchdogServiceDeps = {})
const input = await collectClassifierInput(watchdog.companyId, watchdog);
const classification = classifyTaskWatchdogSubtree(input);
if (classification.state === "pending_first_run") {
const pending = new Set(classification.pendingIssueIds);
for (const issue of input.issues) if (pending.has(issue.id)) {
const createdAt = toEpochMs(issue.createdAt);
if (createdAt !== null) opts.onDeferredDeadline?.(createdAt + TASK_WATCHDOG_FIRST_RUN_GRACE_MS);
}
}
if (classification.state !== "stopped") {
return { state: classification.state, reason: classification.reason, classification };
}
@@ -1614,13 +1632,14 @@ export function taskWatchdogService(db: Db, deps: TaskWatchdogServiceDeps = {})
};
}
async function listActiveWatchdogsForCompany(companyId?: string | null) {
async function listActiveWatchdogsForCompany(companyId?: string | null, watchdogId?: string) {
return db
.select()
.from(issueWatchdogs)
.where(and(
eq(issueWatchdogs.status, "active"),
...(companyId ? [eq(issueWatchdogs.companyId, companyId)] : []),
watchdogId ? eq(issueWatchdogs.id, watchdogId) : undefined,
));
}
@@ -1750,14 +1769,16 @@ export function taskWatchdogService(db: Db, deps: TaskWatchdogServiceDeps = {})
issueId: string,
actor: ActorFields = {},
): Promise<IssueWatchdog | null> => {
await assertWatchedIssue(db, companyId, issueId);
const existing = await db
return db.transaction(async tx => {
await assertWatchedIssue(tx, companyId, issueId);
const existing = await tx
.select()
.from(issueWatchdogs)
.where(and(eq(issueWatchdogs.companyId, companyId), eq(issueWatchdogs.issueId, issueId)))
.then((rows) => rows[0] ?? null);
if (!existing || existing.status === "disabled") return null;
const [updated] = await db
await notifyIssueRecovery(tx, { companyId });
const [updated] = await tx
.update(issueWatchdogs)
.set({
status: "disabled",
@@ -1769,14 +1790,17 @@ export function taskWatchdogService(db: Db, deps: TaskWatchdogServiceDeps = {})
.where(eq(issueWatchdogs.id, existing.id))
.returning();
return toIssueWatchdog(updated);
});
},
reconcileTaskWatchdogs: async (opts: {
companyId?: string | null;
runId?: string | null;
watchdogId?: string;
onDeferredDeadline?: (at: number) => void;
issueCreatedAtGte?: Date | null;
} = {}) => {
let rows = await listActiveWatchdogsForCompany(opts.companyId ?? null);
let rows = await listActiveWatchdogsForCompany(opts.companyId ?? null, opts.watchdogId);
if (opts.issueCreatedAtGte) {
const watchdogIssueIds = [...new Set(rows.map((row) => row.issueId))];
const eligibleIssueIds = new Set(
@@ -1806,7 +1830,7 @@ export function taskWatchdogService(db: Db, deps: TaskWatchdogServiceDeps = {})
};
for (const row of rows) {
result.checked += 1;
const evaluated = await evaluateWatchdog(row, { runId: opts.runId ?? null });
const evaluated = await evaluateWatchdog(row, { runId: opts.runId ?? null, onDeferredDeadline: opts.onDeferredDeadline });
if (evaluated.state === "triggered") {
result.triggered += 1;
result.watchdogIssueIds.push(evaluated.watchdogIssueId);