merge: preserve routine projections across retained execution producers

Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
DottaandPaperclip committed 2026-10-30 20:26:14 -05:00
commit 40bd4df17b
201 files changed
+8576 -1103

No files matched your search

+4
View File
@@ -1,3 +1,5 @@
import { notifyStatusCardIssueChange } from "./status-card-notifications.js";
import { notifyDeferredExecutionEligibility } from "./deferred-execution-notifications.js";
import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js";
import { notifyNativeStatusReassessment } from "./native-status-reassessment-notifications.js";
import { notifyUnsafeWorkspaceExportRecovery } from "./unsafe-workspace-export-notifications.js";
@@ -743,8 +745,10 @@ export function accessService(db: Db) {
sql`${issues.status} not in ('done', 'cancelled')`,
);
await notifyNativeStatusReassessment(tx, { companyId });
await notifyDeferredExecutionEligibility(tx, { companyId });
await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors);
await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers);
await notifyStatusCardIssueChange(tx, companyId);
const resetInProgress = await tx
.update(issues)
.set({
@@ -1,3 +1,4 @@
import { notifyAttentionRetentionWork } from "./retention-work-notifications.js";
import { and, eq, sql } from "drizzle-orm";
import { agents, agentRuntimeState, companies, heartbeatRuns, type Db } from "@paperclipai/db";
import { normalizeCents, usdToCents, type AccountingFinding, type AccountingHealth, type AccountingInspection } from "@paperclipai/shared";
@@ -75,6 +76,8 @@ export function accountingIntegrityService(db: Db, hooks: BudgetServiceHooks = {
if (!reason.trim()) throw conflict("A repair reason is required");
const before = await inspect(tx, companyId);
if (before.fingerprint !== fingerprint) throw conflict("Accounting changed since inspection; inspect again before repairing");
if (before.findings.some(f => f.repairable && f.kind === "agent_projection"))
await notifyAttentionRetentionWork(tx, { companyId, sourceKind: "agent_error_alert" });
for (const finding of before.findings.filter(f => f.repairable)) {
if (finding.kind === "company_projection") await tx.update(companies).set({ spentMonthlyCents: sql`${finding.expected.cents}::numeric`, updatedAt: new Date() }).where(eq(companies.id, companyId));
if (finding.kind === "agent_projection") await tx.update(agents).set({ spentMonthlyCents: sql`${finding.expected.cents}::numeric`, updatedAt: new Date() }).where(and(eq(agents.id, finding.entityId), eq(agents.companyId, companyId)));
@@ -1,3 +1,6 @@
import { notifyAttentionRetentionWork } from "./retention-work-notifications.js";
import { notifyDeferredExecutionEligibility } from "./deferred-execution-notifications.js";
import { notifyRunDispatch } from "./run-dispatch-notifications.js";
import { notifySkillVersionPinChange } from "./skill-version-retention-notifications.js";
import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js";
import { notifyExternalConversationSettlement } from "./external-conversation-settlement-notifications.js";
@@ -16,6 +19,7 @@ import { clearPrimaryAgent } from "./primary-agent.js";
import { budgetServiceInTransaction } from "./budgets.js";
export async function prepareAgentConfiguration(db: Db, existing: AgentConfigurationRecord, data: AgentConfigurationPatch, options?: UpdateAgentOptions): Promise<AgentConfigurationPatch> {
await notifyAttentionRetentionWork(db, { companyId: existing.companyId });
const id = existing.id;
const { ensureManager, assertNoCycle, assertCompanyShortnameAvailable, assertBuiltInAgentMetadataMutationAllowed } = agentRecordQueries(db);
if (existing.status === "pending_approval" && !options?.allowPendingApprovalConfigUpdate) {
@@ -83,6 +87,8 @@ export async function prepareAgentConfiguration(db: Db, existing: AgentConfigura
existing.adapterConfig,
);
}
await notifyRunDispatch(db, { companyId: existing.companyId });
await notifyDeferredExecutionEligibility(db, { companyId: existing.companyId });
if (normalizedPatch.runtimeConfig !== undefined || normalizedPatch.reportsTo !== undefined) {
await notifyDeliveryWork(db, DELIVERY_QUEUES.agentTimers);
}
@@ -1,3 +1,5 @@
import { notifyStatusCardIssueChange } from "./status-card-notifications.js";
import { notifyIssueRecovery } from "./issue-recovery-notifications.js";
import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js";
import { notifyUnsafeWorkspaceExportRecovery } from "./unsafe-workspace-export-notifications.js";
import { FAST_RESPONSE_AGENT_GUIDANCE, fastResponseHistoryBody } from "@paperclipai/shared";
@@ -269,6 +271,7 @@ export async function prepareConversationTurn(
context.conversationReplayThroughCommentId = replayThrough?.id ?? commentId;
}
}
await notifyStatusCardIssueChange(tx, issue.companyId);
await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors);
await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers);
await tx
@@ -389,6 +392,7 @@ export async function settleConversationTurn(
issue.conversationState === conversationState
)
return true;
await notifyStatusCardIssueChange(tx, issue.companyId);
await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors);
await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers);
if (status === "in_review") await notifyUnsafeWorkspaceExportRecovery(tx, { companyId: issue.companyId, issueId: issue.id });
@@ -544,6 +548,7 @@ export async function resumeConversationForReset(db: Db, comment: typeof issueCo
eq(issues.id, comment.issueId), eq(issues.companyId, comment.companyId),
)).for("update");
if (!isConversation(issue) || comment.conversationSessionGeneration != null) return;
await notifyIssueRecovery(tx, { companyId: issue.companyId });
const released = await tx.update(issueTreeHolds).set({
status: "released", releasedAt: new Date(), updatedAt: new Date(),
releasedByActorType: "user", releasedByUserId: comment.authorUserId,
+4
View File
@@ -1,3 +1,5 @@
import { notifyAttentionRetentionWork } from "./retention-work-notifications.js";
import { notifyStatusCardIssueChange } from "./status-card-notifications.js";
import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js";
import { notifyExternalConversationSettlement } from "./external-conversation-settlement-notifications.js";
import { notifyUnsafeWorkspaceExportRecovery } from "./unsafe-workspace-export-notifications.js";
@@ -34,10 +36,12 @@ export async function deleteAgentDependencies(tx: Db, companyId: string, id: str
if (classifierHold) throw conflict("Wait for active prompt injection checks or resolve their unknown charges in Costs before deleting this agent", {
code: "agent_prompt_injection_accounting_pending",
});
await notifyAttentionRetentionWork(tx, { companyId });
await notifyExternalConversationSettlement(tx, { companyId });
await issueThreadInteractionService(tx).cancelPendingForDeletedAddressee(companyId, id);
await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors);
await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers);
await notifyStatusCardIssueChange(tx, companyId);
await tx.update(agents).set({ reportsTo: null }).where(eq(agents.reportsTo, id));
await tx
.update(issues)
+2
View File
@@ -1,3 +1,4 @@
import { notifyAttentionRetentionWork } from "./retention-work-notifications.js";
import { notifySkillVersionRetention, notifySkillVersionPinChange } from "./skill-version-retention-notifications.js";
import { agentAppearanceSchema, randomAgentAppearance } from "@paperclipai/shared";
import { normalizePaperclipRunnerAdapterConfig } from "@paperclipai/adapter-utils/server-utils";
@@ -16,6 +17,7 @@ import { initializePrimaryAgent } from "./primary-agent.js";
import { resolveAgentRunnerTaskDriverForCompany, resolveNewAgentRunnerForCompany } from "./agent-runner-selection.js";
export async function prepareAgentHire(db: Db, companyId: string, input: CreateAgentData, options?: CreateAgentOptions): Promise<CreateAgentData> {
await notifyAttentionRetentionWork(db, { companyId });
const { runner, ...data } = input;
const selection = options?.runnerResolved
? { adapterType: data.adapterType ?? "process", adapterConfig: isPlainRecord(data.adapterConfig) ? data.adapterConfig : {} }
+9 -1
View File
@@ -1,3 +1,6 @@
import { notifyAttentionRetentionWork } from "./retention-work-notifications.js";
import { notifyDeferredExecutionEligibility } from "./deferred-execution-notifications.js";
import { notifyRunDispatch } from "./run-dispatch-notifications.js";
import { notifyNativeReviewResponse } from "./native-review-response-notifications.js";
import { isAgentStatusInvokable } from "@paperclipai/shared";
import { notifyNativeResponsePresentation } from "./native-response-presentation-notifications.js";
@@ -38,7 +41,12 @@ export function createAgentLifecycleEffects(_hooks: BudgetServiceHooks = {}): li
},
notifyWork: tx => notifyDeliveryWork(tx, DELIVERY_QUEUES.agentLifecycle),
async beforeStatusChange(tx, companyId, agentId, before, after) {
if (before !== after) await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers);
await notifyAttentionRetentionWork(tx, { companyId });
if (before !== after) {
await notifyRunDispatch(tx, { companyId });
await notifyDeferredExecutionEligibility(tx, { companyId });
await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers);
}
if (!isAgentStatusInvokable(before) && isAgentStatusInvokable(after)) {
await notifyNativeResponsePresentation(tx, { companyId, agentId });
await notifyNativeReviewResponse(tx, { companyId, agentId });
@@ -1,3 +1,4 @@
import { notifyAttentionRetentionWork } from "./retention-work-notifications.js";
import { and, eq, inArray } from "drizzle-orm";
import { agents, projects, type Db } from "@paperclipai/db";
import { trustAuthorizationPolicySchema, type AgentProjectAccess } from "@paperclipai/shared";
@@ -55,6 +56,7 @@ export async function addAgentProjectAccess(db: Db, actor: AuthorizationActor, c
const added = requested.filter(id => !previous.includes(id));
const next = [...previous, ...added];
if (added.length) {
await notifyAttentionRetentionWork(tx, { companyId, sourceKind: "agent_error_alert" });
await tx.update(agents).set({ permissions: { ...agent.permissions, authorizationPolicy: {
...policy, trustBoundary: { ...boundary, mode: "low_trust_review", companyId, projectIds: next },
} }, updatedAt: new Date() }).where(and(eq(agents.id, agentId), eq(agents.companyId, companyId)));
+5 -1
View File
@@ -1,3 +1,4 @@
import { notifyAttentionRetentionWork } from "./retention-work-notifications.js";
import { type BudgetServiceHooks } from "./budgets.js";
import { and, desc, eq, inArray } from "drizzle-orm";
@@ -52,7 +53,9 @@ export function agentService(db: Db, budgetHooks: BudgetServiceHooks = {}) {
});
}
const updated = await db
const updated = await db.transaction(async tx => {
await notifyAttentionRetentionWork(tx, { companyId: existing.companyId });
return tx
.update(agents)
.set({
permissions: normalizeAgentPermissions({ ...existing.permissions, ...permissions }),
@@ -61,6 +64,7 @@ export function agentService(db: Db, budgetHooks: BudgetServiceHooks = {}) {
.where(eq(agents.id, id))
.returning()
.then((rows) => rows[0] ?? null);
});
return updated ? getById(updated.id) : null;
},
+320
View File
@@ -0,0 +1,320 @@
import { and, eq, getTableColumns, inArray, isNull, sql, type SQL } from "drizzle-orm";
import { agents, decisionRetention, decisions, heartbeatRuns,
issueApprovals, issueRecoveryActions, issueThreadInteractions, issues, approvals, type Db } from "@paperclipai/db";
import { evaluateAgentInvokability } from "./agent-invokability.js";
import { HUMAN_RECOVERY_OWNER_TYPES, ATTENTION_BUDGET_WARNING_PERCENT, ATTENTION_AGENT_ERROR_STATUS, collapsePendingConfirmationsToNewest, listFailedRunsWithNewerRun, readRunIssueId, requiresReviewAttention, terminalBlockerRequiresAttention } from "./attention-source-eligibility.js";
import { decisionRetentionService, DEFAULT_DECISION_ARCHIVE_DAYS } from "./decision-retention.js";
import type { createDeliveryWorkCoordinator } from "./delivery-work-coordinator.js";
import { DELIVERY_QUEUES } from "./delivery-work-notifications.js";
import { attentionIssueCondition } from "./issue-visibility.js";
import { RETRY_EXHAUSTION_MESSAGE_PREFIX } from "./heartbeat-run-events.js";
import { issueService } from "./issues.js";
import { parseIssueExecutionState } from "./issue-execution-policy.js";
import { isProspectiveBlockedTransition } from "./routable-blocked.js";
import { registerScopedDeadlineWork } from "./scoped-deadline-work.js";
import type { AttentionRetentionScope } from "./retention-work-notifications.js";
const archiveDelay = DEFAULT_DECISION_ARCHIVE_DAYS * 86_400_000;
const kinds = ["decision", "issue_thread_interaction", "recovery_action", "blocker_attention", "review",
"failed_run", "budget_alert", "agent_error_alert"] as const;
type Kind = typeof kinds[number];
type Scope = AttentionRetentionScope;
type Candidate = { sourceKind: Kind; sourceId: string; companyId: string; activityAt: string; deadlineAt: number; issueId: string | null };
type Entry = Candidate & { id: string; dueAt: number; rootId: string; variant?: "owner" | "terminal" };
type Cursor = { sourceKind: string; sourceId: string };
type PageIssue = typeof issues.$inferSelect & { activityAt: string; deadlineAt: number;
blockedActivityAt: string | null; blockedDeadlineAt: number | null };
type PageEvidence = {
decisions: Map<string, { originIssueId: string }>;
recoveries: Map<string, typeof issueRecoveryActions.$inferSelect>;
runs: Map<string, typeof heartbeatRuns.$inferSelect>;
agents: Set<string>;
visibleIssues: Map<string, PageIssue>;
newerRuns: Set<string>;
deferredRoots: Set<string>;
blockers: Awaited<ReturnType<ReturnType<typeof issueService>["listBlockerAttention"]>>;
};
const sourceTables: Record<Kind, SQL> = {
decision: sql`decisions`, issue_thread_interaction: sql`issue_thread_interactions`,
recovery_action: sql`issue_recovery_actions`, blocker_attention: sql`issues`, review: sql`issues`,
failed_run: sql`heartbeat_runs`, budget_alert: sql`budget_incidents`, agent_error_alert: sql`agents`,
};
const sources: Record<Kind, SQL> = {
decision: sql`select 'decision' as source_kind, d.id::text as source_id, d.company_id, d.updated_at as activity_at, d.origin_issue_id::text as issue_id
from decisions d where d.status = 'open'`,
issue_thread_interaction: sql`select 'issue_thread_interaction', i.id::text, i.company_id, i.updated_at, i.issue_id::text
from issue_thread_interactions i where i.status = 'pending' and i.addressee_user_id is null`,
recovery_action: sql`select 'recovery_action', r.id::text, r.company_id, r.updated_at, r.source_issue_id::text
from issue_recovery_actions r where r.status in ('active', 'escalated') and r.owner_type in (${sql.join(HUMAN_RECOVERY_OWNER_TYPES.map(owner => sql`${owner}`), sql`, `)})`,
blocker_attention: sql`select 'blocker_attention', i.id::text, i.company_id, i.updated_at, i.id::text
from issues i where i.status = 'blocked' and i.hidden_at is null and i.harness_kind is null`,
review: sql`select 'review', i.id::text, i.company_id, i.updated_at, i.id::text
from issues i where i.status = 'in_review' and i.hidden_at is null and i.harness_kind is null`,
failed_run: sql`select 'failed_run', r.id::text, r.company_id, coalesce(r.finished_at, r.updated_at, r.created_at), coalesce(r.context_snapshot->>'issueId', r.context_snapshot->>'taskId')
from heartbeat_runs r where r.status in ('failed', 'timed_out') and exists (
select 1 from heartbeat_run_events e where e.run_id = r.id and e.company_id = r.company_id
and e.event_type = 'lifecycle' and e.message like ${`${RETRY_EXHAUSTION_MESSAGE_PREFIX}%`})`,
budget_alert: sql`select 'budget_alert', b.id::text, b.company_id, b.updated_at, null::text from budget_incidents b
where b.status = 'open' and (b.threshold_type = 'hard' or (b.amount_limit > 0
and round(b.amount_observed::numeric / b.amount_limit * 10000) / 100 >= ${ATTENTION_BUDGET_WARNING_PERCENT}))`,
agent_error_alert: sql`select 'agent_error_alert', a.id::text, a.company_id, a.updated_at, null::text from agents a where a.status = ${ATTENTION_AGENT_ERROR_STATUS}`,
};
export function attentionRetentionService(db: Db) {
const cursors = new Map<string, Cursor>();
async function candidates(database: Db, scope: Scope = {}, cursor?: Cursor, sourceId?: string): Promise<Candidate[]> {
const issueIds = scope.issueIds ?? (scope.issueId ? [scope.issueId] : []);
const selected = kinds.filter(kind => (!scope.sourceKind || scope.sourceKind === kind)
&& (!issueIds.length || !["budget_alert", "agent_error_alert"].includes(kind)));
if (!selected.length) return [];
const rows = await database.execute(sql`with source_candidates(source_kind, source_id, company_id, activity_at, issue_id) as (${sql.join(selected.map(kind => sources[kind]), sql` union all `)})
select s.source_kind as "sourceKind", s.source_id as "sourceId", s.company_id as "companyId",
s.issue_id as "issueId", s.activity_at::text as "activityAt", ceil(extract(epoch from s.activity_at) * 1000 + ${archiveDelay}) as "deadlineAt"
from source_candidates s inner join companies c on c.id = s.company_id
left join decision_retention r on r.company_id = s.company_id and r.source_kind = s.source_kind and r.source_id = s.source_id
where c.status = 'active'
${issueIds.length ? sql`and s.issue_id in (${sql.join(issueIds.map(id => sql`${id}`), sql`, `)})` : sql``}
and (${scope.companyId ?? null}::uuid is null or s.company_id = ${scope.companyId ?? null}::uuid)
and (${sourceId ?? scope.sourceId ?? null}::text is null or s.source_id = ${sourceId ?? scope.sourceId ?? null})
and (s.source_kind = 'blocker_attention' or ((r.id is null or r.keep = false) and r.archived_at is null))
${cursor ? sql`and (s.source_kind, s.source_id) > (${cursor.sourceKind}, ${cursor.sourceId})` : sql``}
order by s.source_kind, s.source_id limit 100`) as unknown as Candidate[];
return rows.map(row => ({ ...row, deadlineAt: Number(row.deadlineAt) }));
}
async function visibleIssue(database: Db, companyId: string, issueId: string, conversations = false) {
const [row] = await database.select().from(issues).where(and(eq(issues.companyId, companyId), eq(issues.id, issueId),
attentionIssueCondition(conversations)));
return row;
}
async function interactionEligibility(database: Db, candidates: Candidate[]) {
const result = new Map<string, boolean>();
const byCompany = new Map<string, string[]>();
for (const candidate of candidates) {
if (candidate.sourceKind !== "issue_thread_interaction") continue;
const ids = byCompany.get(candidate.companyId) ?? [];
ids.push(candidate.sourceId);
byCompany.set(candidate.companyId, ids);
}
for (const [companyId, ids] of byCompany) {
const rows = await database.select().from(issueThreadInteractions).where(and(
eq(issueThreadInteractions.companyId, companyId), inArray(issueThreadInteractions.id, ids),
eq(issueThreadInteractions.status, "pending"), isNull(issueThreadInteractions.addresseeUserId)));
if (!rows.length) continue;
const visible = await database.select({ id: issues.id }).from(issues).where(and(eq(issues.companyId, companyId),
inArray(issues.id, rows.map(row => row.issueId)), attentionIssueCondition(true)));
const visibleIds = new Set(visible.map(row => row.id));
const roster = await database.select().from(agents).where(eq(agents.companyId, companyId));
const boardAddressed = (row: typeof issueThreadInteractions.$inferSelect) => !row.addresseeAgentId
|| !evaluateAgentInvokability(roster.find(agent => agent.id === row.addresseeAgentId), roster).invokable;
const confirmationIssueIds = rows.filter(row => row.kind === "request_confirmation").map(row => row.issueId);
const siblings = confirmationIssueIds.length ? await database.select().from(issueThreadInteractions).where(and(
eq(issueThreadInteractions.companyId, companyId), inArray(issueThreadInteractions.issueId, confirmationIssueIds),
eq(issueThreadInteractions.status, "pending"), eq(issueThreadInteractions.kind, "request_confirmation"),
isNull(issueThreadInteractions.addresseeUserId))) : [];
const newest = new Set(collapsePendingConfirmationsToNewest(siblings.filter(boardAddressed)).map(row => row.id));
for (const row of rows) result.set(row.id, visibleIds.has(row.issueId) && boardAddressed(row)
&& (row.kind !== "request_confirmation" || newest.has(row.id)));
}
return result;
}
async function reviewEligibility(database: Db, candidates: Candidate[]) {
const result = new Map<string, boolean>();
const byCompany = new Map<string, string[]>();
for (const candidate of candidates) {
if (candidate.sourceKind !== "review") continue;
const ids = byCompany.get(candidate.companyId) ?? [];
ids.push(candidate.sourceId);
byCompany.set(candidate.companyId, ids);
}
for (const [companyId, ids] of byCompany) {
const rows = await database.select().from(issues).where(and(eq(issues.companyId, companyId),
inArray(issues.id, ids), attentionIssueCondition(false)));
const pendingApprovals = await database.select({ issueId: issueApprovals.issueId }).from(issueApprovals)
.innerJoin(approvals, eq(approvals.id, issueApprovals.approvalId)).where(and(
eq(issueApprovals.companyId, companyId), eq(approvals.companyId, companyId),
inArray(issueApprovals.issueId, ids), eq(approvals.status, "pending")));
const approved = new Set(pendingApprovals.map(row => row.issueId));
const stalled = await issueService(database).listReviewAttention(companyId, rows);
for (const row of rows) {
const state = parseIssueExecutionState(row.executionState);
result.set(row.id, requiresReviewAttention({
hasHumanParticipant: state?.status === "pending" && state.currentParticipant?.type === "user",
hasUserAssignee: Boolean(row.assigneeUserId), hasPendingApproval: approved.has(row.id),
stalled: stalled.get(row.id)?.state === "stalled",
}));
}
}
return result;
}
async function pageEvidence(database: Db, candidates: Candidate[]) {
const byCompany = new Map<string, Candidate[]>();
for (const candidate of candidates) {
if (!["decision", "recovery_action", "failed_run", "blocker_attention"].includes(candidate.sourceKind)) continue;
const rows = byCompany.get(candidate.companyId) ?? [];
rows.push(candidate);
byCompany.set(candidate.companyId, rows);
}
const result = new Map<string, PageEvidence>();
for (const [companyId, rows] of byCompany) {
const ids = (kind: Kind) => rows.filter(row => row.sourceKind === kind).map(row => row.sourceId);
const decisionIds = ids("decision"), recoveryIds = ids("recovery_action"), runIds = ids("failed_run"), blockerIds = ids("blocker_attention");
const decisionRows = decisionIds.length ? await database.select({ id: decisions.id, originIssueId: decisions.originIssueId })
.from(decisions).where(and(eq(decisions.companyId, companyId), inArray(decisions.id, decisionIds), eq(decisions.status, "open"))) : [];
const recoveryRows = recoveryIds.length ? await database.select().from(issueRecoveryActions).where(and(
eq(issueRecoveryActions.companyId, companyId), inArray(issueRecoveryActions.id, recoveryIds))) : [];
const runRows = runIds.length ? await database.select().from(heartbeatRuns).where(and(
eq(heartbeatRuns.companyId, companyId), inArray(heartbeatRuns.id, runIds))) : [];
const runAgents = runRows.length ? await database.select({ id: agents.id }).from(agents).where(and(
eq(agents.companyId, companyId), inArray(agents.id, [...new Set(runRows.map(row => row.agentId))]),
sql`${agents.status} <> 'terminated'`)) : [];
const readIssues = async (issueIds: string[]) => issueIds.length ? database.select({ ...getTableColumns(issues),
activityAt: sql<string>`${issues.updatedAt}::text`,
deadlineAt: sql<number>`ceil(extract(epoch from ${issues.updatedAt}) * 1000 + ${archiveDelay})`.mapWith(Number),
blockedActivityAt: sql<string | null>`${issues.blockedTransitionAt}::text`,
blockedDeadlineAt: sql<number | null>`ceil(extract(epoch from ${issues.blockedTransitionAt}) * 1000 + ${archiveDelay})`.mapWith(Number),
}).from(issues).where(and(eq(issues.companyId, companyId), inArray(issues.id, [...new Set(issueIds)]),
attentionIssueCondition(false))) : [];
const issueIds = [...blockerIds, ...decisionRows.map(row => row.originIssueId), ...recoveryRows.map(row => row.sourceIssueId),
...runRows.map(row => readRunIssueId(row.contextSnapshot))].filter((id): id is string => Boolean(id));
const visibleRows = await readIssues(issueIds);
const visible = new Map(visibleRows.map(row => [row.id, row]));
const blockerRoots = visibleRows.filter(row => blockerIds.includes(row.id));
const blockerBatch = blockerRoots.length ? await issueService(database).listBlockerAttentionBatch(companyId, blockerRoots)
: { attention: new Map(), deferredRootIds: new Set<string>() };
const blockers: PageEvidence["blockers"] = blockerBatch.attention;
const terminalIds = [...blockers.values()].filter(terminalBlockerRequiresAttention)
.map(attention => attention.terminalBlockerIssueId).filter((id): id is string => id != null && !visible.has(id));
for (const row of await readIssues(terminalIds)) visible.set(row.id, row);
result.set(companyId, {
decisions: new Map(decisionRows.map(row => [row.id, row])), recoveries: new Map(recoveryRows.map(row => [row.id, row])),
runs: new Map(runRows.map(row => [row.id, row])), agents: new Set(runAgents.map(row => row.id)), visibleIssues: visible,
newerRuns: await listFailedRunsWithNewerRun(database, companyId,
runRows.map(run => ({ ...run, issueId: readRunIssueId(run.contextSnapshot) }))), blockers, deferredRoots: blockerBatch.deferredRootIds,
});
}
return result;
}
async function eligible(database: Db, candidate: Candidate, variant?: "owner" | "terminal", lock = false, reviewAttention?: Map<string, boolean>, interactionAttention?: Map<string, boolean>, evidence?: PageEvidence): Promise<Entry | null> {
if (candidate.sourceKind === "blocker_attention" && variant === "terminal" && evidence?.deferredRoots.has(candidate.sourceId)) {
return { ...candidate, rootId: candidate.sourceId, variant,
id: `${candidate.sourceKind}:${candidate.sourceId}:terminal`, dueAt: Date.now() };
}
let sourceId = candidate.sourceId;
let activityAt = candidate.activityAt;
let dueAt = candidate.deadlineAt;
if (candidate.sourceKind === "decision") {
const row = evidence ? evidence.decisions.get(sourceId) : (await database.select({ originIssueId: decisions.originIssueId }).from(decisions)
.where(and(eq(decisions.companyId, candidate.companyId), eq(decisions.id, sourceId), eq(decisions.status, "open"))))[0];
if (!row || row.originIssueId && !(evidence ? evidence.visibleIssues.has(row.originIssueId)
: await visibleIssue(database, candidate.companyId, row.originIssueId))) return null;
} else if (candidate.sourceKind === "issue_thread_interaction") {
const attention = interactionAttention ?? await interactionEligibility(database, [candidate]);
if (!attention.get(sourceId)) return null;
} else if (candidate.sourceKind === "recovery_action") {
const row = evidence ? evidence.recoveries.get(sourceId) : (await database.select().from(issueRecoveryActions).where(and(eq(issueRecoveryActions.companyId, candidate.companyId), eq(issueRecoveryActions.id, sourceId))))[0];
if (!row || !(evidence ? evidence.visibleIssues.has(row.sourceIssueId)
: await visibleIssue(database, candidate.companyId, row.sourceIssueId))) return null;
} else if (candidate.sourceKind === "review") {
const attention = reviewAttention ?? await reviewEligibility(database, [candidate]);
if (!attention.get(sourceId)) return null;
} else if (candidate.sourceKind === "blocker_attention") {
const row = evidence ? evidence.visibleIssues.get(sourceId) : await visibleIssue(database, candidate.companyId, sourceId);
if (!row) return null;
if (variant === "owner") {
if (row.unblockDescriptor?.owner !== "board" || !isProspectiveBlockedTransition(row)) return null;
const activity = evidence ? { at: evidence.visibleIssues.get(sourceId)!.blockedActivityAt!, due: evidence.visibleIssues.get(sourceId)!.blockedDeadlineAt! } : (await database.execute(sql`select blocked_transition_at::text as at,
ceil(extract(epoch from blocked_transition_at) * 1000 + ${archiveDelay}) as due from issues
where company_id = ${candidate.companyId}::uuid and id = ${row.id}::uuid`) as unknown as Array<{ at: string; due: string }>)[0];
activityAt = activity.at; dueAt = Number(activity.due);
} else {
const attention = evidence ? evidence.blockers.get(row.id) : (await issueService(database).listBlockerAttention(candidate.companyId, [row])).get(row.id);
if (!terminalBlockerRequiresAttention(attention)) return null;
sourceId = attention.terminalBlockerIssueId ?? row.id;
if (lock && sourceId !== candidate.sourceId) await database.execute(sql`select id from issues
where company_id = ${candidate.companyId}::uuid and id = ${sourceId}::uuid for update`);
const terminal = evidence ? evidence.visibleIssues.get(sourceId) : await visibleIssue(database, candidate.companyId, sourceId);
if (!terminal) return null;
const activity = evidence ? { at: evidence.visibleIssues.get(sourceId)!.activityAt, due: evidence.visibleIssues.get(sourceId)!.deadlineAt } : (await database.execute(sql`select updated_at::text as at,
ceil(extract(epoch from updated_at) * 1000 + ${archiveDelay}) as due from issues
where company_id = ${candidate.companyId}::uuid and id = ${sourceId}::uuid`) as unknown as Array<{ at: string; due: string }>)[0];
activityAt = activity.at; dueAt = Number(activity.due);
}
const [retention] = await database.select().from(decisionRetention).where(and(eq(decisionRetention.companyId, candidate.companyId),
eq(decisionRetention.sourceKind, candidate.sourceKind), eq(decisionRetention.sourceId, sourceId)));
if (retention?.keep || retention?.archivedAt) return null;
} else if (candidate.sourceKind === "failed_run") {
const run = evidence ? evidence.runs.get(sourceId) : (await database.select().from(heartbeatRuns).where(and(eq(heartbeatRuns.companyId, candidate.companyId), eq(heartbeatRuns.id, sourceId))))[0];
if (!run) return null;
const agent = evidence ? evidence.agents.has(run.agentId) : (await database.select({ id: agents.id }).from(agents).where(and(eq(agents.companyId, candidate.companyId),
eq(agents.id, run.agentId), sql`${agents.status} <> 'terminated'`)))[0];
const issueId = readRunIssueId(run.contextSnapshot);
if (!agent || issueId && !(evidence ? evidence.visibleIssues.has(issueId)
: await visibleIssue(database, candidate.companyId, issueId))) return null;
if ((evidence ? evidence.newerRuns : await listFailedRunsWithNewerRun(database, candidate.companyId, [{ ...run, issueId }])).has(run.id)) return null;
}
return { ...candidate, sourceId, activityAt, rootId: candidate.sourceId, variant,
id: `${candidate.sourceKind}:${candidate.sourceId}:${variant ?? "source"}`, dueAt };
}
async function load(scope?: Scope, signal?: AbortSignal) {
const key = JSON.stringify(scope ?? {});
const rows = await candidates(db, scope, cursors.get(key));
const entries: Entry[] = [];
const reviews = await reviewEligibility(db, rows);
const interactions = await interactionEligibility(db, rows);
const evidence = await pageEvidence(db, rows);
for (const row of rows) {
if (signal?.aborted) return { rows: null, complete: true };
for (const variant of row.sourceKind === "blocker_attention" ? ["owner", "terminal"] as const : [undefined]) {
const entry = await eligible(db, row, variant, false, reviews, interactions, evidence.get(row.companyId));
if (entry) entries.push(entry);
}
}
if (rows.length === 100) {
const last = rows[rows.length - 1]!;
cursors.set(key, { sourceKind: last.sourceKind, sourceId: last.sourceId });
} else cursors.delete(key);
return { rows: entries, complete: rows.length < 100 };
}
async function check(entry: Entry) {
return db.transaction(async transaction => {
const database = transaction as unknown as Db;
await database.execute(sql`select id from ${sourceTables[entry.sourceKind]}
where company_id = ${entry.companyId}::uuid and id = ${entry.rootId}::uuid for update`);
const [candidate] = await candidates(database, { companyId: entry.companyId, sourceKind: entry.sourceKind }, undefined, entry.rootId);
if (!candidate) return null;
const current = await eligible(database, candidate, entry.variant, true);
if (!current || current.dueAt > Date.now()) return current;
const retention = decisionRetentionService(database);
const item = { sourceKind: current.sourceKind, subject: { id: current.sourceId }, activityAt: current.activityAt };
const state = (await retention.syncItems(current.companyId, [item])).get(`${current.sourceKind}:${current.sourceId}`)!;
await retention.autoArchive({ companyId: current.companyId, items: [{ ...item,
keep: state.keep, archivedAt: state.archivedAt?.toISOString() ?? null, retentionVersion: state.version }] });
return null;
});
}
return { load, check };
}
export function registerAttentionRetentionWork(coordinator: ReturnType<typeof createDeliveryWorkCoordinator>,
service: ReturnType<typeof attentionRetentionService>, canRun: () => boolean) {
return registerScopedDeadlineWork(coordinator, {
queue: DELIVERY_QUEUES.attentionRetention,
decodeScope: hint => hint === "*" ? undefined : JSON.parse(hint) as Scope,
matches: (entry, scope) => (!scope?.companyId || scope.companyId === entry.companyId)
&& (!scope?.sourceKind || scope.sourceKind === entry.sourceKind)
&& (!scope?.sourceId || scope.sourceId === entry.sourceId)
&& (!scope?.issueId || scope.issueId === entry.issueId)
&& (!scope?.issueIds?.length || entry.issueId !== null && scope.issueIds.includes(entry.issueId)),
load: service.load, check: service.check,
}, canRun);
}
@@ -0,0 +1,112 @@
import { and, eq, gt, inArray, sql, type SQL } from "drizzle-orm";
import { heartbeatRuns, type Db } from "@paperclipai/db";
export const HUMAN_RECOVERY_OWNER_TYPES = ["user", "board"] as const;
export const ATTENTION_BUDGET_WARNING_PERCENT = 85;
export const ATTENTION_AGENT_ERROR_STATUS = "error";
export function collapsePendingConfirmationsToNewest<T extends {
id: string;
issueId: string;
kind: string;
createdAt: Date;
payload: unknown;
}>(rows: T[]) {
const independent = (row: T) => Boolean(row.payload && typeof row.payload === "object" && "qualityReview" in row.payload && row.payload.qualityReview);
const newestByGroup = new Map<string, T>();
for (const row of rows) {
if (row.kind !== "request_confirmation" || independent(row)) continue;
const groupKey = `${row.issueId}:${row.kind}`;
const newest = newestByGroup.get(groupKey);
if (
!newest
|| row.createdAt.getTime() > newest.createdAt.getTime()
|| (row.createdAt.getTime() === newest.createdAt.getTime() && row.id > newest.id)
) {
newestByGroup.set(groupKey, row);
}
}
return rows.filter((row) => (
row.kind !== "request_confirmation"
|| independent(row)
|| newestByGroup.get(`${row.issueId}:${row.kind}`)?.id === row.id
));
}
export function readRunIssueId(contextSnapshot: Record<string, unknown> | null) {
const issueId = contextSnapshot?.issueId ?? contextSnapshot?.taskId;
return typeof issueId === "string" && issueId.length > 0 ? issueId : null;
}
type FailedRunProbe = { id: string; agentId: string; issueId: string | null; createdAt: Date };
const FAILED_RUN_PROBE_BATCH_SIZE = 1000;
export async function listFailedRunsWithNewerRun(db: Db, companyId: string, failedRuns: FailedRunProbe[]) {
const withNewerRun = new Set<string>();
const keyed = failedRuns.filter(run => run.issueId !== null);
for (let start = 0; start < keyed.length; start += FAILED_RUN_PROBE_BATCH_SIZE) {
const batch = keyed.slice(start, start + FAILED_RUN_PROBE_BATCH_SIZE);
for (const id of await listFailedRunBatchWithNewerRun(db, companyId, batch)) withNewerRun.add(id);
}
const unkeyed = failedRuns.filter(run => run.issueId === null);
if (unkeyed.length === 0) return withNewerRun;
const agentIds = [...new Set(unkeyed.map(run => run.agentId))];
const oldestCreatedAt = unkeyed.reduce((oldest, run) => run.createdAt < oldest ? run.createdAt : oldest,
unkeyed[0]!.createdAt);
const latestByAgent = new Map<string, Date>();
for (let start = 0; start < agentIds.length; start += FAILED_RUN_PROBE_BATCH_SIZE) {
const rows = await db.select({
agentId: heartbeatRuns.agentId,
createdAt: sql<Date>`max(${heartbeatRuns.createdAt})`.mapWith(heartbeatRuns.createdAt),
}).from(heartbeatRuns).where(and(
eq(heartbeatRuns.companyId, companyId),
inArray(heartbeatRuns.agentId, agentIds.slice(start, start + FAILED_RUN_PROBE_BATCH_SIZE)),
gt(heartbeatRuns.createdAt, oldestCreatedAt),
sql`coalesce(${heartbeatRuns.contextSnapshot} ->> 'issueId', ${heartbeatRuns.contextSnapshot} ->> 'taskId', '') = ''`,
)).groupBy(heartbeatRuns.agentId);
for (const row of rows) latestByAgent.set(row.agentId, row.createdAt);
}
for (const run of unkeyed) {
if ((latestByAgent.get(run.agentId)?.getTime() ?? 0) > run.createdAt.getTime()) withNewerRun.add(run.id);
}
return withNewerRun;
}
async function listFailedRunBatchWithNewerRun(db: Db, companyId: string, failedRuns: FailedRunProbe[]) {
if (failedRuns.length === 0) return [];
const failed = sql.join(
failedRuns.map((run) => sql`(${run.id}::uuid, ${run.agentId}::uuid, ${run.issueId}::text, ${new Date(run.createdAt.getTime() + 1).toISOString()}::timestamptz)`),
sql`, `,
);
const runIssueId = sql`(${heartbeatRuns.contextSnapshot} ->> 'issueId')`;
const runTaskId = sql`(${heartbeatRuns.contextSnapshot} ->> 'taskId')`;
const newerRunExists = (match: SQL) => sql`exists (
select 1 from ${heartbeatRuns}
where ${heartbeatRuns.companyId} = ${companyId}
and ${heartbeatRuns.agentId} = failed.agent_id
and ${heartbeatRuns.createdAt} >= failed.newer_from
and ${match}
)`;
const rows = await db.execute(sql`
select failed.id
from (values ${failed}) as failed (id, agent_id, issue_id, newer_from)
where ${newerRunExists(sql`${runIssueId} = failed.issue_id`)}
or ${newerRunExists(sql`${runTaskId} = failed.issue_id and ${runIssueId} is null`)}
`) as unknown as Array<{ id: string }>;
return rows.map((row) => row.id);
}
export function requiresReviewAttention(input: {
hasHumanParticipant: boolean; hasUserAssignee: boolean; hasPendingApproval: boolean; stalled: boolean;
}) {
return input.hasHumanParticipant || input.hasUserAssignee || input.hasPendingApproval || input.stalled;
}
export function terminalBlockerRequiresAttention<T extends { state?: string; blockingTreeLive?: boolean }>(
attention: T | null | undefined,
): attention is T & { state: "stalled" | "needs_attention" } {
return Boolean(attention && !attention.blockingTreeLive && ["stalled", "needs_attention"].includes(attention.state ?? ""));
}
+9 -106
View File
@@ -1,5 +1,6 @@
import { HUMAN_RECOVERY_OWNER_TYPES, ATTENTION_BUDGET_WARNING_PERCENT, ATTENTION_AGENT_ERROR_STATUS, collapsePendingConfirmationsToNewest, readRunIssueId, listFailedRunsWithNewerRun, requiresReviewAttention, terminalBlockerRequiresAttention } from "./attention-source-eligibility.js";
import { approvalExecutionIssueCondition, firstApprovalIssueId } from "./approval-visibility.js";
import { and, asc, desc, eq, gt, inArray, isNotNull, isNull, notInArray, or, sql, type SQL } from "drizzle-orm";
import { and, asc, desc, eq, inArray, isNull, notInArray, or, sql, type SQL } from "drizzle-orm";
import type { Db } from "@paperclipai/db";
import {
agents,
@@ -58,7 +59,7 @@ import {
BLOCKER_ATTENTION_MAX_NODES,
issueService,
} from "./issues.js";
import { executionIssueCondition, visibleIssueCondition } from "./issue-visibility.js";
import { attentionIssueCondition, executionIssueCondition } from "./issue-visibility.js";
import { parseIssueExecutionState } from "./issue-execution-policy.js";
import { isProspectiveBlockedTransition } from "./routable-blocked.js";
import { issueReadSqlCondition, type AuthorizationActor } from "./authorization.js";
@@ -107,7 +108,6 @@ const SOURCE_RANK: Record<AttentionSourceKind, number> = {
const PENDING_INTERACTION_STATUSES = ["pending"] as const;
const OPEN_RECOVERY_STATUSES = ["active", "escalated"] as const;
const HUMAN_RECOVERY_OWNER_TYPES = ["user", "board"] as const;
const DETAIL_EXCERPT_LENGTH = 160;
const DETAIL_IMAGE_LIMIT = 3;
const OPEN_DECISION_DEFAULT_LIMIT = 500;
@@ -766,37 +766,6 @@ export function interactionResolverAudience(
};
}
function collapsePendingConfirmationsToNewest<T extends {
id: string;
issueId: string;
kind: string;
createdAt: Date;
payload: unknown;
}>(rows: T[]) {
// Quality gates bind independent outputs; a newer visual approval must not
// hide a still-pending copy approval (or affect its badge count).
const independent = (row: T) => Boolean(row.payload && typeof row.payload === "object" && "qualityReview" in row.payload && row.payload.qualityReview);
const newestByGroup = new Map<string, T>();
for (const row of rows) {
if (row.kind !== "request_confirmation" || independent(row)) continue;
const groupKey = `${row.issueId}:${row.kind}`;
const newest = newestByGroup.get(groupKey);
if (
!newest
|| row.createdAt.getTime() > newest.createdAt.getTime()
|| (row.createdAt.getTime() === newest.createdAt.getTime() && row.id > newest.id)
) {
newestByGroup.set(groupKey, row);
}
}
return rows.filter((row) => (
row.kind !== "request_confirmation"
|| independent(row)
|| newestByGroup.get(`${row.issueId}:${row.kind}`)?.id === row.id
));
}
function budgetObservedPercent(amountObserved: number, amountLimit: number) {
return amountLimit > 0 ? Math.round((amountObserved / amountLimit) * 10_000) / 100 : 0;
}
@@ -866,9 +835,7 @@ async function issueSummaryMap(
.where(and(
eq(issues.companyId, companyId),
inArray(issues.id, ids),
includeConversations
? or(executionIssueCondition(), and(visibleIssueCondition(), isNotNull(issues.conversationAgentId)))
: executionIssueCondition(),
attentionIssueCondition(includeConversations),
readCondition,
));
return new Map(rows.map((row) => [row.id, {
@@ -1108,70 +1075,6 @@ function resolveBlockingIssue(
return null;
}
function readRunIssueId(contextSnapshot: Record<string, unknown> | null) {
const issueId = contextSnapshot?.issueId ?? contextSnapshot?.taskId;
return typeof issueId === "string" && issueId.length > 0 ? issueId : null;
}
type FailedRunProbe = { id: string; agentId: string; issueId: string | null; createdAt: Date };
const FAILED_RUN_PROBE_BATCH_SIZE = 1000;
async function listFailedRunsWithNewerRun(db: Db, companyId: string, failedRuns: FailedRunProbe[]) {
const withNewerRun = new Set<string>();
const keyed = failedRuns.filter(run => run.issueId !== null);
for (let start = 0; start < keyed.length; start += FAILED_RUN_PROBE_BATCH_SIZE) {
const batch = keyed.slice(start, start + FAILED_RUN_PROBE_BATCH_SIZE);
for (const id of await listFailedRunBatchWithNewerRun(db, companyId, batch)) withNewerRun.add(id);
}
const unkeyed = failedRuns.filter(run => run.issueId === null);
if (unkeyed.length === 0) return withNewerRun;
const agentIds = [...new Set(unkeyed.map(run => run.agentId))];
const oldestCreatedAt = unkeyed.reduce((oldest, run) => run.createdAt < oldest ? run.createdAt : oldest,
unkeyed[0]!.createdAt);
const latestByAgent = new Map<string, Date>();
for (let start = 0; start < agentIds.length; start += FAILED_RUN_PROBE_BATCH_SIZE) {
const rows = await db.select({
agentId: heartbeatRuns.agentId,
createdAt: sql<Date>`max(${heartbeatRuns.createdAt})`.mapWith(heartbeatRuns.createdAt),
}).from(heartbeatRuns).where(and(
eq(heartbeatRuns.companyId, companyId),
inArray(heartbeatRuns.agentId, agentIds.slice(start, start + FAILED_RUN_PROBE_BATCH_SIZE)),
gt(heartbeatRuns.createdAt, oldestCreatedAt),
sql`coalesce(${heartbeatRuns.contextSnapshot} ->> 'issueId', ${heartbeatRuns.contextSnapshot} ->> 'taskId', '') = ''`,
)).groupBy(heartbeatRuns.agentId);
for (const row of rows) latestByAgent.set(row.agentId, row.createdAt);
}
for (const run of unkeyed) {
if ((latestByAgent.get(run.agentId)?.getTime() ?? 0) > run.createdAt.getTime()) withNewerRun.add(run.id);
}
return withNewerRun;
}
async function listFailedRunBatchWithNewerRun(db: Db, companyId: string, failedRuns: FailedRunProbe[]) {
if (failedRuns.length === 0) return [];
const failed = sql.join(
failedRuns.map((run) => sql`(${run.id}::uuid, ${run.agentId}::uuid, ${run.issueId}::text, ${new Date(run.createdAt.getTime() + 1).toISOString()}::timestamptz)`),
sql`, `,
);
const runIssueId = sql`(${heartbeatRuns.contextSnapshot} ->> 'issueId')`;
const runTaskId = sql`(${heartbeatRuns.contextSnapshot} ->> 'taskId')`;
const newerRunExists = (match: SQL) => sql`exists (
select 1 from ${heartbeatRuns}
where ${heartbeatRuns.companyId} = ${companyId}
and ${heartbeatRuns.agentId} = failed.agent_id
and ${heartbeatRuns.createdAt} >= failed.newer_from
and ${match}
)`;
const rows = await db.execute(sql`
select failed.id
from (values ${failed}) as failed (id, agent_id, issue_id, newer_from)
where ${newerRunExists(sql`${runIssueId} = failed.issue_id`)}
or ${newerRunExists(sql`${runTaskId} = failed.issue_id and ${runIssueId} is null`)}
`) as unknown as Array<{ id: string }>;
return rows.map((row) => row.id);
}
export function attentionService(db: Db, serviceOptions: AttentionServiceOptions = {}) {
const openDecisionLimit = Math.min(
Math.max(Math.trunc(serviceOptions.openDecisionLimit ?? OPEN_DECISION_DEFAULT_LIMIT), 1),
@@ -1636,8 +1539,7 @@ export function attentionService(db: Db, serviceOptions: AttentionServiceOptions
}));
}
const blockerAttention = issue.blockerAttention;
if (blockerAttention?.state !== "stalled" && blockerAttention?.state !== "needs_attention") continue;
if (blockerAttention.blockingTreeLive) continue;
if (!terminalBlockerRequiresAttention(blockerAttention)) continue;
const issueSummary = blockedIssueSummaries.get(issue.id) ?? null;
const terminalIssueId = blockerAttention.terminalBlockerIssueId ?? issue.id;
const terminalSummary = terminalBlockerSummaries.get(terminalIssueId)
@@ -1739,7 +1641,8 @@ export function attentionService(db: Db, serviceOptions: AttentionServiceOptions
const pendingApprovalId = pendingApprovalByIssueId.get(review.id) ?? null;
const reviewAttention = reviewAttentionByIssueId.get(review.id);
const stalled = reviewAttention?.state === "stalled";
if (!hasHumanParticipant && !review.assigneeUserId && !pendingApprovalId && !stalled) continue;
if (!requiresReviewAttention({ hasHumanParticipant, hasUserAssignee: Boolean(review.assigneeUserId),
hasPendingApproval: Boolean(pendingApprovalId), stalled })) continue;
const issue = reviewIssueMap.get(review.id);
if (!issue) continue;
const dedupKey = `review:${review.id}`;
@@ -1858,7 +1761,7 @@ export function attentionService(db: Db, serviceOptions: AttentionServiceOptions
const budgetOverview = await budgetService(db).overview(companyId);
for (const incident of budgetOverview.activeIncidents) {
const observedPercent = budgetObservedPercent(incident.amountObserved, incident.amountLimit);
if (incident.thresholdType !== "hard" && observedPercent < 85) continue;
if (incident.thresholdType !== "hard" && observedPercent < ATTENTION_BUDGET_WARNING_PERCENT) continue;
const dedupKey = `budget:${incident.policyId}:${toIso(incident.windowStart)}:${incident.thresholdType}`;
add(createItem({
companyId,
@@ -1921,7 +1824,7 @@ export function attentionService(db: Db, serviceOptions: AttentionServiceOptions
updatedAt: agents.updatedAt,
})
.from(agents)
.where(and(eq(agents.companyId, companyId), eq(agents.status, "error")))
.where(and(eq(agents.companyId, companyId), eq(agents.status, ATTENTION_AGENT_ERROR_STATUS)))
.orderBy(desc(agents.updatedAt), desc(agents.id));
for (const agent of erroredAgents) {
+49 -11
View File
@@ -1,5 +1,8 @@
import { notifyStatusCardIssueChange } from "./status-card-notifications.js";
import { notifyDeferredExecutionEligibility } from "./deferred-execution-notifications.js";
import { notifyRunDispatch } from "./run-dispatch-notifications.js";
import { notifyApprovalWork } from "./approval-work-notifications.js";
import { notifyCompanyRetentionWork } from "./retention-work-notifications.js";
import { notifyAttentionRetentionWork, notifyCompanyRetentionWork } from "./retention-work-notifications.js";
import { addCents, compareCents, normalizeCents } from "@paperclipai/shared";
import { and, desc, eq, inArray, sql, or, isNull } from "drizzle-orm";
import type { Db } from "@paperclipai/db";
@@ -163,6 +166,19 @@ async function resolveScopeRecord(db: Db, scopeType: BudgetScopeType, scopeId: s
};
}
async function resolveSummaryScopeRecord(db: Db, companyId: string, scopeType: BudgetScopeType, scopeId: string): Promise<ScopeRecord> {
let scope: ScopeRecord;
try {
scope = await resolveScopeRecord(db, scopeType, scopeId);
} catch (error) {
if (!(error instanceof HttpError) || error.status !== 404 || (scopeType !== "agent" && scopeType !== "project")) throw error;
return { companyId, name: scopeType === "agent" ? "Deleted agent" : "Deleted project",
paused: false, pauseReason: null, budgetHeld: false };
}
if (scope.companyId !== companyId) throw notFound("Budget scope not found");
return scope;
}
// Keep a large dashboard from filling the database pool with queued reads.
async function mapBudgetReads<T, R>(rows: T[], read: (row: T) => Promise<R>): Promise<R[]> {
const result: R[] = [];
@@ -287,6 +303,8 @@ export function budgetServiceInTransaction(db: Db, publications: ActivityPublica
}
if (policy.scopeType === "project") {
await notifyRunDispatch(db, { companyId: policy.companyId });
await notifyDeferredExecutionEligibility(db, { companyId: policy.companyId });
await db
.update(projects)
.set({
@@ -300,7 +318,10 @@ export function budgetServiceInTransaction(db: Db, publications: ActivityPublica
const scope = await resolveScopeRecord(db, "company", policy.scopeId);
if (scope.paused) return;
await notifyRunDispatch(db, { companyId: policy.companyId });
await notifyDeferredExecutionEligibility(db, { companyId: policy.companyId });
await notifyCompanyRetentionWork(db, policy.companyId);
await notifyStatusCardIssueChange(db, policy.companyId);
await notifyDeliveryWork(db, DELIVERY_QUEUES.agentTimers);
await notifyDeliveryWork(db, DELIVERY_QUEUES.routineSchedules);
await notifyDeliveryWork(db, DELIVERY_QUEUES.issueMonitors);
@@ -346,6 +367,8 @@ export function budgetServiceInTransaction(db: Db, publications: ActivityPublica
}
if (policy.scopeType === "project") {
await notifyRunDispatch(db, { companyId: policy.companyId });
await notifyDeferredExecutionEligibility(db, { companyId: policy.companyId });
await db
.update(projects)
.set({
@@ -360,7 +383,10 @@ export function budgetServiceInTransaction(db: Db, publications: ActivityPublica
await notifyDeliveryWork(db, DELIVERY_QUEUES.skillSourceSync);
const scope = await resolveScopeRecord(db, "company", policy.scopeId);
if (scope.pauseReason !== "budget") return;
await notifyRunDispatch(db, { companyId: policy.companyId });
await notifyDeferredExecutionEligibility(db, { companyId: policy.companyId });
await notifyCompanyRetentionWork(db, policy.companyId);
await notifyStatusCardIssueChange(db, policy.companyId);
await notifyDeliveryWork(db, DELIVERY_QUEUES.agentTimers);
await notifyDeliveryWork(db, DELIVERY_QUEUES.routineSchedules);
await notifyDeliveryWork(db, DELIVERY_QUEUES.issueMonitors);
@@ -395,7 +421,7 @@ export function budgetServiceInTransaction(db: Db, publications: ActivityPublica
}
async function buildPolicySummary(policy: PolicyRow): Promise<BudgetPolicySummary> {
const scope = await resolveScopeRecord(db, policy.scopeType as BudgetScopeType, policy.scopeId);
const scope = await resolveSummaryScopeRecord(db, policy.companyId, policy.scopeType as BudgetScopeType, policy.scopeId);
const observed = await computeObservedSpend(db, policy);
const { total: observedAmount, totalExact: observedAmountExact, unpricedEventCount, pendingRunCount } = observed;
const { windowStart: start, windowEnd: end } = observed;
@@ -466,6 +492,7 @@ export function budgetServiceInTransaction(db: Db, publications: ActivityPublica
windowEnd: end,
});
await notifyAttentionRetentionWork(db, { companyId: policy.companyId, sourceKind: "budget_alert" });
if (thresholdType === "hard") await notifyApprovalWork(db, policy.companyId);
const approval = thresholdType === "hard"
? await db
@@ -504,7 +531,8 @@ export function budgetServiceInTransaction(db: Db, publications: ActivityPublica
return incident ? { incident, created: true } : null;
}
async function resolveOpenSoftIncidents(policyId: string) {
async function resolveOpenSoftIncidents(policyId: string, companyId: string) {
await notifyAttentionRetentionWork(db, { companyId, sourceKind: "budget_alert" });
await db
.update(budgetIncidents)
.set({
@@ -523,10 +551,12 @@ export function budgetServiceInTransaction(db: Db, publications: ActivityPublica
async function resolveOpenIncidentsForPolicy(
policyId: string,
companyId: string,
approvalStatus: "approved" | "rejected" | null,
decidedByUserId: string | null,
thresholdType?: BudgetThresholdType,
) {
await notifyAttentionRetentionWork(db, { companyId, sourceKind: "budget_alert" });
const condition = and(eq(budgetIncidents.policyId, policyId), eq(budgetIncidents.status, "open"),
thresholdType ? eq(budgetIncidents.thresholdType, thresholdType) : undefined);
const openRows = await db
@@ -560,7 +590,7 @@ export function budgetServiceInTransaction(db: Db, publications: ActivityPublica
const approvalStatusById = new Map(approvalRows.map((row) => [row.id, row.status]));
return mapBudgetReads(rows, async (row) => {
const scope = await resolveScopeRecord(db, row.scopeType as BudgetScopeType, row.scopeId);
const scope = await resolveSummaryScopeRecord(db, row.companyId, row.scopeType as BudgetScopeType, row.scopeId);
return {
id: row.id,
companyId: row.companyId,
@@ -615,9 +645,9 @@ export function budgetServiceInTransaction(db: Db, publications: ActivityPublica
if (policy.isActive && policy.hardStopEnabled && policy.amount > 0 && observed.recoveringRunCount > 0) recoveringPolicy ??= policy;
// A closed UTC window, inactive policy, or budget raise releases only
// budget-owned pauses. Manual pauses remain operator decisions.
await resolveOpenIncidentsForPolicy(policy.id, "approved", "budget_service", "hard");
await resolveOpenIncidentsForPolicy(policy.id, policy.companyId, "approved", "budget_service", "hard");
if (!policy.isActive || !policy.notifyEnabled || compareCents(observed.totalExact, Math.ceil(policy.amount * policy.warnPercent / 100)) < 0) {
await resolveOpenIncidentsForPolicy(policy.id, "approved", "budget_service", "soft");
await resolveOpenIncidentsForPolicy(policy.id, policy.companyId, "approved", "budget_service", "soft");
}
}
// Agent status has its own admission gate and error code. Preserve a
@@ -674,7 +704,7 @@ export function budgetServiceInTransaction(db: Db, publications: ActivityPublica
}
if (observedBlocks(policy, observed)) {
await resolveOpenSoftIncidents(policy.id);
await resolveOpenSoftIncidents(policy.id, policy.companyId);
const hardIncident = await createIncidentIfNeeded(policy, "hard", observed);
await pauseAndCancelScopeForBudget(policy);
if (hardIncident?.created) {
@@ -844,11 +874,11 @@ export function budgetServiceInTransaction(db: Db, publications: ActivityPublica
const observed = await computeObservedSpend(db, row);
if (!observedBlocks(row, observed)) {
await resumeScopeFromBudget(row);
await resolveOpenIncidentsForPolicy(row.id, "approved", actorUserId ?? "budget_service", "hard");
await resolveOpenIncidentsForPolicy(row.id, row.companyId, "approved", actorUserId ?? "budget_service", "hard");
if (row.notifyEnabled && compareCents(observed.totalExact, Math.ceil(row.amount * row.warnPercent / 100)) >= 0) {
await createIncidentIfNeeded(row, "soft", observed);
} else {
await resolveOpenIncidentsForPolicy(row.id, null, null, "soft");
await resolveOpenIncidentsForPolicy(row.id, row.companyId, null, null, "soft");
}
} else {
const softThreshold = Math.ceil((row.amount * row.warnPercent) / 100);
@@ -856,14 +886,14 @@ export function budgetServiceInTransaction(db: Db, publications: ActivityPublica
await createIncidentIfNeeded(row, "soft", observed);
}
if (observedBlocks(row, observed)) {
await resolveOpenSoftIncidents(row.id);
await resolveOpenSoftIncidents(row.id, row.companyId);
await createIncidentIfNeeded(row, "hard", observed);
await pauseAndCancelScopeForBudget(row);
}
}
} else {
await resumeScopeFromBudget(row);
await resolveOpenIncidentsForPolicy(row.id, "approved", actorUserId ?? "budget_service");
await resolveOpenIncidentsForPolicy(row.id, row.companyId, "approved", actorUserId ?? "budget_service");
}
await recordActivity({
@@ -925,6 +955,13 @@ export function budgetServiceInTransaction(db: Db, publications: ActivityPublica
if (incident.companyId !== companyId) throw notFound("Budget incident not found");
const policy = await getPolicyRow(incident.policyId);
if (policy.companyId !== companyId || policy.scopeType !== incident.scopeType || policy.scopeId !== incident.scopeId) {
throw notFound("Budget scope not found");
}
const scope = input.action === "keep_paused"
? await resolveSummaryScopeRecord(db, companyId, policy.scopeType as BudgetScopeType, policy.scopeId)
: await resolveScopeRecord(db, policy.scopeType as BudgetScopeType, policy.scopeId);
if (scope.companyId !== companyId) throw notFound("Budget scope not found");
if (input.action === "raise_budget_and_resume") {
const nextAmount = Math.max(0, Math.floor(input.amount ?? 0));
const observed = await computeObservedSpend(db, policy);
@@ -937,6 +974,7 @@ export function budgetServiceInTransaction(db: Db, publications: ActivityPublica
}
const now = new Date();
await notifyAttentionRetentionWork(db, { companyId, sourceKind: "budget_alert" });
await notifyDeliveryWork(db, DELIVERY_QUEUES.budgetEnforcement, companyId);
await db
.update(budgetPolicies)
@@ -1,3 +1,4 @@
import { notifyDeferredExecutionWork, notifyTaskRetrySettlement } from "../deferred-execution-notifications.js";
import { notifyNativeResponsePresentation } from "../native-response-presentation-notifications.js";
import { notifyChatInboundWakeNotices } from "../chat-inbound-wake-notice-notifications.js";
import { createFailedRetryNotices } from "./failed-retry-notices.js";
@@ -1424,6 +1425,7 @@ export function createChatFailedRunRetry(db: Db, options: ChatFailedRunRetryOpti
// Source authorization above checked every external receipt and its owner.
// Reset only this certified unstarted batch under the task's admission lock.
const skippedIds = source.taskRetryInput?.skippedActionIds ?? [];
await notifyDeferredExecutionWork(tx, { companyId: input.companyId });
await tx.update(agentWakeupRequests).set({ status: "deferred_issue_execution", runId: null, finishedAt: null,
payload: sql`${agentWakeupRequests.payload} - 'coalescedIntoWakeupRequestId'`, updatedAt: new Date(),
}).where(and(eq(agentWakeupRequests.companyId, input.companyId), eq(agentWakeupRequests.agentId, input.agentId),
@@ -1784,6 +1786,9 @@ export function createChatFailedRunRetry(db: Db, options: ChatFailedRunRetryOpti
const attemptCount = Number(candidate.result?.attemptCount ?? 0) + 1;
const settle = async (status: string, result: Record<string, unknown>) => {
const [current] = await db.transaction(async (tx) => {
if (typeof claimed.payload.taskRetryIntentId === "string") await notifyTaskRetrySettlement(tx, {
companyId: claimed.companyId, requestId: claimed.payload.taskRetryIntentId,
});
await notifyChatActionWork(tx, "failed_run_retry");
if (status === "processed") await notifyNativeResponsePresentation(tx, {
companyId: claimed.companyId, issueId: String(claimed.payload.issueId),
@@ -1,3 +1,4 @@
import { notifyDeferredExecutionWork } from "../deferred-execution-notifications.js";
import { z } from "zod";
import { and, asc, eq, gt, inArray, isNull, or, sql } from "drizzle-orm";
import { agentWakeupRequests, chatActions, chatConversations, chatEndpoints, heartbeatRuns, issues, type Db } from "@paperclipai/db";
@@ -91,6 +92,7 @@ export async function releaseUnadmittedTaskRetryInput(db: Db, input: { companyId
)).for("update");
if (receipt?.runId) return;
await notifyChatInboundWakeNotices(tx, { companyId: input.companyId, issueId: issue.id });
await notifyDeferredExecutionWork(tx, { companyId: input.companyId });
await tx.update(agentWakeupRequests).set({ status: "deferred_issue_execution", finishedAt: null,
payload: sql`${agentWakeupRequests.payload} - 'coalescedIntoWakeupRequestId'`, updatedAt: new Date(),
}).where(and(eq(agentWakeupRequests.companyId, input.companyId), eq(agentWakeupRequests.status, "coalesced"),
@@ -1,3 +1,5 @@
import { notifyStatusCardIssueChange } from "../../status-card-notifications.js";
import { notifyIssueRecovery } from "../../issue-recovery-notifications.js";
import { createProviderEffects, type ProviderEffectTarget } from "../provider-effects.js";
import type { EndpointRow, RuntimeContext, InboundRuntimeContext } from "../runtime-types.js";
@@ -78,10 +80,12 @@ export function createTelegramAttachmentFailure(db: Db, options: TelegramAttachm
})(),
);
const effect = await db.transaction(async (tx) => {
await notifyIssueRecovery(tx, { companyId: input.endpoint.companyId });
// Replace the optimistic attachment placeholder with an explicit safe
// Paperclip-visible failure. The message link stays authoritative for
// exact provider redelivery, while no agent wakeup is enqueued for an
// empty turn that contains no usable bytes.
await notifyStatusCardIssueChange(tx, input.endpoint.companyId);
await tx
.update(issueComments)
.set({ body: visibleFailure, updatedAt: new Date() })
@@ -1,3 +1,4 @@
import { notifyRunDispatch } from "./run-dispatch-notifications.js";
import { notifyIssueReviewState } from "./issue-review-state-notifications.js";
import { notifyNativeResponsePresentation } from "./native-response-presentation-notifications.js";
import { notifyChatResponseAvailability } from "./chat-response-availability.js";
@@ -11,6 +12,7 @@ import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notificatio
export type ChatRunMilestoneScope = { companyId: string; issueId?: string; runId?: string };
export async function notifyChatRunMilestones(tx: object, scope: ChatRunMilestoneScope) {
if (!scope.issueId && !scope.runId) await notifyRunDispatch(tx, scope);
if (scope.issueId || scope.runId) {
await notifyIssueReviewState(tx, { companyId: scope.companyId, issueId: scope.issueId,
runId: scope.issueId ? undefined : scope.runId });
+9
View File
@@ -1,3 +1,6 @@
import { notifyStatusCardIssueChange } from "./status-card-notifications.js";
import { notifyDeferredExecutionEligibility } from "./deferred-execution-notifications.js";
import { notifyRunDispatch } from "./run-dispatch-notifications.js";
import { notifyChatInboundWakeNotices } from "./chat-inbound-wake-notice-notifications.js";
import { notifyCompanyRetentionWork } from "./retention-work-notifications.js";
import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js";
@@ -332,10 +335,13 @@ export function companyService(db: Db, _budgetHooks: BudgetServiceHooks = {}) {
const renamedPrefix = await resolveRenamedIssuePrefix(tx, id, companyPatch);
if (companyPatch.status !== undefined && companyPatch.status !== existing.status) {
await notifyRunDispatch(tx, { companyId: id });
await notifyDeferredExecutionEligibility(tx, { companyId: id });
await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers);
await notifyDeliveryWork(tx, DELIVERY_QUEUES.skillSourceSync);
await notifyDeliveryWork(tx, DELIVERY_QUEUES.routineSchedules);
await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors);
await notifyStatusCardIssueChange(tx, id);
await notifyCompanyRetentionWork(tx, id);
await notifyChatInboundWakeNotices(tx, { companyId: id });
await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentLifecycle, id);
@@ -488,10 +494,13 @@ export function companyService(db: Db, _budgetHooks: BudgetServiceHooks = {}) {
const wasAlreadyArchived = existing.status === "archived";
if (!wasAlreadyArchived) {
await notifyRunDispatch(tx, { companyId: id });
await notifyDeferredExecutionEligibility(tx, { companyId: id });
await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers);
await notifyDeliveryWork(tx, DELIVERY_QUEUES.skillSourceSync);
await notifyDeliveryWork(tx, DELIVERY_QUEUES.routineSchedules);
await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors);
await notifyStatusCardIssueChange(tx, id);
await notifyCompanyRetentionWork(tx, id);
await notifyChatInboundWakeNotices(tx, { companyId: id });
await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentLifecycle, id);
+4
View File
@@ -1,3 +1,5 @@
import { notifyStatusCardEvaluation } from "./status-card-notifications.js";
import { notifyIssueRecovery } from "./issue-recovery-notifications.js";
import { notifyUnsafeWorkspaceExportRecovery } from "./unsafe-workspace-export-notifications.js";
import { notifyCancelledWorkspacePreservation } from "./cancelled-workspace-preservation-notifications.js";
import { notifyInstructionCollection } from "./instruction-collection-notifications.js";
@@ -71,12 +73,14 @@ export async function deleteCompany(db: Db, id: string) {
const [existing] = await tx.select({ id: companies.id }).from(companies)
.where(eq(companies.id, id)).for("no key update");
if (!existing) return null;
await notifyIssueRecovery(tx, { companyId: id });
await notifyChatCompanyConfiguration(tx, id);
await chatCompanyDeletion.deleteCompanyData(tx, id);
await notifyDeliveryWork(tx, DELIVERY_QUEUES.skillSourceSync);
await notifyDeliveryWork(tx, DELIVERY_QUEUES.skillPackages);
await notifyDeliveryWork(tx, DELIVERY_QUEUES.routineSchedules);
await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors);
await notifyStatusCardEvaluation(tx, id);
await notifyDeliveryWork(tx, DELIVERY_QUEUES.secretProposalExpiry, id);
await notifyToolReviewChange(tx, id);
await agentObservabilityService(tx as unknown as Db).removeForCompanyDeletion(id);
+7 -1
View File
@@ -1,3 +1,5 @@
import { notifyAttentionRetentionWork } from "./retention-work-notifications.js";
import { notifyIssueRecovery } from "./issue-recovery-notifications.js";
import { createAgentLifecycle } from "./agent-lifecycle.js";
import { agentAppearanceSchema, agentHarnessType, paperclipRunnerProfileForHarness, agentRunner, type AgentRunnerChoice } from "@paperclipai/shared";
import { resolveNewAgentRunnerForCompany } from "./agent-runner-selection.js";
@@ -6465,7 +6467,10 @@ export function companyPortabilityService(db: Db, storage?: StorageService) {
}
if (relationRows.length > 0) {
const relationCompanyId = targetCompany.id;
await db
await db.transaction(async tx => {
await notifyAttentionRetentionWork(tx, { companyId: relationCompanyId });
await notifyIssueRecovery(tx, { companyId: relationCompanyId });
await tx
.insert(issueRelations)
.values(relationRows.map((row) => ({
companyId: relationCompanyId,
@@ -6476,6 +6481,7 @@ export function companyPortabilityService(db: Db, storage?: StorageService) {
createdByUserId: actorUserId ?? null,
})))
.onConflictDoNothing();
});
}
}
+22 -3
View File
@@ -207,14 +207,20 @@ function priorityRank(priority: string | null | undefined) {
return index >= 0 ? index : ISSUE_PRIORITIES.length;
}
function updatedWithinStart(value: string | undefined, now = new Date()): Date | null {
/** Duration used by saved search membership and its deadline consumers. */
export function companySearchUpdatedWithinMs(value: string | undefined): number | null {
if (!value) return null;
const match = /^(\d+)(h|d|w|m)$/.exec(value);
if (!match) return null;
const amount = Number.parseInt(match[1]!, 10);
const unit = match[2];
const hours = unit === "h" ? amount : unit === "d" ? amount * 24 : unit === "w" ? amount * 24 * 7 : amount * 24 * 30;
return new Date(now.getTime() - hours * 60 * 60 * 1000);
return hours * 60 * 60 * 1000;
}
function updatedWithinStart(value: string | undefined, now = new Date()): Date | null {
const duration = companySearchUpdatedWithinMs(value);
return duration === null ? null : new Date(now.getTime() - duration);
}
function issueOnlyFiltersActive(query: CompanySearchQuery) {
@@ -542,7 +548,7 @@ export function companySearchService(db: Db) {
search: async (
companyId: string,
query: CompanySearchQuery,
options?: { issueReadCondition?: SQL<boolean>; projectReadCondition?: SQL<boolean> },
options?: { issueReadCondition?: SQL<boolean>; projectReadCondition?: SQL<boolean>; onNextMembershipChange?: (at: Date | null) => void },
): Promise<CompanySearchResponse> => {
const taskSearch = parseTaskSearch(query.q);
const normalizedQuery = taskSearch.normalizedQuery;
@@ -553,6 +559,7 @@ export function companySearchService(db: Db) {
const limit = query.limit;
const offset = query.offset;
if (!hasSearchText && !issueOnlyFiltersActive(query)) {
options?.onNextMembershipChange?.(null);
return {
query: query.q,
normalizedQuery,
@@ -721,6 +728,13 @@ export function companySearchService(db: Db) {
}
}
const membershipWindow = companySearchUpdatedWithinMs(query.updatedWithin);
if (options?.onNextMembershipChange && membershipWindow !== null && scopeIncludesIssues(scope)) {
// Includes rows outside the visible page: their expiry can move an
// offset/priority-ranked result into the watched set without a write.
branches.push(sql`SELECT 'membership-expiry' AS kind, min(m.updated_at)::text AS value,
0::int AS count ${countTail} FROM matched m ${branchWhere([scopeCond, ...facetsAll])}`);
}
const resultRows = await db.execute(sql`
${taskSearchCtes(companyId, taskSearch, scope !== "issues", and(...issueFilters), options?.issueReadCondition)}
${sql.join(branches, sql` UNION ALL `)}
@@ -732,6 +746,7 @@ export function companySearchService(db: Db) {
totals: { current: 0, unfiltered: 0, omit: {} },
};
const issueRowsRaw: IssueSearchRow[] = [];
let nextMembershipChange: Date | null = null;
for (const row of resultRows) {
if (row.kind === "result") {
issueRowsRaw.push({
@@ -756,6 +771,9 @@ export function companySearchService(db: Db) {
});
continue;
}
if (row.kind === "membership-expiry" && row.value && membershipWindow !== null) {
nextMembershipChange = new Date(new Date(row.value).getTime() + membershipWindow + 1);
}
const count = Number(row.count ?? 0);
if (row.kind === "type:issue") aggregates.typeCounts.issue = count;
else if (row.kind === "type:comment") aggregates.typeCounts.comment = count;
@@ -776,6 +794,7 @@ export function companySearchService(db: Db) {
aggregates.totals.omit[row.kind.slice("total:omit:".length) as CompanySearchIssueFilterKey] = count;
}
}
options?.onNextMembershipChange?.(nextMembershipChange);
return { rows: await enrichIssueSnippets(issueRowsRaw), aggregates };
}
+2
View File
@@ -1,3 +1,4 @@
import { notifyStatusCardIssueChange } from "./status-card-notifications.js";
import { skillExecutionContent } from "./skill-execution-content.js";
import { notifySkillVersionRetention } from "./skill-version-retention-notifications.js";
import { managedSkillContent, type SkillContentPublication } from "./managed-skill-content.js";
@@ -7340,6 +7341,7 @@ export function companySkillService(db: Db, options: { actor?: AuthorizationActo
for (const row of rows) {
await db.transaction(async (tx) => {
await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers);
await notifyStatusCardIssueChange(tx, companyId);
await tx
.update(issues)
.set({ hiddenAt: now, updatedAt: now })
+8 -5
View File
@@ -26,7 +26,7 @@ export function registerDeadlineWork(
pausePassiveMaintenance: true,
onSettled: hints => { requested++; task.onSettled?.(hints); },
run: async signal => {
if (!task.canRun()) { requested++; next = null; return; }
if (!task.canRun()) { requested++; return; }
if (Date.now() < earliest) { next = earliest; return; }
if (Date.now() < coalesceUntil && (next === null || next > Date.now())) {
next = next === null ? coalesceUntil : Math.min(next, coalesceUntil);
@@ -39,8 +39,6 @@ export function registerDeadlineWork(
if (signal.aborted || !task.canRun()) { next = null; return; }
const deadline = await task.next();
if (!task.canRun()) { next = null; return; }
earliest = deadline === null || deadline > Date.now()
? (task.minimumSpacingMs ? Date.now() + task.minimumSpacingMs : 0)
@@ -49,8 +47,13 @@ export function registerDeadlineWork(
coalesceUntil = Date.now() + (task.notificationCoalesceMs ?? 0);
reconciled = generation;
},
hasPending: async () => requested !== reconciled,
nextRunAt: () => next,
// A settled producer must release its admission hold during drain. Keep
// the unreconciled generation and an in-memory resume wake, without asking
// the domain to probe its database while dispatch is closed.
hasPending: async () => task.canRun() && requested !== reconciled,
nextRunAt: () => !task.canRun() && requested !== reconciled
? Date.now() + task.retryMs
: next,
});
return { ...worker, wake: () => { requested++; worker.wake(); } };
}
+1 -1
View File
@@ -23,7 +23,7 @@ export function decisionExpiryService(db: Db, deliverContinuation: (id: string)
agentId: row.originAgentId, runId: row.originRunId, action: "decision.expired", entityType: "decision",
entityId: row.id, details: { expiredReason: reason } }, publications);
return row;
});
}, { companyId: decision.companyId, sourceId: decision.id });
if (!updated) return 0;
for (const publication of publications) publishActivity(publication);
await deliverContinuation(updated.id);
+29 -14
View File
@@ -1,3 +1,4 @@
import { isProspectiveBlockedTransition } from "./routable-blocked.js";
import { notifyRetentionWork } from "./retention-work-notifications.js";
import { decisionArchiveDeliveryService, type ArchiveNotificationBatch } from "./decision-archive-delivery.js";
import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js";
@@ -15,7 +16,6 @@ import {
issueRecoveryActions,
issueThreadInteractions,
issues,
joinRequests,
} from "@paperclipai/db";
import type {
AttentionArchiveManifestEntry,
@@ -145,7 +145,7 @@ export function decisionRetentionService(
const existing = new Set(rows.map(row => sourceKey(row.sourceKind, row.sourceId)));
const missing = unique.filter(item => !existing.has(itemSourceKey(item)));
if (missing.length) {
for (const kind of new Set(missing.map(item => item.sourceKind))) await notifyRetentionWork(tx, kind);
for (const item of missing) await notifyRetentionWork(tx, item.sourceKind, { companyId, sourceId: item.subject.id });
await tx.insert(decisionRetention).values(missing.map(item => ({
companyId, sourceKind: item.sourceKind, sourceId: item.subject.id, sourceActivityAt: new Date(item.activityAt),
}))).onConflictDoNothing({ target: [decisionRetention.companyId, decisionRetention.sourceKind, decisionRetention.sourceId] });
@@ -155,7 +155,7 @@ export function decisionRetentionService(
for (const item of unique) {
const key = itemSourceKey(item), row = byKey.get(key), activityAt = new Date(item.activityAt);
if (!row || row.sourceActivityAt.getTime() === activityAt.getTime()) continue;
await notifyRetentionWork(tx, item.sourceKind);
await notifyRetentionWork(tx, item.sourceKind, { companyId, sourceId: item.subject.id });
const [updated] = await tx.update(decisionRetention).set({
sourceActivityAt: activityAt, version: sql`${decisionRetention.version} + 1`, updatedAt: new Date(),
}).where(and(eq(decisionRetention.id, row.id), ne(decisionRetention.sourceActivityAt, activityAt))).returning();
@@ -169,9 +169,9 @@ export function decisionRetentionService(
return sourceKey(item.sourceKind, item.subject.id);
}
async function writeRetention<T>(kind: AttentionSourceKind, write: (tx: Db) => Promise<T>) {
async function writeRetention<T>(companyId: string, kind: AttentionSourceKind, sourceId: string, write: (tx: Db) => Promise<T>) {
return db.transaction(async tx => {
await notifyRetentionWork(tx, kind);
await notifyRetentionWork(tx, kind, { companyId, sourceId });
return write(tx as unknown as Db);
});
}
@@ -185,13 +185,28 @@ export function decisionRetentionService(
}
async function materializeSourceState(database: Db, input: { companyId: string; sourceKind: AttentionSourceKind; sourceId: string }) {
const table = input.sourceKind === "approval" ? approvals : input.sourceKind === "join_request" ? joinRequests : null;
if (!table) return;
const [source] = await database.select({ updatedAt: table.updatedAt }).from(table)
.where(and(eq(table.companyId, input.companyId), eq(table.id, input.sourceId))).for("update");
const tables: Record<AttentionSourceKind, string> = {
approval: "approvals", join_request: "join_requests", decision: "decisions",
issue_thread_interaction: "issue_thread_interactions", recovery_action: "issue_recovery_actions",
failed_run: "heartbeat_runs", productivity_review: "issues", blocker_attention: "issues", review: "issues",
budget_alert: "budget_incidents", agent_error_alert: "agents",
};
if (input.sourceKind === "blocker_attention") {
const [source] = await database.select().from(issues).where(and(eq(issues.companyId, input.companyId), eq(issues.id, input.sourceId))).for("update");
if (!source) throw notFound("Attention source not found");
const activityAt = source.unblockDescriptor?.owner === "board" && isProspectiveBlockedTransition(source)
? source.blockedTransitionAt : source.updatedAt;
await decisionRetentionService(database).syncItems(input.companyId, [{
sourceKind: input.sourceKind, subject: { id: input.sourceId }, activityAt: activityAt.toISOString(),
}]);
return;
}
const activity = input.sourceKind === "failed_run" ? sql`coalesce(finished_at, updated_at, created_at)` : sql`updated_at`;
const [source] = await database.execute(sql`select ${activity} as "activityAt" from ${sql.identifier(tables[input.sourceKind])}
where company_id = ${input.companyId}::uuid and id = ${input.sourceId}::uuid for update`) as unknown as Array<{ activityAt: Date | string }>;
if (!source) throw notFound("Attention source not found");
await decisionRetentionService(database).syncItems(input.companyId, [{
sourceKind: input.sourceKind, subject: { id: input.sourceId }, activityAt: source.updatedAt.toISOString(),
sourceKind: input.sourceKind, subject: { id: input.sourceId }, activityAt: new Date(source.activityAt).toISOString(),
}]);
}
@@ -206,7 +221,7 @@ export function decisionRetentionService(
if (!(await canReadDecisionSource(db, input.authActor, input.companyId, input.sourceKind, input.sourceId))) {
throw notFound("Attention source not found");
}
const updated = await writeRetention(input.sourceKind, async tx => {
const updated = await writeRetention(input.companyId, input.sourceKind, input.sourceId, async tx => {
await materializeSourceState(tx, input);
return tx.update(decisionRetention).set({
keep: input.keep,
@@ -247,7 +262,7 @@ export function decisionRetentionService(
throw notFound("Attention source not found");
}
const now = new Date();
const updated = await writeRetention(input.sourceKind, async tx => {
const updated = await writeRetention(input.companyId, input.sourceKind, input.sourceId, async tx => {
await materializeSourceState(tx, input);
return tx.update(decisionRetention).set(input.archived ? {
archivedAt: now,
@@ -324,7 +339,7 @@ export function decisionRetentionService(
}
const now = new Date();
for (const entry of input.manifest) {
await notifyRetentionWork(txDb, entry.sourceKind as AttentionSourceKind);
await notifyRetentionWork(txDb, entry.sourceKind as AttentionSourceKind, { companyId: input.companyId, sourceId: entry.sourceId });
const archived = await txDb.update(decisionRetention).set({
archivedAt: now,
archivedReason: "accepted_proposal",
@@ -368,7 +383,7 @@ export function decisionRetentionService(
await db.transaction(async (tx) => {
const txDb = tx as unknown as Db;
for (const item of candidates) {
await notifyRetentionWork(txDb, item.sourceKind);
await notifyRetentionWork(txDb, item.sourceKind, { companyId: input.companyId, sourceId: item.subject.id });
const updated = await txDb.update(decisionRetention).set({
archivedAt: now,
archivedReason: "idle_ttl",
@@ -1,18 +1,23 @@
import { notifyAttentionRetentionWork } from "./retention-work-notifications.js";
import type { Db } from "@paperclipai/db";
import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js";
export function notifyDecisionCreated(transaction: object) {
type DecisionWorkScope = { companyId: string; sourceId?: string };
export async function notifyDecisionCreated(transaction: object, scope: DecisionWorkScope) {
await notifyAttentionRetentionWork(transaction, { ...scope, sourceKind: "decision" });
return notifyDeliveryWork(transaction, DELIVERY_QUEUES.decisionExpiry);
}
export async function notifyDecisionChange(transaction: object) {
export async function notifyDecisionChange(transaction: object, scope: DecisionWorkScope) {
await notifyAttentionRetentionWork(transaction, { ...scope, sourceKind: "decision" });
await notifyDeliveryWork(transaction, DELIVERY_QUEUES.decisionContinuation);
await notifyDeliveryWork(transaction, DELIVERY_QUEUES.decisionExpiry);
}
export function writeDecisionChange<T>(db: Db, write: (transaction: Db) => Promise<T>): Promise<T> {
export function writeDecisionChange<T>(db: Db, write: (transaction: Db) => Promise<T>, scope: DecisionWorkScope): Promise<T> {
return db.transaction(async transaction => {
await notifyDecisionChange(transaction);
await notifyDecisionChange(transaction, scope);
await notifyDeliveryWork(transaction, DELIVERY_QUEUES.decisionEffectRecovery);
return write(transaction as unknown as Db);
});
+13 -12
View File
@@ -239,7 +239,7 @@ export function decisionService(db: Db, options: DecisionServiceOptions) {
...(input.additionalTargetSnapshots ?? {}),
};
const id = randomUUID();
await notifyDecisionCreated(dbOrTx);
await notifyDecisionCreated(dbOrTx, { companyId: input.companyId, sourceId: id });
await notifyDecisionTargetWork(dbOrTx, `decision:${id}`);
const [created] = await dbOrTx.insert(decisions).values({ id, companyId: input.companyId, bundleId: input.bundleId ?? null,
originAgentId: input.agentId, originIssueId: provenance.issueId, originRunId: input.runId, ruleKey: input.ruleKey ?? null,
@@ -581,7 +581,8 @@ export function decisionService(db: Db, options: DecisionServiceOptions) {
return attempt;
}
async function settleInvalidEffectOutcome(id: string) {
async function settleInvalidEffectOutcome(decision: typeof decisions.$inferSelect) {
const id = decision.id;
const publications: ActivityPublication[] = [];
await writeDecisionChange(db, async tx => {
const [current] = await tx.select().from(decisions).where(eq(decisions.id, id)).for("update");
@@ -601,14 +602,14 @@ export function decisionService(db: Db, options: DecisionServiceOptions) {
action: "decision.recovery_failed", entityType: "decision", entityId: id,
details: { reason, executionStatus, chosenOptionId: current.chosenOptionId, decidedByUserId: current.decidedByUserId },
}, publications);
});
}, { companyId: decision.companyId, sourceId: id });
for (const publication of publications) publishActivity(publication);
return outcome(id);
}
async function executeDecisionEffects(decision: typeof decisions.$inferSelect, userActor: AuthorizationActor) {
const option = decision.options.find((item) => item.id === decision.chosenOptionId);
if (!option || !decision.decidedByUserId) return settleInvalidEffectOutcome(decision.id);
if (!option || !decision.decidedByUserId) return settleInvalidEffectOutcome(decision);
const run = await db.select({ responsibleUserId: heartbeatRuns.responsibleUserId }).from(heartbeatRuns).where(eq(heartbeatRuns.id, decision.originRunId)).then((rows) => rows[0] ?? null);
const metadata = decision.metadata as Record<string, unknown>;
if (metadata.kind === "attention_archive_proposal") {
@@ -618,7 +619,7 @@ export function decisionService(db: Db, options: DecisionServiceOptions) {
targetSnapshots: decision.targetSnapshots as Record<string, Snapshot>,
}), decision.signedSpec)) {
await writeDecisionChange(db, async tx => tx.update(decisions).set({ executionStatus: "failed", updatedAt: new Date(), metadata: { ...metadata, archiveProposalError: "invalid_signature", ...(decision.continuationPolicy === "wake_origin_agent" ? { continuationPending: true } : {}) } })
.where(and(eq(decisions.id, decision.id), eq(decisions.executionStatus, "running"))));
.where(and(eq(decisions.id, decision.id), eq(decisions.executionStatus, "running"))), { companyId: decision.companyId, sourceId: decision.id });
return outcome(decision.id);
}
if (option.id === "archive") {
@@ -629,7 +630,7 @@ export function decisionService(db: Db, options: DecisionServiceOptions) {
const manifestHash = manifest ? hashAttentionArchiveManifest(manifest) : null;
if (!manifest || manifestHash !== metadata.manifestHash) {
await writeDecisionChange(db, async tx => tx.update(decisions).set({ executionStatus: "failed", updatedAt: new Date(), metadata: { ...metadata, archiveProposalError: "manifest_mismatch", ...(decision.continuationPolicy === "wake_origin_agent" ? { continuationPending: true } : {}) } })
.where(and(eq(decisions.id, decision.id), eq(decisions.executionStatus, "running"))));
.where(and(eq(decisions.id, decision.id), eq(decisions.executionStatus, "running"))), { companyId: decision.companyId, sourceId: decision.id });
return outcome(decision.id);
}
const originActor: AuthorizationActor = {
@@ -651,7 +652,7 @@ export function decisionService(db: Db, options: DecisionServiceOptions) {
} catch (error) {
const message = error instanceof Error ? error.message : "archive_proposal_failed";
await writeDecisionChange(db, async tx => tx.update(decisions).set({ executionStatus: "failed", updatedAt: new Date(), metadata: { ...metadata, archiveProposalError: message, ...(decision.continuationPolicy === "wake_origin_agent" ? { continuationPending: true } : {}) } })
.where(and(eq(decisions.id, decision.id), eq(decisions.executionStatus, "running"))));
.where(and(eq(decisions.id, decision.id), eq(decisions.executionStatus, "running"))), { companyId: decision.companyId, sourceId: decision.id });
return outcome(decision.id);
}
}
@@ -659,7 +660,7 @@ export function decisionService(db: Db, options: DecisionServiceOptions) {
executionStatus: "succeeded",
updatedAt: new Date(),
metadata: { ...metadata, ...(decision.continuationPolicy === "wake_origin_agent" ? { continuationPending: true } : {}) },
}).where(and(eq(decisions.id, decision.id), eq(decisions.executionStatus, "running"))));
}).where(and(eq(decisions.id, decision.id), eq(decisions.executionStatus, "running"))), { companyId: decision.companyId, sourceId: decision.id });
return outcome(decision.id);
}
for (let index = 0; index < option.effects.length; index += 1) await executeEffect(decision, option.effects[index]!, index, userActor, decision.decidedByUserId, run?.responsibleUserId ?? null);
@@ -667,7 +668,7 @@ export function decisionService(db: Db, options: DecisionServiceOptions) {
const successful = rows.filter((row) => row.status === "executed").length;
const status = rows.every((row) => row.status === "executed") ? "succeeded" : successful ? "partial" : "failed";
await writeDecisionChange(db, async tx => tx.update(decisions).set({ executionStatus: status, updatedAt: new Date(), metadata: { ...decision.metadata,
...(decision.continuationPolicy === "wake_origin_agent" ? { continuationPending: true } : {}) } }).where(and(eq(decisions.id, decision.id), eq(decisions.executionStatus, "running"))));
...(decision.continuationPolicy === "wake_origin_agent" ? { continuationPending: true } : {}) } }).where(and(eq(decisions.id, decision.id), eq(decisions.executionStatus, "running"))), { companyId: decision.companyId, sourceId: decision.id });
return outcome(decision.id);
}
@@ -706,7 +707,7 @@ export function decisionService(db: Db, options: DecisionServiceOptions) {
decideIdempotencyKey: input.idempotencyKey ?? null,
...(current.continuationPolicy === "wake_origin_agent" ? { continuationPending: true } : {}),
...(input.dismissed ? { dismissed: true, dismissReason: input.dismissReason ?? null } : {}) } })
.where(and(eq(decisions.id, current.id), eq(decisions.status, "open"))).returning());
.where(and(eq(decisions.id, current.id), eq(decisions.status, "open"))).returning(), { companyId: current.companyId, sourceId: current.id });
if (!claimed) throw conflict("decision_already_resolved", { code: "decision_already_resolved" });
const run = await db.select({ responsibleUserId: heartbeatRuns.responsibleUserId }).from(heartbeatRuns).where(eq(heartbeatRuns.id, claimed.originRunId)).then((rows) => rows[0] ?? null);
await logActivity(db, { companyId: claimed.companyId, actorType: "system", actorId: "decision-executor", agentId: claimed.originAgentId, runId: claimed.originRunId,
@@ -724,7 +725,7 @@ export function decisionService(db: Db, options: DecisionServiceOptions) {
return (await get(id))!;
}
const [updated] = await writeDecisionChange(db, async tx => tx.update(decisions).set({ status: "cancelled", updatedAt: new Date(), metadata: { ...current.metadata,
...(current.continuationPolicy === "wake_origin_agent" ? { continuationPending: true } : {}) } }).where(and(eq(decisions.id, id), eq(decisions.status, "open"))).returning());
...(current.continuationPolicy === "wake_origin_agent" ? { continuationPending: true } : {}) } }).where(and(eq(decisions.id, id), eq(decisions.status, "open"))).returning(), { companyId: current.companyId, sourceId: id });
if (!updated) throw conflict("decision_already_resolved", { code: "decision_already_resolved" });
await logActivity(db, { companyId: updated.companyId, actorType: actor.actorType, actorId: actor.actorId, runId: actor.runId, action: "decision.cancelled", entityType: "decision", entityId: id });
await continuations.deliver(updated.id);
@@ -738,7 +739,7 @@ export function decisionService(db: Db, options: DecisionServiceOptions) {
if (empty) return decide({ id, optionId: empty.id, decidedByUserId: userId, userActor, dismissed: true, dismissReason: reason });
const [updated] = await writeDecisionChange(db, async tx => tx.update(decisions).set({ status: "decided", executionStatus: "succeeded", chosenOptionId: "dismissed", decidedByUserId: userId,
decidedAt: new Date(), updatedAt: new Date(), metadata: { ...current.metadata, dismissed: true, dismissReason: reason ?? null,
...(current.continuationPolicy === "wake_origin_agent" ? { continuationPending: true } : {}) } }).where(and(eq(decisions.id, id), eq(decisions.status, "open"))).returning());
...(current.continuationPolicy === "wake_origin_agent" ? { continuationPending: true } : {}) } }).where(and(eq(decisions.id, id), eq(decisions.status, "open"))).returning(), { companyId: current.companyId, sourceId: id });
if (!updated) throw conflict("decision_already_resolved", { code: "decision_already_resolved" });
await logActivity(db, { companyId: updated.companyId, actorType: "system", actorId: "decision-executor", agentId: updated.originAgentId,
runId: updated.originRunId, responsibleUserIdOverride: userId, action: "decision.dismissed", entityType: "decision", entityId: updated.id,
@@ -0,0 +1,34 @@
import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js";
export type DeferredExecutionScope = { companyId: string; requestId?: string };
export type DeferredReceiptRecoveryOptions = {
companyId?: string;
requestId?: string;
onError?: (error: unknown) => void;
};
function hint(scope: DeferredExecutionScope) {
return JSON.stringify({ companyId: scope.companyId, requestId: scope.requestId });
}
export async function notifyDeferredExecutionWork(tx: object, scope: DeferredExecutionScope) {
await notifyDeliveryWork(tx, DELIVERY_QUEUES.deferredExecutionReceipts, hint(scope));
}
export async function notifyNativeStatusWakeWork(tx: object, scope: DeferredExecutionScope) {
await notifyDeliveryWork(tx, DELIVERY_QUEUES.nativeStatusWakeIntents, hint(scope));
}
export async function notifyTaskRetrySettlement(tx: object, scope: DeferredExecutionScope) {
await notifyDeliveryWork(tx, DELIVERY_QUEUES.taskRetryReceiptSettlement, hint(scope));
}
export async function notifyDeferredExecutionEligibility(tx: object, scope?: { companyId: string }) {
if (scope) {
await notifyDeferredExecutionWork(tx, scope);
await notifyNativeStatusWakeWork(tx, scope);
} else {
await notifyDeliveryWork(tx, DELIVERY_QUEUES.deferredExecutionReceipts, "*");
await notifyDeliveryWork(tx, DELIVERY_QUEUES.nativeStatusWakeIntents, "*");
}
}
@@ -0,0 +1,24 @@
import { and, eq, sql } from "drizzle-orm";
import { agentWakeupRequests, type Db } from "@paperclipai/db";
import { notifyDeferredExecutionWork } from "./deferred-execution-notifications.js";
export async function writeDeferredExecutionWait(db: Db, scope: { companyId: string; requestId: string },
condition: Record<string, unknown>, options: { replace?: boolean } = {}) {
return db.transaction(async tx => {
const [current] = await tx.select({ payload: agentWakeupRequests.payload }).from(agentWakeupRequests)
.where(and(eq(agentWakeupRequests.companyId, scope.companyId), eq(agentWakeupRequests.id, scope.requestId),
eq(agentWakeupRequests.status, "deferred_issue_execution"))).for("update");
if (!current) return;
const previous = current.payload?.executionWait;
const previousReason = previous && typeof previous === "object" && !Array.isArray(previous)
? (previous as Record<string, unknown>).reason : undefined;
if (!previous || previousReason !== condition.reason) await notifyDeferredExecutionWork(tx, scope);
await tx.update(agentWakeupRequests).set({
payload: sql`jsonb_set(coalesce(${agentWakeupRequests.payload}, '{}'::jsonb), '{executionWait}',
${options.replace ? sql`'{}'::jsonb` : sql`coalesce(${agentWakeupRequests.payload}->'executionWait', '{}'::jsonb)`}
|| ${JSON.stringify(condition)}::jsonb)`,
updatedAt: new Date(),
}).where(and(eq(agentWakeupRequests.companyId, scope.companyId), eq(agentWakeupRequests.id, scope.requestId),
eq(agentWakeupRequests.status, "deferred_issue_execution")));
});
}
@@ -2,6 +2,18 @@ import { signalDatabaseWork, subscribeDatabaseWork } from "@paperclipai/db";
/** Topics for existing durable queues; register intent before their writes. */
export const DELIVERY_QUEUES = {
deferredExecutionReceipts: "deferred-execution-receipts",
taskRetryReceiptSettlement: "task-retry-receipt-settlement",
nativeStatusWakeIntents: "native-status-wake-intents",
orphanBlockers: "orphan-blockers",
activeRecoveryActions: "active-recovery-actions",
strandedAssignments: "stranded-assignments",
resolvedDependencies: "resolved-dependencies",
taskWatchdogs: "task-watchdogs",
staleIssueLocks: "stale-issue-locks",
scheduledRetryDispatch: "scheduled-retry-dispatch",
queuedRunDispatch: "queued-run-dispatch",
statusCardEvaluation: "status-card-evaluation",
runEvidenceMigration: "run-evidence-migration",
providerTraceRetention: "provider-trace-retention",
nativeRunnerEvidenceRetention: "native-runner-evidence-retention",
@@ -35,6 +47,7 @@ export const DELIVERY_QUEUES = {
executionDisposition: "execution-disposition",
environmentCleanup: "environment-cleanup",
approvalQueueSeeding: "approval-queue-seeding",
attentionRetention: "attention-retention",
approvalRetention: "approval-retention",
joinRequestRetention: "join-request-retention",
decisionTargets: "decision-targets",
+16
View File
@@ -1,3 +1,5 @@
import { notifyStatusCardIssueChange } from "./status-card-notifications.js";
import { notifyIssueRecovery } from "./issue-recovery-notifications.js";
import { randomUUID } from "node:crypto";
import { and, asc, desc, eq, sql } from "drizzle-orm";
import type { Db } from "@paperclipai/db";
@@ -238,6 +240,7 @@ export function documentService(db: Db) {
for (let attempt = 0; attempt < maxAttempts; attempt += 1) {
try {
return await db.transaction(async (tx) => {
await notifyStatusCardIssueChange(tx, issue.companyId);
const now = new Date();
const existing = await tx
.select({
@@ -275,6 +278,7 @@ export function documentService(db: Db) {
.where(eq(issueDocuments.issueId, issue.id));
const fallbackKey = nextAvailableDocumentKey(key, issueDocumentKeys.map((row) => row.key));
await notifyIssueRecovery(tx, { companyId: issue.companyId });
const [document] = await tx
.insert(documents)
.values({
@@ -376,6 +380,7 @@ export function documentService(db: Db) {
});
}
await notifyIssueRecovery(tx, { companyId: issue.companyId });
const nextRevisionNumber = existing.latestRevisionNumber + 1;
const [revision] = await tx
.insert(documentRevisions)
@@ -438,6 +443,7 @@ export function documentService(db: Db) {
throw conflict("Document does not exist yet", { key });
}
await notifyIssueRecovery(tx, { companyId: issue.companyId });
const [document] = await tx
.insert(documents)
.values({
@@ -592,6 +598,10 @@ export function documentService(db: Db) {
});
}
await db.transaction(async (tx) => {
for (const companyId of new Set(rows.map(row => row.companyId))) {
await notifyStatusCardIssueChange(tx, companyId);
await notifyIssueRecovery(tx, { companyId });
}
await insertRowsInChunks(tx, documents, documentRows);
await insertRowsInChunks(tx, documentRevisions, revisionRows);
await insertRowsInChunks(tx, issueDocuments, linkRows);
@@ -644,6 +654,7 @@ export function documentService(db: Db) {
});
}
await notifyIssueRecovery(tx, { companyId: existing.companyId });
const now = new Date();
const nextRevisionNumber = existing.latestRevisionNumber + 1;
const [restoredRevision] = await tx
@@ -662,6 +673,7 @@ export function documentService(db: Db) {
})
.returning();
await notifyStatusCardIssueChange(tx, existing.companyId);
await tx
.update(documents)
.set({
@@ -722,6 +734,7 @@ export function documentService(db: Db) {
};
}
await notifyIssueRecovery(tx, { companyId: existing.companyId });
const now = new Date();
await tx
.update(documents)
@@ -769,6 +782,7 @@ export function documentService(db: Db) {
};
}
await notifyIssueRecovery(tx, { companyId: existing.companyId });
const now = new Date();
await tx
.update(documents)
@@ -817,7 +831,9 @@ export function documentService(db: Db) {
});
}
await notifyIssueRecovery(tx, { companyId: existing.companyId });
await tx.delete(issueDocuments).where(eq(issueDocuments.documentId, existing.id));
await notifyStatusCardIssueChange(tx, existing.companyId);
await tx.delete(documents).where(eq(documents.id, existing.id));
return {
+7 -1
View File
@@ -15,6 +15,7 @@ import { hasStopOnlyCleanup, prepareSandboxStopAndRetain, readStopOnlyCleanup, s
import { JsonRpcCallError, readEnvironmentAcquisitionDiagnostic, readEnvironmentCreationCleanupError } from "@paperclipai/plugin-sdk";
import { preserveEnvironmentSyncOutErrorDiagnostic } from "./environment-sync-out-error.js";
import { remoteTerminationReceipt } from "./remote-execution-termination.js";
import { inspectCloudSandboxCleanupAbsence } from "./sandbox-absence-proof.js";
import { hasNativeWorkspaceExportResume, releaseCompletedNativeWorkspaceExportRetention } from "./native-runtime/native-workspace-export-resume.js";
import { createHash, randomUUID } from "node:crypto";
import { and, eq, inArray, ne, or, sql } from "drizzle-orm";
@@ -4021,7 +4022,12 @@ export function environmentRuntimeService(
const lease = await environmentsSvc.getLeaseById(leaseId);
if (!lease || lease.status !== "failed" || lease.leasePolicy !== "ephemeral" || !lease.providerLeaseId) return undefined;
const environment = lease.environmentId ? await environmentsSvc.getById(lease.environmentId) : null;
return await getDriver(getLeaseDriverKey(lease, environment))?.inspectLeaseResource?.({ environment, lease });
// Legacy Cloud leases can lack a credential version that permits a
// provider-local observation. Cloud's manager verifies the historical
// organization and signs only exact absence; no replacement key or
// cleanup label is treated as proof. Explicit provider answers win.
return await getDriver(getLeaseDriverKey(lease, environment))?.inspectLeaseResource?.({ environment, lease })
?? await inspectCloudSandboxCleanupAbsence(lease);
},
async retryPendingSandboxTeardown(input: {
@@ -1,4 +1,8 @@
import { notifyRoutineExecutionStatus } from "./routine-execution-status-notifications.js";
import { notifyAttentionRetentionWork } from "./retention-work-notifications.js";
import { notifyStatusCardIssueChange } from "./status-card-notifications.js";
import { notifyDeferredExecutionEligibility } from "./deferred-execution-notifications.js";
import { notifyRunDispatch } from "./run-dispatch-notifications.js";
import type { Db } from "@paperclipai/db";
import { DELIVERY_QUEUES, notifyDeliveryWork, type DeliveryQueue } from "./delivery-work-notifications.js";
@@ -6,6 +10,12 @@ export type ExecutionWorkScope = { companyId: string; agentId?: string; runId?:
export async function notifyExecutionRunWork(transaction: object, scope?: ExecutionWorkScope) {
if (scope) await notifyRoutineExecutionStatus(transaction, scope.issueId, scope.companyId);
if (scope) {
await notifyAttentionRetentionWork(transaction, scope);
await notifyStatusCardIssueChange(transaction, scope.companyId);
await notifyRunDispatch(transaction, scope);
await notifyDeferredExecutionEligibility(transaction, scope);
}
for (const queue of [DELIVERY_QUEUES.executionStatus, DELIVERY_QUEUES.executionFinalization,
DELIVERY_QUEUES.executionReplacement, DELIVERY_QUEUES.executionDisposition, DELIVERY_QUEUES.issueMonitors]) {
await notifyDeliveryWork(transaction, queue);
@@ -20,6 +30,10 @@ export function writeExecutionControl<T>(
): Promise<T> {
return db.transaction(async transaction => {
if (scope) await notifyRoutineExecutionStatus(transaction, scope.issueId, scope.companyId);
if (scope) await notifyAttentionRetentionWork(transaction, scope);
if (scope) await notifyStatusCardIssueChange(transaction, scope.companyId);
await notifyRunDispatch(transaction, scope);
if (scope) await notifyDeferredExecutionEligibility(transaction, scope);
await notifyDeliveryWork(transaction, DELIVERY_QUEUES.issueMonitors);
for (const topic of typeof queue === "string" ? [queue] : queue) await notifyDeliveryWork(transaction, topic);
return write(transaction as unknown as Db);
+8 -1
View File
@@ -1,3 +1,4 @@
import { notifyStatusCardIssueChange } from "./status-card-notifications.js";
import { effectiveProjectWorkspacePolicy } from "./project-workspace-provider.js";
import { inspectPluginWorkspace, removePluginWorkspace, restorePluginWorkspace, type WorkspaceWorkers } from "./plugin-workspaces.js";
import { acquireExecutionWorkspaceLifecycleLock, withExecutionWorkspaceLifecycleLock } from "./execution-workspace-lifecycle-lock.js";
@@ -1590,6 +1591,7 @@ export function executionWorkspaceService(db: Db, opts: ExecutionWorkspaceServic
await validateSelection(input, tx);
const intent: TaskWorkspaceIntent = { version: 1, request: { key: input.requestKey, expectedBindingRevision: input.expectedBindingRevision }, selection: input.selection, source: input.source ?? "explicit" };
const nextRevision = task.workspaceBindingRevision + 1;
await notifyStatusCardIssueChange(tx, task.companyId);
await tx.update(issues).set({ workspaceBindingRevision: nextRevision,
workspacePendingSelection: { version: 1, requestKey: input.requestKey,
expectedBindingRevision: nextRevision, intent }, updatedAt: new Date() }).where(eq(issues.id, task.id));
@@ -1633,6 +1635,7 @@ export function executionWorkspaceService(db: Db, opts: ExecutionWorkspaceServic
}
}
}
await notifyStatusCardIssueChange(tx, task.companyId);
await tx.update(issues).set({ ...patch, assigneeAdapterOverrides, workspaceSelection: pending.intent, workspacePendingSelection: null,
workspaceBindingRevision: task.workspaceBindingRevision + 1, updatedAt: new Date() }).where(eq(issues.id, task.id));
return true;
@@ -1651,6 +1654,7 @@ export function executionWorkspaceService(db: Db, opts: ExecutionWorkspaceServic
const pendingAfterFirstBinding = !task.executionWorkspaceId && pending?.expectedBindingRevision === task.workspaceBindingRevision
? { ...pending, expectedBindingRevision: nextRevision }
: pending;
await notifyStatusCardIssueChange(tx, companyId);
const [bound] = await tx.update(issues).set({ ...patch, executionWorkspaceId: workspaceId,
workspaceBindingRevision: nextRevision, workspacePendingSelection: pendingAfterFirstBinding, updatedAt: new Date() })
.where(eq(issues.id, issueId)).returning();
@@ -2526,13 +2530,16 @@ export function executionWorkspaceService(db: Db, opts: ExecutionWorkspaceServic
forceWorktreeRemoval: false,
});
if (cleanup.cleaned && workspace.mode === "shared_workspace") {
await db
await db.transaction(async tx => {
await notifyStatusCardIssueChange(tx, workspace.companyId);
await tx
.update(issues)
.set({ executionWorkspaceId: null, updatedAt: now() })
.where(and(
eq(issues.companyId, workspace.companyId),
eq(issues.executionWorkspaceId, workspace.id),
));
});
}
const cleanupReason = [ISSUE_TERMINAL_WORKSPACE_CLEANUP_REASON, ...cleanup.warnings].join(" | ");
if (!cleanup.cleaned || cleanup.warnings.length > 0) {
@@ -1,7 +1,9 @@
import { notifyIssueRecovery } from "./issue-recovery-notifications.js";
import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js";
import type { ExternalConversationSettlementScope } from "./slack-conversation-lifecycle.js";
export function notifyExternalConversationSettlement(tx: object, scope?: ExternalConversationSettlementScope) {
export async function notifyExternalConversationSettlement(tx: object, scope?: ExternalConversationSettlementScope) {
await notifyIssueRecovery(tx, scope);
return notifyDeliveryWork(tx, DELIVERY_QUEUES.externalConversationSettlement,
scope ? JSON.stringify({ companyId: scope.companyId, issueId: scope.issueId }) : "*");
}
@@ -1,3 +1,4 @@
import { notifyAttentionRetentionWork } from "../retention-work-notifications.js";
import { and, eq, inArray, isNotNull, or, sql } from "drizzle-orm";
import { dotRunnerAssignments, environmentLeases, heartbeatRuns, issueRecoveryActions, issues, nativeRunFinalizations, type Db } from "@paperclipai/db";
import { authorizeExternalBotOperator } from "../external-bot-authority.js";
@@ -74,6 +75,7 @@ export async function inspectGrokWorkspaceObligations(db: Db, companyId: string,
return { ...rest, externalWorkspaceInspection: inspection };
};
await tx.update(heartbeatRuns).set({ resultJson: retireFileObligation(run.resultJson!) }).where(eq(heartbeatRuns.id, run.id));
if (actions.length) await notifyAttentionRetentionWork(tx, { companyId, sourceKind: "recovery_action" });
for (const action of actions) {
await tx.update(issueRecoveryActions).set({ evidence: retireFileObligation(action.evidence), updatedAt: new Date(),
nextAction: "Inspect Grok and verify its connection before sending a new task message. The old work will not be replayed.",
+7 -2
View File
@@ -1,3 +1,4 @@
import { notifyAttentionRetentionWork } from "./retention-work-notifications.js";
import { readNativeEventEvidenceOutcome } from "./native-runtime/evidence-classifier.js";
import { notifyNativeStatusReassessment } from "./native-status-reassessment-notifications.js";
import { isNativeResponseEvidence, notifyNativeResponsePresentation } from "./native-response-presentation-notifications.js";
@@ -9,6 +10,8 @@ import type { Db } from "@paperclipai/db";
import { heartbeatRunEvents, heartbeatRuns } from "@paperclipai/db";
import { nativeSha256 } from "./native-runtime/canonical.js";
export const RETRY_EXHAUSTION_MESSAGE_PREFIX = "Bounded retry exhausted";
export interface AppendHeartbeatRunEventInput {
companyId: string;
runId: string;
@@ -74,10 +77,11 @@ export async function appendHeartbeatRunEvent(
db: Db,
input: AppendHeartbeatRunEventInput,
): Promise<AppendHeartbeatRunEventResult> {
const projectsRetention = input.eventType === "lifecycle" && Boolean(input.message?.startsWith(RETRY_EXHAUSTION_MESSAGE_PREFIX));
const projectsReassessment = readNativeEventEvidenceOutcome(input.payload?.prpEvent) !== null;
const projectsProgress = isSafeNativeChatProgressEventType(input.eventType);
const projectsResponse = isNativeResponseEvidence(input.eventType, input.payload?.prpEvent);
if (!input.nativeSource && !input.retryExhaustion && !projectsProgress && !projectsResponse && !projectsReassessment) {
if (!input.nativeSource && !input.retryExhaustion && !projectsProgress && !projectsResponse && !projectsReassessment && !projectsRetention) {
const allocated = db.$with("allocated_run_event", {}).as(sql`
update ${heartbeatRuns}
set next_event_seq = ${heartbeatRuns.nextEventSeq} + 1,
@@ -129,7 +133,7 @@ export async function appendHeartbeatRunEvent(
eq(heartbeatRunEvents.runId, input.runId),
eq(heartbeatRunEvents.agentId, input.agentId),
eq(heartbeatRunEvents.eventType, "lifecycle"),
sql`${heartbeatRunEvents.message} like 'Bounded retry exhausted%'`,
sql`${heartbeatRunEvents.message} like ${`${RETRY_EXHAUSTION_MESSAGE_PREFIX}%`}`,
sql`${heartbeatRunEvents.payload} @> ${JSON.stringify(input.retryExhaustion)}::jsonb`,
))
.limit(1)
@@ -187,6 +191,7 @@ export async function appendHeartbeatRunEvent(
if (projectsResponse && run.runtimeMode === "native") {
await notifyNativeResponsePresentation(tx, { companyId: run.companyId, runId: input.runId });
}
if (projectsRetention) await notifyAttentionRetentionWork(tx, { companyId: input.companyId, sourceKind: "failed_run" });
const seq = await allocateHeartbeatRunEventSeq(
tx as unknown as Db,
input.runId,
+43 -55
View File
@@ -1,3 +1,7 @@
import { registerDeferredReceiptWork } from "./heartbeat/deferred-receipt-work.js";
import { reconcileTaskChatRetryReceipts } from "./task-retry.js";
import { registerIssueRecoveryWork } from "./recovery/work.js";
import { registerRunDispatchWork } from "./heartbeat/dispatch-work.js";
import { externalConversationSettlementWork } from "./external-conversation-settlement-work.js";
import { nativeRecoveryNoticeWork } from "./native-runtime/native-recovery-notice-work.js";
import { nativeRunProjectionWork } from "./native-runtime/native-run-projection-work.js";
@@ -999,6 +1003,7 @@ export function heartbeatService(
finalizeAiConnectionBusyDeferral,
finalizeWorkspaceBusyDeferral,
promoteDueScheduledRetries,
nextScheduledRetryAt,
retryScheduledRetryNow,
scheduleBoundedRetry,
} = createHeartbeatRetries(db, {
@@ -1333,8 +1338,8 @@ export function heartbeatService(
return heartbeatQueue.dispatchPendingNativeStatusWakeups(...args);
}
function resumeQueuedRunsAfterDrain() {
const promise = heartbeatQueue.resumeQueuedRunsAfterDrain();
function resumeQueuedRunsAfterDrain(...args: Parameters<typeof heartbeatQueue.resumeQueuedRunsAfterDrain>) {
const promise = heartbeatQueue.resumeQueuedRunsAfterDrain(...args);
activeWakeupPromises.add(promise);
void promise.then(
() => { activeWakeupPromises.delete(promise); },
@@ -1609,15 +1614,15 @@ export function heartbeatService(
void reportRunFailure(db, updated, failureReport);
}
async function reconcileStrandedAssignedIssues(opts?: { skipExternalConversationSettlement?: boolean }) {
async function reconcileStrandedAssignedIssues(opts?: Parameters<typeof recovery.reconcileStrandedAssignedIssues>[0]) {
return recovery.reconcileStrandedAssignedIssues({
skipExternalConversationSettlement: opts?.skipExternalConversationSettlement,
...opts,
issueCreatedAtGte: await getWorktreeExecutionCutoff(),
});
}
async function sweepStaleIssueLocks() {
return recovery.sweepStaleIssueLocks();
async function sweepStaleIssueLocks(opts?: Parameters<typeof recovery.sweepStaleIssueLocks>[0]) {
return recovery.sweepStaleIssueLocks(opts);
}
function issueIdFromRunContext(contextSnapshot: unknown) {
@@ -1647,10 +1652,7 @@ export function heartbeatService(
});
}
async function reconcileTaskWatchdogs(opts?: {
companyId?: string | null;
runId?: string | null;
}) {
async function reconcileTaskWatchdogs(opts?: Parameters<typeof taskWatchdogs.reconcileTaskWatchdogs>[0]) {
return taskWatchdogs.reconcileTaskWatchdogs({
...opts,
issueCreatedAtGte: await getWorktreeExecutionCutoff(),
@@ -1675,10 +1677,7 @@ export function heartbeatService(
return recovery.buildRunOutputSilence(run, now);
}
async function reconcileResolvedDependencyWakes(opts?: {
runId?: string | null;
companyId?: string | null;
}) {
async function reconcileResolvedDependencyWakes(opts?: Parameters<typeof recovery.reconcileResolvedDependencyWakeBackstop>[0]) {
return recovery.reconcileResolvedDependencyWakeBackstop(opts);
}
@@ -2007,8 +2006,11 @@ export function heartbeatService(
const dispatchIssueId = readNonEmptyString(parseObject(run.contextSnapshot).issueId);
const resumingAdmittedConversationTurn = !!runOptions.nativeLeaseOwner
&& typeof run.contextSnapshot?.conversationSessionGeneration === "number";
if (dispatchIssueId && isConversation(await getIssueExecutionContext(run.companyId, dispatchIssueId))
&& !resumingAdmittedConversationTurn && !(await instanceSettings.getExperimental()).enableAgentChat) {
const conversationIdentity = dispatchIssueId && !resumingAdmittedConversationTurn
? await db.select({ conversationAgentId: issues.conversationAgentId, conversationUserId: issues.conversationUserId })
.from(issues).where(and(eq(issues.companyId, run.companyId), eq(issues.id, dispatchIssueId))).then(rows => rows[0] ?? null)
: null;
if (isConversation(conversationIdentity) && !(await instanceSettings.getExperimental()).enableAgentChat) {
await setRunStatus(run.id, "cancelled", { finishedAt: new Date(), error: "Agent Chat is disabled", errorCode: "agent_chat_disabled" });
await setWakeupStatus(run.wakeupRequestId, "cancelled", { finishedAt: new Date() });
await releaseIssueExecutionAndPromote((await getRun(run.id))!, { suppressImmediateRecovery: true });
@@ -2204,44 +2206,6 @@ export function heartbeatService(
context[PAPERCLIP_EXTERNAL_CHAT_EXECUTION_BOUND_KEY] = true;
}
const wakeCommentId = deriveCommentId(context, null);
const wakeCommentContext =
issueContext && wakeCommentId
? await db
.select({
id: issueComments.id,
body: issueComments.body,
authorType: issueComments.authorType,
authorAgentId: issueComments.authorAgentId,
authorUserId: issueComments.authorUserId,
presentation: issueComments.presentation,
metadata: issueComments.metadata,
deletedAt: issueComments.deletedAt,
deletedByType: issueComments.deletedByType,
deletedByAgentId: issueComments.deletedByAgentId,
deletedByUserId: issueComments.deletedByUserId,
deletedByRunId: issueComments.deletedByRunId,
sourceTrust: issueComments.sourceTrust,
})
.from(issueComments)
.where(
and(
eq(issueComments.id, wakeCommentId),
eq(issueComments.issueId, issueContext.id),
eq(issueComments.companyId, agent.companyId),
),
)
.then((rows) => {
const row = rows[0] ?? null;
return row?.deletedAt
? {
...row,
body: "",
presentation: null,
metadata: null,
}
: row;
})
: null;
let issueAssigneeOverrides =
issueContext && issueContext.assigneeAgentId === agent.id
? parseIssueAssigneeAdapterOverrides(
@@ -2610,7 +2574,7 @@ export function heartbeatService(
continuationSummary, executionContinuation, paperclipWakePayload, safeWakeComments, safeWakeCommentContext, pinnedSkillTest,
} = await timePreparation("task_context", () => prepareHeartbeatTaskContext(db, {
run, agent,
task: { issueRef, issueContext, wakeCommentContext, context, taskSession },
task: { issueRef, issueContext, context, taskSession },
policy: { trustPreset, experimentalInstanceSettings, recovering: Boolean(runOptions.nativeRestartRecovery || runOptions.nativeLeaseOwner) },
services: { issuesSvc, getPinnedSkillTestContext },
}));
@@ -4246,6 +4210,30 @@ export function heartbeatService(
computeTaskDrain,
applyTaskDrain,
registerDeferredReceiptWork: (coordinator: Parameters<typeof registerDeferredReceiptWork>[1], canRun: () => boolean) =>
registerDeferredReceiptWork(db, coordinator, {
resume: scope => heartbeatQueue.resumeDeferredExecutionReceipts(scope),
settleTaskChat: scope => reconcileTaskChatRetryReceipts(db, { ...scope, settleOnly: true }),
dispatchNative: dispatchPendingNativeStatusWakeups,
schedulingSuppressed: async () => (await getSchedulingSuppression(true)).suppressed,
}, () => canRun() && readTaskDrain(new Date()) === null),
registerIssueStateRecoveryWork: (coordinator: Parameters<typeof registerIssueRecoveryWork>[1], canRun: () => boolean) =>
registerIssueRecoveryWork(db, coordinator, {
stranded: scope => reconcileStrandedAssignedIssues({ ...scope, skipExternalConversationSettlement: true, skipRelatedRecovery: true }),
orphanBlockers: recovery.reconcileUnassignedBlockingIssues,
recoveryActions: recovery.reconcileActiveRecoveryActions,
dependencies: reconcileResolvedDependencyWakes,
locks: sweepStaleIssueLocks,
watchdogs: reconcileTaskWatchdogs,
schedulingSuppressed: async () => (await getSchedulingSuppression(true)).suppressed,
}, () => canRun() && readTaskDrain(new Date()) === null),
registerRunDispatchWork: (coordinator: Parameters<typeof registerRunDispatchWork>[0], canRun: () => boolean) =>
registerRunDispatchWork(coordinator, {
promoteDueScheduledRetries,
nextScheduledRetryAt,
resumeQueuedRunsAfterDrain,
schedulingSuppressed: async () => (await getSchedulingSuppression(true)).suppressed,
}, () => canRun() && readTaskDrain(new Date()) === null),
promoteDueScheduledRetries,
retryScheduledRetryNow,
@@ -1,3 +1,4 @@
import { notifyAttentionRetentionWork } from "../retention-work-notifications.js";
import { DELIVERY_QUEUES, notifyDeliveryWork } from "../delivery-work-notifications.js";
import { agents, heartbeatRuns, type Db } from "@paperclipai/db";
import { and, asc, desc, eq, lte, sql } from "drizzle-orm";
@@ -91,6 +92,7 @@ export function createHeartbeatAgentStatus(db: Db) {
)).limit(1);
const nextStatus = running ? "running" : settledStatus(outcome, options?.keepIdleOnFailure);
const now = new Date();
await notifyAttentionRetentionWork(tx, { companyId: existing.companyId });
await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers);
const [updated] = await tx.update(agents).set({
status: nextStatus,
@@ -140,6 +142,7 @@ export function createHeartbeatAgentStatus(db: Db) {
eq(heartbeatRuns.agentId, candidate.id)))
.orderBy(desc(heartbeatRuns.finishedAt), desc(heartbeatRuns.createdAt), desc(heartbeatRuns.id)).limit(1);
const nextStatus = settledStatus(lastRun?.status);
await notifyAttentionRetentionWork(tx, { companyId: candidate.companyId });
const [agent] = await tx.update(agents).set({ status: nextStatus,
errorReason: nextStatus === "error" ? errorReason(lastRun?.error) : null, updatedAt: now,
}).where(and(eq(agents.id, candidate.id), eq(agents.companyId, candidate.companyId),
@@ -0,0 +1,118 @@
import { and, asc, eq, gt, inArray, isNull, or, sql } from "drizzle-orm";
import { agentWakeupRequests, type Db } from "@paperclipai/db";
import type { createDeliveryWorkCoordinator } from "../delivery-work-coordinator.js";
import { DELIVERY_QUEUES } from "../delivery-work-notifications.js";
import type { DeferredExecutionScope, DeferredReceiptRecoveryOptions } from "../deferred-execution-notifications.js";
import { registerScopedDeadlineWork } from "../scoped-deadline-work.js";
type Entry = { id: string; companyId: string; dueAt: number };
type NativeIntentOptions = DeferredExecutionScope & {
onDeferredDeadline(at: number): void;
onError(error: unknown): void;
};
export function registerDeferredReceiptWork(
db: Db,
coordinator: ReturnType<typeof createDeliveryWorkCoordinator>,
owner: {
resume(scope: DeferredReceiptRecoveryOptions): Promise<unknown>;
settleTaskChat(scope: DeferredExecutionScope): Promise<unknown>;
dispatchNative(scope: NativeIntentOptions): Promise<unknown>;
schedulingSuppressed(): Promise<boolean>;
},
canRun: () => boolean,
) {
function register(kind: "deferred" | "task_chat" | "native") {
const cursors = new Map<string, string>();
let enabled = false;
const queue = kind === "deferred" ? DELIVERY_QUEUES.deferredExecutionReceipts
: kind === "task_chat" ? DELIVERY_QUEUES.taskRetryReceiptSettlement : DELIVERY_QUEUES.nativeStatusWakeIntents;
return registerScopedDeadlineWork<Entry, DeferredExecutionScope>(coordinator, {
queue,
reconstructionIntervalMs: 30_000,
beforePass: async () => { enabled = !(await owner.schedulingSuppressed()); },
decodeScope: hint => hint === "*" ? undefined : JSON.parse(hint),
matches: (entry, scope) => !scope || entry.companyId === scope.companyId && (!scope.requestId || entry.id === scope.requestId),
load: async scope => {
const key = scope ? JSON.stringify({ companyId: scope.companyId, requestId: scope.requestId }) : "*";
if (!enabled) { cursors.delete(key); return []; }
const cursor = cursors.get(key);
const rows = await db.select({ id: agentWakeupRequests.id, companyId: agentWakeupRequests.companyId,
status: agentWakeupRequests.status, updatedAt: agentWakeupRequests.updatedAt,
claimedAt: agentWakeupRequests.claimedAt, error: agentWakeupRequests.error,
delayed: sql<boolean>`${agentWakeupRequests.payload}->'queuedCommentInterrupt' is not null
or ${agentWakeupRequests.payload}->'executionWait' is not null`,
interruptedLegacy: sql<boolean>`exists (select 1 from heartbeat_runs r
where r.company_id = ${agentWakeupRequests.companyId} and r.agent_id = ${agentWakeupRequests.agentId}
and r.runtime_mode = 'legacy' and r.status = 'cancelled'
and r.result_json->>'queuedCommentInterruptQueueId' = ${agentWakeupRequests.id}::text)`,
}).from(agentWakeupRequests).where(and(
scope ? eq(agentWakeupRequests.companyId, scope.companyId) : undefined,
scope?.requestId ? eq(agentWakeupRequests.id, scope.requestId) : undefined,
cursor ? gt(agentWakeupRequests.id, cursor) : undefined,
kind === "native" ? and(
eq(agentWakeupRequests.requestedByActorType, "system"),
eq(agentWakeupRequests.requestedByActorId, "native-status-committer"),
inArray(agentWakeupRequests.status, ["queued", "claimed"]), isNull(agentWakeupRequests.runId),
) : and(eq(agentWakeupRequests.status, "deferred_issue_execution"),
kind === "task_chat" ? and(eq(agentWakeupRequests.reason, "task_retry"),
sql`${agentWakeupRequests.payload}#>>'{taskRetry,chatActionId}' is not null`) : or(
eq(agentWakeupRequests.reason, "task_retry"),
sql`${agentWakeupRequests.payload}->'queuedCommentInterrupt' is not null`,
sql`${agentWakeupRequests.payload}->'executionWait' is not null`,
sql`jsonb_typeof(${agentWakeupRequests.payload}#>'{_paperclipWakeContext,wakeCommentIds}') = 'array'
and ${agentWakeupRequests.payload}#>'{_paperclipWakeContext,wakeCommentIds}' <> '[]'::jsonb`,
sql`${agentWakeupRequests.payload}->>'mutation' = 'interaction'`,
sql`exists (select 1 from heartbeat_runs r where r.company_id = ${agentWakeupRequests.companyId}
and r.runtime_mode = 'legacy' and r.status = 'cancelled'
and r.result_json->>'queuedCommentInterruptQueueId' = ${agentWakeupRequests.id}::text)`,
)),
)).orderBy(asc(agentWakeupRequests.id)).limit(100);
const complete = rows.length < 100;
if (complete) cursors.delete(key); else cursors.set(key, rows.at(-1)!.id);
return { rows: rows.map(row => ({ id: row.id, companyId: row.companyId,
dueAt: kind === "native" && row.status === "claimed" && row.claimedAt ? row.claimedAt.getTime() + 60_000
: kind === "native" && row.error ? row.updatedAt.getTime() + 30_000
: kind === "deferred" && row.delayed && !row.interruptedLegacy ? row.updatedAt.getTime() + 30_000
: Date.now(),
})), complete };
},
check: async entry => {
if (!enabled) return null;
if (kind !== "task_chat") {
const [current] = await db.select({ status: agentWakeupRequests.status, payload: agentWakeupRequests.payload,
updatedAt: agentWakeupRequests.updatedAt, claimedAt: agentWakeupRequests.claimedAt, error: agentWakeupRequests.error,
interruptedLegacy: sql<boolean>`exists (select 1 from heartbeat_runs r
where r.company_id = ${agentWakeupRequests.companyId} and r.agent_id = ${agentWakeupRequests.agentId}
and r.runtime_mode = 'legacy' and r.status = 'cancelled'
and r.result_json->>'queuedCommentInterruptQueueId' = ${agentWakeupRequests.id}::text)`,
}).from(agentWakeupRequests).where(and(eq(agentWakeupRequests.companyId, entry.companyId),
eq(agentWakeupRequests.id, entry.id)));
if (!current) return null;
const due = kind === "native" && current.status === "claimed" && current.claimedAt
? current.claimedAt.getTime() + 60_000
: kind === "native" && current.status === "queued" && current.error ? current.updatedAt.getTime() + 30_000
: kind === "deferred" && current.status === "deferred_issue_execution" && !current.interruptedLegacy
&& (current.payload?.queuedCommentInterrupt || current.payload?.executionWait)
? current.updatedAt.getTime() + 30_000 : null;
if (due !== null && due > Date.now()) return { ...entry, dueAt: due };
}
const errors: unknown[] = [];
let next: number | null = null;
const scope = { companyId: entry.companyId, requestId: entry.id };
if (kind === "native") await owner.dispatchNative({ ...scope,
onDeferredDeadline: at => { next = next === null ? at : Math.min(next, at); },
onError: error => { errors.push(error); },
});
else if (kind === "task_chat") await owner.settleTaskChat(scope);
else await owner.resume({ ...scope, onError: error => { errors.push(error); } });
if (errors.length) throw new AggregateError(errors, `Deferred receipt ${entry.id} could not settle`);
return next === null ? null : { ...entry, dueAt: next };
},
}, canRun);
}
const workers = [register("deferred"), register("task_chat"), register("native")];
return { ready: Promise.all(workers.map(worker => worker.ready)),
wake() { for (const worker of workers) worker.invalidate(); },
async stop() { await Promise.all(workers.map(worker => worker.stop())); } };
}
@@ -0,0 +1,58 @@
import type { RunDispatchScope } from "../run-dispatch-notifications.js";
import type { createDeliveryWorkCoordinator } from "../delivery-work-coordinator.js";
import { registerDeadlineWork } from "../deadline-work.js";
import { DELIVERY_QUEUES } from "../delivery-work-notifications.js";
export function registerRunDispatchWork(
coordinator: ReturnType<typeof createDeliveryWorkCoordinator>,
owner: {
promoteDueScheduledRetries(): Promise<unknown>;
nextScheduledRetryAt(): Promise<number | null>;
resumeQueuedRunsAfterDrain(scope?: RunDispatchScope): Promise<unknown>;
schedulingSuppressed(): Promise<boolean>;
},
canRun: () => boolean,
) {
let suppressed = false;
const retries = registerDeadlineWork(coordinator, DELIVERY_QUEUES.scheduledRetryDispatch, {
retryMs: 30_000,
continuationMs: 1_000,
notificationCoalesceMs: 1_000,
canRun,
run: async signal => {
suppressed = await owner.schedulingSuppressed();
if (!signal.aborted && !suppressed && canRun()) await owner.promoteDueScheduledRetries();
},
next: () => suppressed ? Promise.resolve(null) : owner.nextScheduledRetryAt(),
});
const scopes = new Set<string>(["*"]);
const queue = registerDeadlineWork(coordinator, DELIVERY_QUEUES.queuedRunDispatch, {
retryMs: 30_000,
notificationCoalesceMs: 1_000,
canRun,
onSettled: hints => { for (const hint of hints ?? ["*"]) scopes.add(hint); },
run: async signal => {
const requested = scopes.has("*") ? ["*"] : [...scopes];
if (requested[0] === "*") scopes.clear();
else for (const hint of requested) scopes.delete(hint);
for (let index = 0; index < requested.length; index++) {
if (signal.aborted || !canRun()) {
for (const hint of requested.slice(index)) scopes.add(hint);
return;
}
try {
await owner.resumeQueuedRunsAfterDrain(requested[index] === "*" ? undefined : JSON.parse(requested[index]) as RunDispatchScope);
} catch (error) {
for (const hint of requested.slice(index)) scopes.add(hint);
throw error;
}
}
},
next: async () => null,
});
return {
ready: Promise.all([retries.ready, queue.ready]),
wake() { scopes.add("*"); retries.wake(); queue.wake(); },
async stop() { await Promise.all([retries.stop(), queue.stop()]); },
};
}
@@ -139,28 +139,27 @@ export async function prepareHeartbeatExecutionConfiguration(db: Db, input: Hear
stripWorkspaceRuntimeFromExecutionRunConfig(mergedConfig);
const skillActor: AuthorizationActor = { type: "agent", agentId: agent.id, companyId: agent.companyId,
runId: run.id, source: "agent_jwt", onBehalfOfUserId: responsibleUserId === "local-board" ? null : responsibleUserId };
const runScopedMentionedSkillKeys =
preconfiguredHome || pinnedSkillTest ? [] : await time("configuration_mentions", () => resolveRunScopedMentionedSkillKeys({
const [mentionedSkills, githubIdentity] = await Promise.allSettled([
preconfiguredHome || pinnedSkillTest ? Promise.resolve([]) : time("configuration_mentions", () => resolveRunScopedMentionedSkillKeys({
db,
companyId: agent.companyId,
issueId,
commentIds: extractWakeCommentIds(context),
actor: skillActor,
}));
})),
time("configuration_github_identity", () => resolveManagedGitHubIdentitySelection(db, agent.companyId, {
agentId: agent.id, responsibleUserId, allowStandingDelegation: false,
})),
]);
if (mentionedSkills.status === "rejected") throw mentionedSkills.reason;
if (githubIdentity.status === "rejected") throw githubIdentity.reason;
const runScopedMentionedSkillKeys = mentionedSkills.value;
const runScopedSkillKeys =
!pinnedSkillTest && acceptedPlanContinuationWake &&
!acceptedPlanWakeRoutingDecision?.suppressAcceptedContinuation
? [...runScopedMentionedSkillKeys, ACCEPTED_PLAN_CONVERSION_SKILL_KEY]
: runScopedMentionedSkillKeys;
const githubSelection = await time("configuration_github_identity", () => resolveManagedGitHubIdentitySelection(
db,
agent.companyId,
{
agentId: agent.id,
responsibleUserId,
allowStandingDelegation: false,
},
));
const githubSelection = githubIdentity.value;
const useHostGitHub =
!githubSelection.configured &&
trustPreset.kind === "standard" &&
@@ -274,6 +273,7 @@ export async function prepareHeartbeatExecutionConfiguration(db: Db, input: Hear
);
const nativeRunnerPreparationSpans: NativeRunHistoricalSpan[] = [];
const skillsPrepareStartedAtMs = Date.now();
let skillsPrepareEndedAtMs = skillsPrepareStartedAtMs;
let skillDeliveryConfig = effectiveResolvedConfig;
const prepareSkills = async () => {
@@ -346,24 +346,35 @@ export async function prepareHeartbeatExecutionConfiguration(db: Db, input: Hear
throw error;
}
};
const runtimeSkillEntries = preconfiguredHome ? [] : await time("configuration_skills", prepareSkills);
const skillsPreparation = (preconfiguredHome ? Promise.resolve([]) : time("configuration_skills", prepareSkills))
.finally(() => { skillsPrepareEndedAtMs = Date.now(); });
const assignmentsPreparation = preconfiguredHome ? Promise.resolve([]) : time("configuration_connector_assignments", () => resolveConnectorAssignments(db, { companyId: agent.companyId, agentId: agent.id, runId: run.id, issueId: typeof context.issueId === "string" ? context.issueId : undefined }));
const [preparedSkills, preparedAssignments, preparedInstructions] = await Promise.allSettled([
skillsPreparation,
assignmentsPreparation,
assignmentsPreparation.then(() => time("configuration_connection_instructions", () => resolveAssignedConnectionInstructionsForRun(db, { companyId: agent.companyId, agentId: agent.id, runId: run.id }))),
]);
if (preparedSkills.status === "rejected") throw preparedSkills.reason;
if (preparedAssignments.status === "rejected") throw preparedAssignments.reason;
if (preparedInstructions.status === "rejected") throw preparedInstructions.reason;
const runtimeSkillEntries = preparedSkills.value;
nativeRunnerPreparationSpans.push({
name: "skills.prepare",
parentName: "task.prepare",
startedAtMs: skillsPrepareStartedAtMs,
endedAtMs: Date.now(),
endedAtMs: skillsPrepareEndedAtMs,
});
const connectorAssignments = preconfiguredHome ? [] : await time("configuration_connector_assignments", () => resolveConnectorAssignments(db, { companyId: agent.companyId, agentId: agent.id, runId: run.id, issueId: typeof context.issueId === "string" ? context.issueId : undefined }));
const connectorAssignments = preparedAssignments.value;
const connectorSkillConfig = await time("configuration_connector_skills", () => applyConnectorSkills(skillDeliveryConfig, runtimeSkillEntries, connectorAssignments));
// Both CLI adapters and native context materialization use the same resolved set.
runtimeSkillEntries.splice(0, runtimeSkillEntries.length, ...connectorSkillConfig.paperclipRuntimeSkills);
const connectorDelivery = await time("configuration_connector_delivery", () => prepareConnectorSkillDelivery(connectorSkillConfig, agent.adapterType));
// Always replace this runtime-only field; caller wake data cannot supply skills.
context.paperclipWake = { ...parseObject(context.paperclipWake), connectorSkillInstructions: connectorDelivery.instructions };
let runtimeConfig = await time("configuration_connection_instructions", () => prepareConnectionInstructionDelivery({
resolve: () => resolveAssignedConnectionInstructionsForRun(db, { companyId: agent.companyId, agentId: agent.id, runId: run.id }),
let runtimeConfig = await prepareConnectionInstructionDelivery({
resolve: async () => preparedInstructions.value,
context, config: connectorDelivery.config, native: agent.adapterType === "paperclip_runner",
}));
});
const agentIdentity = supportsManagedAgentIdentity(agent.adapterType, agent.adapterConfig, run.runtimeMode === "native" ? run.driverKind : undefined)
? await time("configuration_agent_identity", () => agentIdentityService(db).ensureAgentIdentity(agent.companyId, agent.id))
: undefined;
+47 -22
View File
@@ -1,4 +1,9 @@
import { notifyRoutineExecutionStatus } from "../routine-execution-status-notifications.js";
import { notifyStatusCardIssueChange } from "../status-card-notifications.js";
import { writeDeferredExecutionWait } from "../deferred-execution-wait.js";
import { notifyDeferredExecutionWork } from "../deferred-execution-notifications.js";
import type { DeferredReceiptRecoveryOptions } from "../deferred-execution-notifications.js";
import { notifyRunDispatch } from "../run-dispatch-notifications.js";
import { DELIVERY_QUEUES, notifyDeliveryWork } from "../delivery-work-notifications.js";
import { notifyExternalAgentSettlement } from "../../modules/external-agents/index.js";
import { notifyNativeStatusReassessment } from "../native-status-reassessment-notifications.js";
@@ -222,7 +227,7 @@ export interface HeartbeatQueueDependencies extends Pick<HeartbeatRetryDependenc
publishRunLifecyclePluginEvent: (run: HeartbeatRun) => void;
cancelRunInternal: (runId: string, reason?: string, options?: { errorCode?: string }) => Promise<unknown>;
cancelActiveForAgentInternal: (agentId: string, reason: string) => Promise<unknown>;
resumeExecutionWaitComments: () => Promise<unknown>;
resumeExecutionWaitComments: (scope?: DeferredReceiptRecoveryOptions) => Promise<unknown>;
resumeQueuedCommentInterrupt: (companyId: string, queueId: string) => Promise<unknown>;
resumeSavedLegacyComments: (companyId: string, queueId: string) => Promise<unknown>;
formatIssueIdentifierLink: (identifier: string | null, fallback: string) => string;
@@ -1268,6 +1273,7 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
}
async function bindClaimedIssueExecution(tx: Db, ownsIssue: boolean, claimedRun: typeof heartbeatRuns.$inferSelect | null | undefined) {
if (!claimedRun || !issueId || !ownsIssue) return;
await notifyStatusCardIssueChange(tx, run.companyId);
await tx.update(issues).set({
executionRunId: claimedRun.id,
executionAgentNameKey: normalizeAgentNameKey(agent.name),
@@ -1851,13 +1857,15 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
return cancelled;
}
async function resumeQueuedRuns() {
async function resumeDeferredExecutionReceipts(scope: DeferredReceiptRecoveryOptions = {}) {
if ((await getSchedulingSuppression()).suppressed) return;
await resumeExecutionWaitComments();
await resumeExecutionWaitComments(scope);
const cutoff = await getWorktreeExecutionCutoff();
const pendingInterrupts = await db.select({ id: agentWakeupRequests.id, companyId: agentWakeupRequests.companyId })
.from(agentWakeupRequests).innerJoin(companies, eq(companies.id, agentWakeupRequests.companyId))
.where(and(eq(agentWakeupRequests.status, "deferred_issue_execution"),
.where(and(scope.companyId ? eq(agentWakeupRequests.companyId, scope.companyId) : undefined,
scope.requestId ? eq(agentWakeupRequests.id, scope.requestId) : undefined,
eq(agentWakeupRequests.status, "deferred_issue_execution"),
eq(companies.status, "active"),
sql`${agentWakeupRequests.payload}->'queuedCommentInterrupt' is not null`,
lte(agentWakeupRequests.updatedAt, new Date(Date.now() - 30_000)),
@@ -1868,6 +1876,7 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
eq(agentWakeupRequests.id, wake.id), eq(agentWakeupRequests.status, "deferred_issue_execution"),
));
await resumeQueuedCommentInterrupt(wake.companyId, wake.id).catch(err => {
scope.onError?.(err);
logger.warn({ err, queueId: wake.id }, "failed to resume interrupted comment queue");
});
}
@@ -1881,7 +1890,9 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
or(eq(issues.assigneeAgentId, agentWakeupRequests.agentId),
sql`${agentWakeupRequests.payload} #>> '{_paperclipWakeContext,wakeReason}' = 'issue_comment_mentioned'`)))
.innerJoin(companies, and(eq(companies.id, issues.companyId), eq(companies.status, "active")))
.where(and(eq(agentWakeupRequests.status, "deferred_issue_execution"),
.where(and(scope.companyId ? eq(agentWakeupRequests.companyId, scope.companyId) : undefined,
scope.requestId ? eq(agentWakeupRequests.id, scope.requestId) : undefined,
eq(agentWakeupRequests.status, "deferred_issue_execution"),
isNull(issues.executionRunId),
or(and(
sql`jsonb_typeof(${agentWakeupRequests.payload} #> '{_paperclipWakeContext,wakeCommentIds}') = 'array'`,
@@ -1920,7 +1931,8 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
if (latest.runtimeMode !== "legacy" && latest.runtimeMode !== "native") continue;
if (wake.idempotencyKey?.startsWith("chat-inbound:")) {
await resumeDeferredChatWake(db, { companyId: wake.companyId, agentId: wake.agentId,
issueId: String(wake.payload?.issueId), requestId: wake.id }).catch(() => {
issueId: String(wake.payload?.issueId), requestId: wake.id }).catch(err => {
scope.onError?.(err);
logger.warn({ requestId: wake.id }, "saved chat source could not resume after recovery");
});
continue;
@@ -1932,6 +1944,7 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
latest.errorCode === "execution_reconciliation_required";
if ((latest.status !== "cancelled" || cancelledAdmission) && await getExecutionBlocker(db, wake.companyId, String(wake.payload?.issueId))) {
await resumeSavedLegacyComments(wake.companyId, wake.id).catch(err => {
scope.onError?.(err);
logger.warn({ err, queueId: wake.id }, "failed to deliver saved legacy comment after recovery stopped");
});
}
@@ -1939,6 +1952,7 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
await releaseIssueExecutionAndPromote(latest, { suppressImmediateRecovery: true, resumeStrandedQueue: true }).then(() => {
if (latest.runtimeMode === "native") releasedNativeSources.add(releaseSource);
}).catch(err => {
scope.onError?.(err);
logger.warn({ err, queueId: wake.id }, "failed to promote stranded comments");
});
}
@@ -1956,6 +1970,8 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
))
.innerJoin(companies, eq(companies.id, heartbeatRuns.companyId))
.where(and(
scope.companyId ? eq(agentWakeupRequests.companyId, scope.companyId) : undefined,
scope.requestId ? eq(agentWakeupRequests.id, scope.requestId) : undefined,
eq(agentWakeupRequests.status, "deferred_issue_execution"),
eq(heartbeatRuns.status, "cancelled"),
eq(heartbeatRuns.runtimeMode, "legacy"),
@@ -1964,16 +1980,21 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
));
for (const run of interruptedQueues) {
await releaseIssueExecutionAndPromote(run, { suppressImmediateRecovery: true }).catch((err) => {
scope.onError?.(err);
logger.error({ err, runId: run.id }, "failed to retry interrupted comment queue");
});
}
}
async function resumeQueuedRuns() {
await resumeDeferredExecutionReceipts();
await resumeQueuedRunsAfterDrain();
}
// Release admission promptly without first scanning unrelated recovery work.
// Every run still passes the ordinary per-agent claim and dispatch gates.
async function resumeQueuedRunsAfterDrain() {
async function resumeQueuedRunsAfterDrain(scope?: { companyId: string }) {
if ((await getSchedulingSuppression()).suppressed) return;
const cutoff = await getWorktreeExecutionCutoff();
const queuedRuns = await db
@@ -1983,6 +2004,7 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
.where(
and(
eq(heartbeatRuns.status, "queued"),
scope ? eq(heartbeatRuns.companyId, scope.companyId) : undefined,
eq(companies.status, "active"),
cutoff ? gte(heartbeatRuns.createdAt, cutoff) : undefined,
),
@@ -2505,17 +2527,11 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
// its one durable receipt ID on a skipped wake.
if (caseRequest) return { created: false };
if (executionWaitRequestId) {
await db.update(agentWakeupRequests).set({
payload: sql`jsonb_set(coalesce(${agentWakeupRequests.payload}, '{}'::jsonb), '{executionWait}',
coalesce(${agentWakeupRequests.payload}->'executionWait', '{}'::jsonb) || ${JSON.stringify({
reason: skipReason, message: patch.error ?? (skipReason === "issue_tree_hold_active"
? "This task is paused. Resume it to send your saved message."
: "Waiting for task execution to be enabled. Your message is saved."),
})}::jsonb)`,
updatedAt: new Date(),
}).where(and(eq(agentWakeupRequests.id, executionWaitRequestId),
eq(agentWakeupRequests.companyId, agent.companyId), eq(agentWakeupRequests.agentId, agentId),
eq(agentWakeupRequests.status, "deferred_issue_execution")));
await writeDeferredExecutionWait(db, { companyId: agent.companyId, requestId: executionWaitRequestId }, {
reason: skipReason, message: patch.error ?? (skipReason === "issue_tree_hold_active"
? "This task is paused. Resume it to send your saved message."
: "Waiting for task execution to be enabled. Your message is saved."),
});
return { created: false };
}
const request = {
@@ -3403,15 +3419,14 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
opts.issueStateGuard?.onExecutionDeferred?.(executionBlocker.cause);
const condition = { recoveryActionId: executionBlocker.recoveryActionId, ...continuationWait };
if (executionWaitRequestId) {
await tx.update(agentWakeupRequests).set({
payload: sql`jsonb_set(coalesce(${agentWakeupRequests.payload}, '{}'::jsonb), '{executionWait}', ${JSON.stringify(condition)}::jsonb)`,
updatedAt: new Date(),
}).where(eq(agentWakeupRequests.id, executionWaitRequestId));
await writeDeferredExecutionWait(tx as unknown as Db, { companyId: agent.companyId,
requestId: executionWaitRequestId }, condition, { replace: true });
return { kind: "deferred" as const, admitted: true };
}
if (durableRequest || wakeCommentId ||
hasInteractionContinuationWakeContext(enrichedContextSnapshot) ||
readNonEmptyString(enrichedContextSnapshot.nativeStatusWakeIntentId)) {
await notifyDeferredExecutionWork(tx, { companyId: agent.companyId });
await tx.insert(agentWakeupRequests).values({
...durableReceiptFields,
companyId: agent.companyId, agentId, source, triggerDetail, reason,
@@ -3759,6 +3774,7 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
.from(agents)
.where(eq(agents.id, legacyRun.agentId))
.then((rows) => rows[0] ?? null);
await notifyStatusCardIssueChange(tx, issue.companyId);
await tx
.update(issues)
.set({
@@ -4030,6 +4046,7 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
}));
}
if (followUp) {
if (!followUp.consumed) await notifyDeferredExecutionWork(tx, { companyId: agent.companyId });
const assignmentId = followUp.assignmentId;
const pending = followUp.consumed ? undefined : pendingFollowUps.find(wake =>
wake.payload?.[followUpPayloadKey] === assignmentId &&
@@ -4634,6 +4651,7 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
}
const queueOutcome = await db.transaction(async (tx) => {
await notifyRunDispatch(tx, { companyId: agent.companyId });
await tx.execute(
sql`select id from agents where id = ${agentId} and company_id = ${agent.companyId} for update`,
);
@@ -4767,6 +4785,9 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
companyId?: string;
limit?: number;
staleClaimMs?: number;
requestId?: string;
onDeferredDeadline?: (at: number) => void;
onError?: (error: unknown) => void;
} = {},
) {
const now = new Date();
@@ -4779,6 +4800,7 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
input.companyId
? eq(agentWakeupRequests.companyId, input.companyId)
: undefined,
input.requestId ? eq(agentWakeupRequests.id, input.requestId) : undefined,
eq(agentWakeupRequests.requestedByActorType, "system"),
eq(agentWakeupRequests.requestedByActorId, "native-status-committer"),
inArray(agentWakeupRequests.status, ["queued", "claimed"]),
@@ -4855,6 +4877,7 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
candidate.claimedAt &&
now.getTime() - candidate.claimedAt.getTime() < staleClaimMs
) {
input.onDeferredDeadline?.(candidate.claimedAt.getTime() + staleClaimMs);
deferred += 1;
continue;
}
@@ -5029,6 +5052,7 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
if (wakeRun) dispatched += 1;
else deferred += 1;
} catch (error) {
input.onError?.(error);
await db
.update(agentWakeupRequests)
.set({
@@ -5072,6 +5096,7 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
withChatControlRecoveryGate,
trackWakeup,
resumeQueuedRuns,
resumeDeferredExecutionReceipts,
resumeQueuedRunsAfterDrain,
parseHeartbeatPolicy,
nextTimerHeartbeatAt,
+2 -2
View File
@@ -1979,7 +1979,7 @@ export function createHeartbeatRecovery(db: Db, dependencies: HeartbeatRecoveryD
let settledProcessesDiscovered = false;
async function reapOrphanedRuns(opts?: { staleThresholdMs?: number; skipScheduledRetryCancellation?: boolean; skipCapturedInstructionCommit?: boolean; skipInstructionDirectoryCleanup?: boolean; skipStoppedInstructionCollection?: boolean; skipCancelledWorkspacePreservation?: boolean; skipObsoletePolicyReviews?: boolean; skipAutomaticCompletionReviews?: boolean; skipUnsafeWorkspaceExports?: boolean; skipNativePendingEffects?: boolean; skipNativeResponsePresentation?: boolean; skipNativeReviewResponse?: boolean; skipNativeStatusReassessment?: boolean; skipRetainedNativeSessionCleanup?: boolean; skipNativeWorkspaceFinalization?: boolean; skipNativeResultCommit?: boolean; skipNativeRunProjection?: boolean; skipNativeRecoveryNotices?: boolean; skipExternalConversationSettlement?: boolean; skipExternalAgentSettlement?: boolean; skipNativeEnvironmentSettlement?: boolean }) {
async function reapOrphanedRuns(opts?: { staleThresholdMs?: number; skipNativeStatusWakeIntents?: boolean; skipScheduledRetryCancellation?: boolean; skipCapturedInstructionCommit?: boolean; skipInstructionDirectoryCleanup?: boolean; skipStoppedInstructionCollection?: boolean; skipCancelledWorkspacePreservation?: boolean; skipObsoletePolicyReviews?: boolean; skipAutomaticCompletionReviews?: boolean; skipUnsafeWorkspaceExports?: boolean; skipNativePendingEffects?: boolean; skipNativeResponsePresentation?: boolean; skipNativeReviewResponse?: boolean; skipNativeStatusReassessment?: boolean; skipRetainedNativeSessionCleanup?: boolean; skipNativeWorkspaceFinalization?: boolean; skipNativeResultCommit?: boolean; skipNativeRunProjection?: boolean; skipNativeRecoveryNotices?: boolean; skipExternalConversationSettlement?: boolean; skipExternalAgentSettlement?: boolean; skipNativeEnvironmentSettlement?: boolean }) {
if (!opts?.skipCancelledWorkspacePreservation) await preserveCancelledNativeWorkspaces(db);
const staleThresholdMs = opts?.staleThresholdMs ?? 0;
const now = new Date();
@@ -2023,7 +2023,7 @@ export function createHeartbeatRecovery(db: Db, dependencies: HeartbeatRecoveryD
});
if (!opts?.skipRetainedNativeSessionCleanup) scheduleRetainedNativeSessionCleanup();
await reconcileRetainedWorkspaces(db, environmentRuntime, resumeRemoteStopComments);
await dispatchPendingNativeStatusWakeups().catch((error) => {
if (!opts?.skipNativeStatusWakeIntents) await dispatchPendingNativeStatusWakeups().catch((error) => {
logger.warn(
{ err: error },
"failed to dispatch persisted native status wake intents before orphan reaping",
@@ -35,6 +35,7 @@ function callbacks() {
getWorktreeExecutionCutoff: vi.fn(async (): Promise<Date | null> => null),
applyRunDispatchPostCommitEffects: vi.fn<(effects: PostCommitEffect[]) => void>(),
runDispatch: {
nextScheduledRetryAt: vi.fn(async () => null),
evaluateScheduledRetryGate: vi.fn<HeartbeatRetryDependencies["runDispatch"]["evaluateScheduledRetryGate"]>(async () => ({ allowed: true })),
promoteDueScheduledRetries: vi.fn<HeartbeatRetryDependencies["runDispatch"]["promoteDueScheduledRetries"]>(async () => ({ promoted: 0, runIds: [], postCommitEffects: [] })),
promoteScheduledRetry: vi.fn<HeartbeatRetryDependencies["runDispatch"]["promoteScheduledRetry"]>(async () => ({ outcome: "not_promoted" })),
@@ -183,7 +184,9 @@ describe("heartbeat retry module callbacks", () => {
});
});
const support = await getEmbeddedPostgresTestSupport();
import { resolveExternalTestDatabaseUrl } from "../../__tests__/helpers/external-test-database.js";
const external = resolveExternalTestDatabaseUrl("PAPERCLIP_HEARTBEAT_RETRIES_TEST_DATABASE_URL");
const support = external ? { supported: true, reason: undefined } : await getEmbeddedPostgresTestSupport();
const describePostgres = support.supported ? describe : describe.skip;
if (!support.supported) console.warn(`Skipping retry PostgreSQL tests: ${support.reason ?? "unsupported host"}`);
@@ -193,11 +196,11 @@ describePostgres("heartbeat retry module database wiring", () => {
const now = new Date("2026-10-08T12:00:00Z");
beforeAll(async () => {
database = await startEmbeddedPostgresTestDatabase("heartbeat-retries-");
db = createDb(database.connectionString);
if (!external) database = await startEmbeddedPostgresTestDatabase("heartbeat-retries-");
db = createDb(external ?? database.connectionString, { maxConnections: 6 });
}, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS);
afterAll(async () => { await database?.cleanup(); });
afterAll(async () => { await db?.$client.end({ timeout: 1 }); await database?.cleanup(); });
afterEach(async () => {
await db.delete(issues);
+13 -1
View File
@@ -1,3 +1,6 @@
import { notifyAttentionRetentionWork } from "../retention-work-notifications.js";
import { notifyStatusCardIssueChange } from "../status-card-notifications.js";
import { notifyRunDispatch } from "../run-dispatch-notifications.js";
import { deriveTaskKeyWithHeartbeatFallback } from "./run-state.js";
import { infrastructureRetryDelayMs } from "../system-recovery-state.js";
import {
@@ -77,7 +80,7 @@ type PlanApprovalResumeFailure = {
export interface HeartbeatRetryDependencies {
hasCaseHandoffForRun: (run: Pick<HeartbeatRun, "id" | "companyId">) => Promise<boolean>;
runDispatch: Pick<ReturnType<typeof createRunDispatch>,
"evaluateScheduledRetryGate" | "promoteDueScheduledRetries" | "promoteScheduledRetry">;
"evaluateScheduledRetryGate" | "promoteDueScheduledRetries" | "promoteScheduledRetry" | "nextScheduledRetryAt">;
getAgent: (agentId: string) => Promise<typeof agents.$inferSelect | null | undefined>;
getRun: (
runId: string,
@@ -997,6 +1000,8 @@ export function createHeartbeatRetries(db: Db, dependencies: HeartbeatRetryDepen
retryContextSnapshot.previousRunId = continuation.previousRunId;
}
await notifyAttentionRetentionWork(tx, { companyId: run.companyId });
await notifyRunDispatch(tx, { companyId: run.companyId });
const wakeupRequest = await tx
.insert(agentWakeupRequests)
.values({
@@ -1178,6 +1183,7 @@ export function createHeartbeatRetries(db: Db, dependencies: HeartbeatRetryDepen
}
if (issueId) {
await notifyStatusCardIssueChange(tx, run.companyId);
await tx
.update(issues)
.set({
@@ -1643,6 +1649,10 @@ export function createHeartbeatRetries(db: Db, dependencies: HeartbeatRetryDepen
});
}
async function nextScheduledRetryAt() {
return runDispatch.nextScheduledRetryAt({ cutoff: await getWorktreeExecutionCutoff() });
}
async function promoteDueScheduledRetries(now = new Date()) {
const cutoff = await getWorktreeExecutionCutoff();
const result = await runDispatch.promoteDueScheduledRetries({
@@ -1750,6 +1760,7 @@ export function createHeartbeatRetries(db: Db, dependencies: HeartbeatRetryDepen
};
const updated = await db.transaction(async (tx) => {
await notifyRunDispatch(tx, { companyId: scheduled.run.companyId });
const row = await tx
.update(heartbeatRuns)
.set({
@@ -1901,6 +1912,7 @@ export function createHeartbeatRetries(db: Db, dependencies: HeartbeatRetryDepen
finalizeAiConnectionBusyDeferral,
finalizeWorkspaceBusyDeferral,
promoteDueScheduledRetries,
nextScheduledRetryAt,
retryScheduledRetryNow,
scheduleBoundedRetry: async (
+16 -11
View File
@@ -1,3 +1,5 @@
import { writeDeferredExecutionWait } from "../deferred-execution-wait.js";
import type { DeferredReceiptRecoveryOptions } from "../deferred-execution-notifications.js";
import { observeStoppedWorkspaceRecovery } from "../workspace-recovery.js";
import { notifyScheduledRetryCancellation } from "../scheduled-retry-cancellation-notifications.js";
import { notifyChatInboundWakeNotices } from "../chat-inbound-wake-notice-notifications.js";
@@ -616,12 +618,7 @@ export function createHeartbeatRunControl(db: Db, dependencies: HeartbeatRunCont
onBlocked: (reason, message) => { wait = { reason, message }; },
});
if (!admitted) {
await db.update(agentWakeupRequests).set({
payload: sql`jsonb_set(coalesce(${agentWakeupRequests.payload}, '{}'::jsonb), '{executionWait}',
coalesce(${agentWakeupRequests.payload}->'executionWait', '{}'::jsonb) || ${JSON.stringify(wait)}::jsonb)`,
updatedAt: new Date(),
}).where(and(eq(agentWakeupRequests.id, wake.id), eq(agentWakeupRequests.companyId, run.companyId),
eq(agentWakeupRequests.status, "deferred_issue_execution")));
await writeDeferredExecutionWait(db, { companyId: run.companyId, requestId: wake.id }, wait);
continue;
}
}
@@ -777,21 +774,25 @@ export function createHeartbeatRunControl(db: Db, dependencies: HeartbeatRunCont
idempotencyKey: `task-retry-dispatch:${wake.id}` }, wake.id);
}
async function resumeExecutionWaitComments() {
async function resumeExecutionWaitComments(scope: DeferredReceiptRecoveryOptions = {}) {
if ((await getSchedulingSuppression()).suppressed) return;
await reconcileTaskChatRetryReceipts(db);
await reconcileTaskChatRetryReceipts(db, scope);
const retries = await db.select({ id: agentWakeupRequests.id, companyId: agentWakeupRequests.companyId })
.from(agentWakeupRequests).innerJoin(issues, and(eq(issues.companyId, agentWakeupRequests.companyId),
sql`${issues.id}::text = ${agentWakeupRequests.payload}->>'issueId'`,
eq(issues.assigneeAgentId, agentWakeupRequests.agentId)))
.innerJoin(companies, and(eq(companies.id, issues.companyId), eq(companies.status, "active")))
.where(and(eq(agentWakeupRequests.reason, "task_retry"), eq(agentWakeupRequests.status, "deferred_issue_execution"),
.where(and(scope.companyId ? eq(agentWakeupRequests.companyId, scope.companyId) : undefined,
scope.requestId ? eq(agentWakeupRequests.id, scope.requestId) : undefined,
eq(agentWakeupRequests.reason, "task_retry"), eq(agentWakeupRequests.status, "deferred_issue_execution"),
sql`${agentWakeupRequests.payload}->'taskRetry'->>'chatActionId' is null`,
notInArray(issues.status, ["done", "cancelled"])))
.orderBy(asc(agentWakeupRequests.updatedAt)).limit(50);
for (const retry of retries) {
await db.update(agentWakeupRequests).set({ updatedAt: new Date() }).where(eq(agentWakeupRequests.id, retry.id));
await resumeTaskRetry(retry.companyId, retry.id).catch(err => logger.warn({ err, requestId: retry.id }, "Task retry dispatch deferred"));
await resumeTaskRetry(retry.companyId, retry.id).catch(err => {
scope.onError?.(err); logger.warn({ err, requestId: retry.id }, "Task retry dispatch deferred");
});
}
const waits = await db.select({ wake: agentWakeupRequests })
.from(agentWakeupRequests)
@@ -799,7 +800,9 @@ export function createHeartbeatRunControl(db: Db, dependencies: HeartbeatRunCont
sql`${issues.id}::text = ${agentWakeupRequests.payload}->>'issueId'`,
eq(issues.assigneeAgentId, agentWakeupRequests.agentId)))
.innerJoin(companies, and(eq(companies.id, issues.companyId), eq(companies.status, "active")))
.where(and(exists(db.select({ id: issueRecoveryActions.id }).from(issueRecoveryActions).where(and(
.where(and(scope.companyId ? eq(agentWakeupRequests.companyId, scope.companyId) : undefined,
scope.requestId ? eq(agentWakeupRequests.id, scope.requestId) : undefined,
exists(db.select({ id: issueRecoveryActions.id }).from(issueRecoveryActions).where(and(
eq(issueRecoveryActions.companyId, issues.companyId), eq(issueRecoveryActions.sourceIssueId, issues.id),
executionBlockerPredicate(),
))), eq(agentWakeupRequests.status, "deferred_issue_execution"),
@@ -831,11 +834,13 @@ export function createHeartbeatRunControl(db: Db, dependencies: HeartbeatRunCont
if (!run || run.companyId !== wake.companyId || run.agentId !== wake.agentId) continue;
if (canContinueCancelledRun(run)) {
await resumeSavedComments(wake.companyId, wake.id).catch(err => {
scope.onError?.(err);
logger.warn({ err, runId: run.id }, "failed to resume saved input after provider cancellation");
});
continue;
}
await resumeRemoteStopComments(run, wake.id).catch(err => {
scope.onError?.(err);
logger.warn({ err, runId: run.id }, "failed to resume saved execution-wait message");
});
}
@@ -1,3 +1,4 @@
import { notifyRunDispatch } from "../run-dispatch-notifications.js";
import { preserveAcceptedRunUsageProjectionSql } from "../usage-receipts.js";
import { notifyNativeEnvironmentSettlement } from "../native-environment-settlement-notifications.js";
import { notifyExternalAgentSettlement } from "../../modules/external-agents/index.js";
@@ -2448,6 +2449,7 @@ export function createHeartbeatLifecycle(db: Db, dependencies: HeartbeatLifecycl
.returning()
.then((rows) => rows[0]);
await notifyRunDispatch(tx, { companyId: run.companyId });
const queuedRun = await tx
.insert(heartbeatRuns)
.values({
@@ -14,6 +14,7 @@ import {
ConfigurationIncompleteFailure,
createHeartbeatRunPreparation,
resolveExecutionRunAdapterConfig,
readHeartbeatWakeComments,
} from "./run-preparation.js";
// Existing integration suites exercise the real database and dispatch path.
@@ -38,6 +39,12 @@ function databaseWithResults(...results: unknown[][]) {
return { db: { select } as unknown as Db, select, scopes };
}
function deferred<T>() {
let resolve!: (value: T) => void;
const promise = new Promise<T>(complete => { resolve = complete; });
return { promise, resolve };
}
const emptyRoutineContext = { routineId: null, env: null, responsibleUserId: null };
describe("heartbeat run preparation module", () => {
@@ -120,6 +127,42 @@ describe("heartbeat run preparation module", () => {
expect(database.select).toHaveBeenCalledTimes(2);
});
it("scopes one selected-comment snapshot for both wake consumers", async () => {
const database = databaseWithResults([{ id: "comment-one", body: "selected wake" }]);
expect(await readHeartbeatWakeComments(database.db, "company-one", "issue-one", ["comment-one"]))
.toEqual([{ id: "comment-one", body: "selected wake" }]);
expect(database.scopes[0]).toEqual(expect.arrayContaining(["company-one", "issue-one", "comment-one"]));
expect(await readHeartbeatWakeComments(database.db, "company-one", "issue-one", [])).toEqual([]);
expect(database.select).toHaveBeenCalledOnce();
});
it("settles independent binding checks before surfacing a failure or resolving values", async () => {
const projectCheck = deferred<[]>();
const firstFailure = new Error("agent binding check failed");
const secretsSvc = {
resolveAdapterConfigForRuntime: vi.fn(), resolveEnvBindings: vi.fn(),
collectMissingRuntimeBindings: vi.fn(async (_company, _env, context) => {
if (context.consumerType === "agent") throw firstFailure;
return context.consumerType === "project" ? projectCheck.promise : [];
}),
collectMissingAdapterConfigRuntimeBindings: vi.fn(async () => []),
};
let settled = false;
const opening = resolveExecutionRunAdapterConfig({
companyId: "company-one", agentId: "agent-one", projectId: "project-one",
environmentId: "environment-one", environmentEnv: { SHARED: "environment" },
executionRunConfig: { env: { SHARED: "agent" } }, projectEnv: { SHARED: "project" }, secretsSvc,
});
void opening.then(() => { settled = true; }, () => { settled = true; });
await vi.waitFor(() => expect(secretsSvc.collectMissingRuntimeBindings).toHaveBeenCalledTimes(3));
expect(secretsSvc.collectMissingAdapterConfigRuntimeBindings).toHaveBeenCalledOnce();
expect(settled).toBe(false);
expect(secretsSvc.resolveAdapterConfigForRuntime).not.toHaveBeenCalled();
expect(secretsSvc.resolveEnvBindings).not.toHaveBeenCalled();
projectCheck.resolve([]);
await expect(opening).rejects.toBe(firstFailure);
});
it("keeps missing scoped credentials as a pre-dispatch failure", async () => {
const secretsSvc = {
resolveAdapterConfigForRuntime: vi.fn(),
@@ -394,76 +394,31 @@ export async function resolveExecutionRunAdapterConfig(input: {
// no binding before resolving any secret value. Missing bindings short-circuit
// to a configuration-incomplete blocker routed to a human owner instead of a
// dispatched-then-failed run (which previously surfaced as opaque setup_failed).
if (typeof input.secretsSvc.collectMissingRuntimeBindings === "function") {
const missingBindings: MissingRuntimeBinding[] = [];
if (environmentEnv && input.environmentId) {
missingBindings.push(
...(await input.secretsSvc.collectMissingRuntimeBindings(
input.companyId,
environmentEnv,
{
consumerType: "environment",
consumerId: input.environmentId,
responsibleUserId: input.responsibleUserId ?? null,
},
)),
);
}
const collectMissing = input.secretsSvc.collectMissingRuntimeBindings;
if (typeof collectMissing === "function") {
const checks: Array<() => Promise<MissingRuntimeBinding[]>> = [];
const checkEnv = (env: unknown, consumerType: "environment" | "agent" | "project" | "routine", consumerId: string) => {
checks.push(() => collectMissing(input.companyId, env, {
consumerType, consumerId, responsibleUserId: input.responsibleUserId ?? null,
}));
};
if (environmentEnv && input.environmentId) checkEnv(environmentEnv, "environment", input.environmentId);
if (input.agentId) {
missingBindings.push(
...(await input.secretsSvc.collectMissingRuntimeBindings(
input.companyId,
parseObject(executionRunConfig.env),
{
consumerType: "agent",
consumerId: input.agentId,
responsibleUserId: input.responsibleUserId ?? null,
},
)),
);
if (
typeof input.secretsSvc.collectMissingAdapterConfigRuntimeBindings ===
"function"
) {
missingBindings.push(
...(await input.secretsSvc.collectMissingAdapterConfigRuntimeBindings(
input.companyId,
executionRunConfig,
input.adapterType ?? null,
{
consumerType: "agent",
consumerId: input.agentId,
responsibleUserId: input.responsibleUserId ?? null,
},
)),
);
checkEnv(parseObject(executionRunConfig.env), "agent", input.agentId);
const collectAdapterMissing = input.secretsSvc.collectMissingAdapterConfigRuntimeBindings;
if (typeof collectAdapterMissing === "function") {
checks.push(() => collectAdapterMissing(input.companyId, executionRunConfig, input.adapterType ?? null, {
consumerType: "agent", consumerId: input.agentId!, responsibleUserId: input.responsibleUserId ?? null,
}));
}
}
if (projectEnv && input.projectId) {
missingBindings.push(
...(await input.secretsSvc.collectMissingRuntimeBindings(
input.companyId,
projectEnv,
{
consumerType: "project",
consumerId: input.projectId,
responsibleUserId: input.responsibleUserId ?? null,
},
)),
);
}
if (routineEnv && input.routineId) {
missingBindings.push(
...(await input.secretsSvc.collectMissingRuntimeBindings(
input.companyId,
routineEnv,
{
consumerType: "routine",
consumerId: input.routineId,
responsibleUserId: input.responsibleUserId ?? null,
},
)),
);
if (projectEnv && input.projectId) checkEnv(projectEnv, "project", input.projectId);
if (routineEnv && input.routineId) checkEnv(routineEnv, "routine", input.routineId);
const results = await Promise.allSettled(checks.map(check => Promise.resolve().then(check)));
const missingBindings: MissingRuntimeBinding[] = [];
for (const result of results) {
if (result.status === "rejected") throw result.reason;
missingBindings.push(...result.value);
}
if (requiredScopedEnvBinding) {
const requiredEnvKeys = new Set(requiredScopedEnvBinding.keys);
@@ -2244,12 +2199,45 @@ export async function attestReviewedExternalChatRun(input: {
throw new Error("reviewed_chat_execution_binding_not_ready");
}
export async function readHeartbeatWakeComments(db: Db, companyId: string, issueId: string | null, commentIds: string[]) {
if (commentIds.length === 0) return [];
return db
.select({
id: issueComments.id,
companyId: issueComments.companyId,
issueId: issueComments.issueId,
body: issueComments.body,
origin: issueComments.origin,
authorType: issueComments.authorType,
authorAgentId: issueComments.authorAgentId,
authorUserId: issueComments.authorUserId,
presentation: issueComments.presentation,
metadata: issueComments.metadata,
deletedAt: issueComments.deletedAt,
deletedByType: issueComments.deletedByType,
deletedByAgentId: issueComments.deletedByAgentId,
deletedByUserId: issueComments.deletedByUserId,
deletedByRunId: issueComments.deletedByRunId,
sourceTrust: issueComments.sourceTrust,
createdAt: issueComments.createdAt,
})
.from(issueComments)
.where(
and(
eq(issueComments.companyId, companyId),
issueId ? eq(issueComments.issueId, issueId) : undefined,
inArray(issueComments.id, commentIds),
),
);
}
export async function buildPaperclipWakePayload(input: {
db: Db;
companyId: string;
agentId?: string | null;
runId?: string | null;
contextSnapshot: Record<string, unknown>;
wakeCommentRows?: Awaited<ReturnType<typeof readHeartbeatWakeComments>>;
continuationSummary?: {
key: string;
title: string | null;
@@ -2314,36 +2302,9 @@ export async function buildPaperclipWakePayload(input: {
)
return null;
const commentRows =
commentIds.length === 0
? []
: await input.db
.select({
id: issueComments.id,
issueId: issueComments.issueId,
body: issueComments.body,
origin: issueComments.origin,
authorType: issueComments.authorType,
authorAgentId: issueComments.authorAgentId,
authorUserId: issueComments.authorUserId,
presentation: issueComments.presentation,
metadata: issueComments.metadata,
deletedAt: issueComments.deletedAt,
deletedByType: issueComments.deletedByType,
deletedByAgentId: issueComments.deletedByAgentId,
deletedByUserId: issueComments.deletedByUserId,
deletedByRunId: issueComments.deletedByRunId,
sourceTrust: issueComments.sourceTrust,
createdAt: issueComments.createdAt,
})
.from(issueComments)
.where(
and(
eq(issueComments.companyId, input.companyId),
issueId ? eq(issueComments.issueId, issueId) : undefined,
inArray(issueComments.id, commentIds),
),
);
const commentRows = input.wakeCommentRows
? input.wakeCommentRows.filter(row => row.companyId === input.companyId && (!issueId || row.issueId === issueId) && commentIds.includes(row.id))
: await readHeartbeatWakeComments(input.db, input.companyId, issueId, commentIds);
const commentsById = new Map(
commentRows.map((comment) => [comment.id, comment]),
@@ -1,3 +1,4 @@
import { notifyStatusCardIssueChange } from "../status-card-notifications.js";
import { DELIVERY_QUEUES, notifyDeliveryWork } from "../delivery-work-notifications.js";
import { notifyIssueReviewState } from "../issue-review-state-notifications.js";
import { insertIssueComment } from "../issue-comment-writes.js";
@@ -595,6 +596,7 @@ export function createHeartbeatScheduling(
try {
await db.transaction(async (tx) => {
await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors);
await notifyStatusCardIssueChange(tx, claimed.companyId);
const released = await tx.update(issues)
.set({ monitorWakeRequestedAt: null, updatedAt: input.now })
.where(issueMonitorClaimCondition(claimed)).returning({ id: issues.id });
@@ -628,6 +630,7 @@ export function createHeartbeatScheduling(
// failed, and never clobber a monitor that was replaced in the meantime.
await db.transaction(async tx => {
await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors);
await notifyStatusCardIssueChange(tx, claimed.companyId);
await tx.update(issues).set({ monitorWakeRequestedAt: null, updatedAt: input.now })
.where(issueMonitorClaimCondition(claimed));
});
@@ -827,6 +830,7 @@ export function createHeartbeatScheduling(
}
const consumed = await db.transaction(async tx => {
await notifyStatusCardIssueChange(tx, claimed.companyId);
await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors);
return tx.update(issues)
.set({
@@ -977,6 +981,7 @@ export function createHeartbeatScheduling(
const staleClaimThreshold = new Date(now.getTime() - 5 * 60 * 1000);
const claimed = await db.transaction(async (tx) => {
await notifyStatusCardIssueChange(tx, issue.companyId);
await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors);
const [updated] = await tx
.update(issues)
@@ -1050,6 +1055,7 @@ export function createHeartbeatScheduling(
for (const due of dueMonitors) {
if (!canContinue()) break;
const claimed = await db.transaction(async (tx) => {
await notifyStatusCardIssueChange(tx, due.companyId);
await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors);
const [updated] = await tx
.update(issues)
+57 -35
View File
@@ -1,5 +1,5 @@
import { createHash } from "node:crypto";
import type { Db, agents, agentTaskSessions, heartbeatRuns, issueComments } from "@paperclipai/db";
import type { Db, agents, agentTaskSessions, heartbeatRuns } from "@paperclipai/db";
import { FAST_RESPONSE_AGENT_GUIDANCE, fastResponseHistoryBody } from "@paperclipai/shared";
import type { PaperclipTurnContext } from "@paperclipai/adapter-utils/server-utils";
import { parseObject } from "../../adapters/utils.js";
@@ -15,8 +15,9 @@ import { getTaskPlanContext } from "../task-plan-context.js";
import { caseExecutionMarkdown, getCaseExecutionContext } from "../case-context.js";
import type { TrustPresetResolution } from "../trust-preset-resolver.js";
import { liveVoiceExecutionGuidance } from "../voice/voice-execution-guidance.js";
import { buildPaperclipWakePayload, ConfigurationIncompleteFailure, PAPERCLIP_WAKE_PAYLOAD_KEY, type createHeartbeatRunPreparation } from "./run-preparation.js";
import { buildPaperclipWakePayload, ConfigurationIncompleteFailure, readHeartbeatWakeComments, PAPERCLIP_WAKE_PAYLOAD_KEY, type createHeartbeatRunPreparation } from "./run-preparation.js";
import { buildPaperclipTaskMarkdown } from "./task-markdown.js";
import { deriveCommentId, extractWakeCommentIds } from "../../modules/run-dispatch/index.js";
import { workspacePreferenceService } from "../workspace-preferences.js";
type RunPreparation = ReturnType<typeof createHeartbeatRunPreparation>;
@@ -30,8 +31,6 @@ export interface HeartbeatTaskContextInput {
"id" | "identifier" | "title" | "status" | "priority" | "workMode" |
"conversationAgentId" | "description" | "projectId"> | null;
issueContext: IssueContext | null;
wakeCommentContext: Pick<typeof issueComments.$inferSelect,
"id" | "body" | "sourceTrust" | "presentation" | "metadata"> | null;
context: Record<string, unknown>;
taskSession: Pick<typeof agentTaskSessions.$inferSelect, "lastRunId"> | null;
};
@@ -53,35 +52,9 @@ function readNonEmptyString(value: unknown): string | null {
/** Rebuilds task prompt fields on the run's context; acquires no execution resources. */
export async function prepareHeartbeatTaskContext(db: Db, input: HeartbeatTaskContextInput) {
const { run, agent } = input;
const { issueRef, issueContext, wakeCommentContext, context, taskSession } = input.task;
const { issueRef, issueContext, context, taskSession } = input.task;
const { trustPreset, experimentalInstanceSettings } = input.policy;
const { issuesSvc, getPinnedSkillTestContext } = input.services;
const continuationSummary = issueRef && !isConversation(issueContext)
? await getIssueContinuationSummaryDocument(db, issueRef.id)
: null;
const exposeLowTrustRaw = trustPreset.kind === "low_trust_review";
const safeContinuationSummary =
continuationSummary && !exposeLowTrustRaw
? redactQuarantinedBodyForHigherTrust(continuationSummary)
: continuationSummary;
const safeWakeCommentContext =
wakeCommentContext && !exposeLowTrustRaw
? sanitizeQuarantinedCommentForHigherTrust(wakeCommentContext)
: wakeCommentContext;
const issueAncestors = issueRef
? await issuesSvc.getAncestors(issueRef.id)
: [];
if (continuationSummary) {
context.paperclipContinuationSummary = {
key: safeContinuationSummary!.key,
title: safeContinuationSummary!.title,
body: safeContinuationSummary!.body,
sourceTrust: safeContinuationSummary!.sourceTrust ?? null,
updatedAt: safeContinuationSummary!.updatedAt.toISOString(),
};
} else {
delete context.paperclipContinuationSummary;
}
const admittedScope = parseObject(context.paperclipAgentJwtScope);
if (!input.policy.recovering || (admittedScope.kind !== "standard" && admittedScope.kind !== "skill_test")) {
const savedIssue = parseObject(context.paperclipIssue);
@@ -93,10 +66,58 @@ export async function prepareHeartbeatTaskContext(db: Db, input: HeartbeatTaskCo
: { kind: "standard" };
}
const isSkillTest = parseObject(context.paperclipAgentJwtScope).kind === "skill_test";
const pinnedSkillTestContext =
isSkillTest && issueRef
? await getPinnedSkillTestContext(agent.companyId, issueRef.id)
: null;
const wakeCommentId = deriveCommentId(context, null);
const commentIds = [...new Set([...extractWakeCommentIds(context), ...(wakeCommentId ? [wakeCommentId] : [])])];
const [summary, ancestors, pinnedSkillTestRead, wakeComments] = await Promise.allSettled([
issueRef && !isConversation(issueContext) ? getIssueContinuationSummaryDocument(db, issueRef.id) : Promise.resolve(null),
issueRef ? issuesSvc.getAncestors(issueRef.id) : Promise.resolve([]),
isSkillTest && issueRef ? getPinnedSkillTestContext(agent.companyId, issueRef.id) : Promise.resolve(null),
readHeartbeatWakeComments(db, agent.companyId, readNonEmptyString(context.issueId), commentIds),
]);
if (summary.status === "rejected") throw summary.reason;
if (ancestors.status === "rejected") throw ancestors.reason;
if (pinnedSkillTestRead.status === "rejected") throw pinnedSkillTestRead.reason;
if (wakeComments.status === "rejected") throw wakeComments.reason;
const wakeCommentRows = wakeComments.value;
const selectedWakeComment = issueRef && wakeCommentId ? wakeCommentRows.find(comment => comment.id === wakeCommentId && comment.issueId === issueRef.id) ?? null : null;
const wakeCommentContext = selectedWakeComment ? {
id: selectedWakeComment.id,
body: selectedWakeComment.deletedAt ? "" : selectedWakeComment.body,
authorType: selectedWakeComment.authorType,
authorAgentId: selectedWakeComment.authorAgentId,
authorUserId: selectedWakeComment.authorUserId,
presentation: selectedWakeComment.deletedAt ? null : selectedWakeComment.presentation,
metadata: selectedWakeComment.deletedAt ? null : selectedWakeComment.metadata,
deletedAt: selectedWakeComment.deletedAt,
deletedByType: selectedWakeComment.deletedByType,
deletedByAgentId: selectedWakeComment.deletedByAgentId,
deletedByUserId: selectedWakeComment.deletedByUserId,
deletedByRunId: selectedWakeComment.deletedByRunId,
sourceTrust: selectedWakeComment.sourceTrust,
} : null;
const continuationSummary = summary.value;
const issueAncestors = ancestors.value;
const pinnedSkillTestContext = pinnedSkillTestRead.value;
const exposeLowTrustRaw = trustPreset.kind === "low_trust_review";
const safeContinuationSummary =
continuationSummary && !exposeLowTrustRaw
? redactQuarantinedBodyForHigherTrust(continuationSummary)
: continuationSummary;
const safeWakeCommentContext =
wakeCommentContext && !exposeLowTrustRaw
? sanitizeQuarantinedCommentForHigherTrust(wakeCommentContext)
: wakeCommentContext;
if (continuationSummary) {
context.paperclipContinuationSummary = {
key: safeContinuationSummary!.key,
title: safeContinuationSummary!.title,
body: safeContinuationSummary!.body,
sourceTrust: safeContinuationSummary!.sourceTrust ?? null,
updatedAt: safeContinuationSummary!.updatedAt.toISOString(),
};
} else {
delete context.paperclipContinuationSummary;
}
if (isSkillTest && !pinnedSkillTestContext) {
throw new ConfigurationIncompleteFailure("The skill test's saved version is unavailable.", {
configurationIncomplete: { reason: "skill_unavailable", actionUrl: "/skills" },
@@ -132,6 +153,7 @@ export async function prepareHeartbeatTaskContext(db: Db, input: HeartbeatTaskCo
agentId: agent.id,
runId: run.id,
contextSnapshot: context,
wakeCommentRows,
continuationSummary,
issueSummary: issueRef
? {
+6
View File
@@ -1,3 +1,6 @@
import { notifyStatusCardEvaluation } from "./status-card-notifications.js";
import { notifyDeferredExecutionEligibility } from "./deferred-execution-notifications.js";
import { notifyRunDispatch } from "./run-dispatch-notifications.js";
import { notifySkillVersionRetentionReconstruction } from "./skill-version-retention-notifications.js";
import { notifyNativeEnvironmentSettlement } from "./native-environment-settlement-notifications.js";
import { notifyExternalAgentSettlement } from "../modules/external-agents/index.js";
@@ -625,11 +628,14 @@ export function instanceSettingsService(db: Db, options: InstanceSettingsService
}
const now = new Date();
const [updated] = await db.transaction(async tx => {
if (patch.enableStatusCards !== undefined) await notifyStatusCardEvaluation(tx);
await notifyDeliveryWork(tx, DELIVERY_QUEUES.mcpEvents, "*");
await notifyDeliveryWork(tx, DELIVERY_QUEUES.dotEvents, "*");
await notifyDeliveryWork(tx, DELIVERY_QUEUES.grokDispatch);
if (patch.enableWorktreeRunExecution !== undefined) await notifyScheduledRetryCancellation(tx);
if (patch.enableWorktreeRunExecution !== undefined) {
await notifyRunDispatch(tx);
await notifyDeferredExecutionEligibility(tx);
await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers);
await notifyExternalAgentSettlement(tx);
await notifyInstructionCommit(tx);
@@ -1,3 +1,5 @@
import { notifyStatusCardIssueChange } from "./status-card-notifications.js";
import { notifyIssueRecovery } from "./issue-recovery-notifications.js";
import type { PgInsertValue } from "drizzle-orm/pg-core";
import { issueComments, type Db } from "@paperclipai/db";
import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js";
@@ -6,6 +8,9 @@ type WriteDb = Db | Parameters<Parameters<Db["transaction"]>[0]>[0];
export function insertIssueComment(db: WriteDb, input: PgInsertValue<typeof issueComments> & { companyId: string }) {
return db.transaction(async tx => {
await notifyStatusCardIssueChange(tx, input.companyId);
await notifyIssueRecovery(tx, { companyId: input.companyId,
issueId: typeof input.issueId === "string" ? input.issueId : undefined });
await notifyDeliveryWork(tx, DELIVERY_QUEUES.dotEvents, `company:${input.companyId}`);
return tx.insert(issueComments).values(input).returning();
});
+2
View File
@@ -1,3 +1,4 @@
import { notifyStatusCardIssueChange } from "./status-card-notifications.js";
import { and, eq, sql } from "drizzle-orm";
import type { PgColumn } from "drizzle-orm/pg-core";
import type { Db } from "@paperclipai/db";
@@ -127,6 +128,7 @@ export async function rekeyCompanyIssueIdentifiers(
// overload, which returns NULL for a non-pattern argument.
const tailStart = sql`${fromPrefix.length + 1}::int`;
await notifyStatusCardIssueChange(tx, companyId);
const rekeyedIssues = await tx
.update(issues)
.set({ identifier: sql`${toPrefix} || substring(${issues.identifier} from ${tailStart})` })
@@ -1,3 +1,5 @@
import { notifyAttentionRetentionWork } from "./retention-work-notifications.js";
import { notifyIssueRecovery } from "./issue-recovery-notifications.js";
import { notifyNativeRecoveryNotices } from "./native-recovery-notice-notifications.js";
import { notifyUnsafeWorkspaceExportRecovery } from "./unsafe-workspace-export-notifications.js";
import { writeExecutionControl } from "./execution-control-notifications.js";
@@ -322,6 +324,8 @@ export function issueRecoveryActionService(db: DbOrTransaction) {
): Promise<IssueRecoveryAction> {
try {
const created = await db.transaction(async (tx) => {
await notifyAttentionRetentionWork(tx, { companyId: input.companyId });
await notifyIssueRecovery(tx, { companyId: input.companyId });
await notifyNativeRecoveryNotices(tx, { companyId: input.companyId, issueId: input.sourceIssueId });
await notifyDeliveryWork(tx, DELIVERY_QUEUES.executionDisposition);
await notifyDeliveryWork(tx, DELIVERY_QUEUES.executionContinuation);
@@ -413,6 +417,7 @@ export function issueRecoveryActionService(db: DbOrTransaction) {
Math.min(nextAttemptCount, effectiveMaxAttempts),
);
const [exhausted] = await writeExecutionControl(db, [DELIVERY_QUEUES.executionDisposition, DELIVERY_QUEUES.executionContinuation], async tx => {
await notifyIssueRecovery(tx, { companyId: input.companyId });
await notifyNativeRecoveryNotices(tx, { companyId: input.companyId, issueId: input.sourceIssueId });
if ((!input.preserveExistingOwner && existing.cause !== input.cause)
|| (input.evidence?.runId !== undefined && input.evidence.runId !== existing.evidence?.runId)) {
@@ -474,6 +479,7 @@ export function issueRecoveryActionService(db: DbOrTransaction) {
return toReadModel(exhausted);
}
const [updated] = await writeExecutionControl(db, [DELIVERY_QUEUES.executionDisposition, DELIVERY_QUEUES.executionContinuation], async tx => {
await notifyIssueRecovery(tx, { companyId: input.companyId });
await notifyNativeRecoveryNotices(tx, { companyId: input.companyId, issueId: input.sourceIssueId });
if ((!input.preserveExistingOwner && existing.cause !== input.cause)
|| (input.evidence?.runId !== undefined && input.evidence.runId !== existing.evidence?.runId)) {
@@ -548,6 +554,7 @@ export function issueRecoveryActionService(db: DbOrTransaction) {
try {
const [created] = await writeExecutionControl(db, [DELIVERY_QUEUES.executionDisposition, DELIVERY_QUEUES.executionContinuation], async tx => {
await notifyIssueRecovery(tx, { companyId: input.companyId });
await notifyNativeRecoveryNotices(tx, { companyId: input.companyId, issueId: input.sourceIssueId });
await notifyUnsafeWorkspaceExportRecovery(tx, { companyId: input.companyId, issueId: input.sourceIssueId });
return tx
@@ -595,6 +602,7 @@ export function issueRecoveryActionService(db: DbOrTransaction) {
const pending = await tx.select({ id: issueRecoveryActions.id }).from(issueRecoveryActions)
.where(and(...predicates)).limit(1);
if (!pending.length) return [];
await notifyIssueRecovery(tx, { companyId: input.companyId });
await notifyNativeRecoveryNotices(tx, { companyId: input.companyId, issueId: input.sourceIssueId });
await notifyUnsafeWorkspaceExportRecovery(tx, { companyId: input.companyId, issueId: input.sourceIssueId });
return tx
@@ -0,0 +1,16 @@
import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js";
/** An issue hint is reserved for local facts such as comments. Lifecycle,
* graph, run and eligibility changes retain company scope because their
* consequences can include ancestors and shared execution references. */
export type IssueRecoveryScope = { companyId: string; issueId?: string };
export async function notifyIssueRecovery(transaction: object, scope?: IssueRecoveryScope) {
const companyHint = scope ? JSON.stringify({ companyId: scope.companyId }) : "*";
const issueHint = scope ? JSON.stringify({ companyId: scope.companyId, issueId: scope.issueId }) : "*";
for (const queue of [DELIVERY_QUEUES.orphanBlockers, DELIVERY_QUEUES.activeRecoveryActions, DELIVERY_QUEUES.strandedAssignments, DELIVERY_QUEUES.resolvedDependencies,
DELIVERY_QUEUES.taskWatchdogs, DELIVERY_QUEUES.staleIssueLocks]) {
await notifyDeliveryWork(transaction, queue,
queue === DELIVERY_QUEUES.strandedAssignments || queue === DELIVERY_QUEUES.staleIssueLocks ? issueHint : companyHint);
}
}
@@ -1,4 +1,6 @@
import { notifyRoutineExecutionStatus } from "./routine-execution-status-notifications.js";
import { notifyStatusCardIssueChange } from "./status-card-notifications.js";
import { notifyRunDispatch } from "./run-dispatch-notifications.js";
import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js";
import { notifyExternalConversationSettlement } from "./external-conversation-settlement-notifications.js";
import { notifyNativeReviewResponse } from "./native-review-response-notifications.js";
@@ -6,6 +8,8 @@ import { notifyNativeStatusReassessment } from "./native-status-reassessment-not
export async function notifyIssueReviewState(tx: object, scope: { companyId: string; issueId?: string; runId?: string; agentId?: string }) {
await notifyRoutineExecutionStatus(tx, scope.issueId, scope.companyId);
await notifyStatusCardIssueChange(tx, scope.companyId);
await notifyRunDispatch(tx, scope);
await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors);
await notifyExternalConversationSettlement(tx, scope);
await notifyNativeReviewResponse(tx, scope);
@@ -1,5 +1,6 @@
import { beforeEach, describe, expect, it, vi } from "vitest";
import { getTableName } from "drizzle-orm";
import { PgDialect } from "drizzle-orm/pg-core";
import { getTableName, type SQL } from "drizzle-orm";
const mockCreateChild = vi.fn();
@@ -48,8 +49,18 @@ function createFakeDb(args: {
const toolActionRequestUpdates: Array<Record<string, unknown>> = [];
const inserts: Array<{ table: string; values: Record<string, unknown> }> = [];
let selectCallCount = 0;
let transactionDepth = 0;
const db: any = {
execute: vi.fn(async (query: SQL) => {
const statement = new PgDialect().sqlToQuery(query);
expect(transactionDepth).toBeGreaterThan(0);
expect(statement.params).toContain(interactionRow.companyId);
expect(statement.params).toContain(interactionRow.issueId);
if (statement.sql.includes("origin_kind")) return [{ id: interactionRow.issueId, originKind: null, originId: null }];
if (statement.sql.includes("with recursive affected")) return [{ issueId: interactionRow.issueId }];
throw new Error(`Unexpected interaction fixture query: ${statement.sql}`);
}),
select: vi.fn(() => {
selectCallCount += 1;
const rows = selectCallCount === 1 ? [interactionRow] : (args.parentRows ?? []);
@@ -84,7 +95,10 @@ function createFakeDb(args: {
inserts.push({ table: getTableName(table as never), values });
},
})),
transaction: async (callback: (tx: typeof db) => Promise<void>) => callback(db),
transaction: async (callback: (tx: typeof db) => Promise<void>) => {
transactionDepth++;
try { return await callback(db); } finally { transactionDepth--; }
},
};
return {
@@ -177,6 +191,7 @@ describe("issueThreadInteractionService", () => {
};
let selectCallCount = 0;
let transactionDepth = 0;
const db: any = {
select: vi.fn(() => {
selectCallCount += 1;
@@ -1,3 +1,4 @@
import { notifyStatusCardIssueChange } from "./status-card-notifications.js";
import { notifyNativeStatusReassessment } from "./native-status-reassessment-notifications.js";
import { notifyIssueReviewState } from "./issue-review-state-notifications.js";
import { notifyChatConfirmationResolution } from "./chat-confirmation-notifications.js";
@@ -376,7 +377,7 @@ type ResolvedInteractionResult = {
};
type IssueThreadInteractionRow = typeof issueThreadInteractions.$inferSelect;
type IssueTouchDb = Pick<Db, "update">;
type IssueTouchDb = Pick<Db, "update" | "transaction">;
function isNativeCompletionReview(
row: Pick<IssueThreadInteractionRow, "kind" | "payload">,
@@ -801,11 +802,11 @@ function hydrateInteraction(
}
}
async function touchIssue(db: IssueTouchDb, issueId: string) {
await db
.update(issues)
.set({ updatedAt: new Date() })
.where(eq(issues.id, issueId));
async function touchIssue(db: IssueTouchDb, issueId: string, companyId: string) {
await db.transaction(async tx => {
await notifyStatusCardIssueChange(tx, companyId);
await tx.update(issues).set({ updatedAt: new Date() }).where(eq(issues.id, issueId));
});
}
function isTerminalIssueStatus(status: string) {
@@ -2013,7 +2014,7 @@ async function expireStaleRequestConfirmationTarget(
"Interaction has already been resolved",
);
}
await touchIssue(tx, args.row.issueId);
await touchIssue(tx, args.row.issueId, args.row.companyId);
const expired = hydrateInteraction(updated);
await enqueueTerminalIssueInteractionChatPublications(tx, expired);
return expired;
@@ -2468,7 +2469,7 @@ export function issueThreadInteractionService(
};
}
} else {
await touchIssue(tx, args.issue.id);
await touchIssue(tx, args.issue.id, args.issue.companyId);
}
// Snapshot the continuation target only after every lifecycle mutation
@@ -2682,7 +2683,7 @@ export function issueThreadInteractionService(
tx,
);
} else {
await touchIssue(tx, args.issue.id);
await touchIssue(tx, args.issue.id, args.issue.companyId);
}
await enqueueTerminalIssueInteractionChatPublications(
tx as unknown as Db,
@@ -2864,7 +2865,7 @@ export function issueThreadInteractionService(
await enqueueTerminalIssueInteractionChatPublications(tx as unknown as Db, hydrateInteraction(previous));
}
}
await touchIssue(tx, issue.id);
await touchIssue(tx, issue.id, issue.companyId);
inserted = true;
await enqueueIssueInteractionChatPublications(tx as unknown as Db, hydrateInteraction(row));
await notifyDeliveryWork(tx, DELIVERY_QUEUES.connection);
@@ -2908,7 +2909,7 @@ export function issueThreadInteractionService(
)
.returning();
if (!updated) throw interactionAlreadyResolvedError();
await touchIssue(db, issue.id);
await touchIssue(db, issue.id, issue.companyId);
return hydrateInteraction(updated) as ConnectionIntentInteraction;
},
resolveConnectionIntent: async (
@@ -2972,7 +2973,7 @@ export function issueThreadInteractionService(
return row;
});
if (!updated) throw interactionAlreadyResolvedError();
await touchIssue(db, issue.id);
await touchIssue(db, issue.id, issue.companyId);
const interaction = hydrateInteraction(
updated,
) as ConnectionIntentInteraction;
@@ -3351,7 +3352,7 @@ export function issueThreadInteractionService(
})
.where(eq(issueThreadInteractions.id, current.id))
.returning();
await touchIssue(tx, issue.id);
await touchIssue(tx, issue.id, issue.companyId);
return row;
});
return hydrateInteraction(updated);
@@ -3414,10 +3415,8 @@ export function issueThreadInteractionService(
if (updated) cancelled.push(updated);
}
for (const issueId of new Set(
cancelled.map((interaction) => interaction.issueId),
)) {
await touchIssue(db, issueId);
for (const [issueId, companyId] of new Map(cancelled.map(interaction => [interaction.issueId, interaction.companyId]))) {
await touchIssue(db, issueId, companyId);
}
await emitResolvedInteractionsTelemetry(db, cancelled);
return cancelled;
@@ -3506,10 +3505,8 @@ export function issueThreadInteractionService(
}
if (expired.length > 0) {
for (const issueId of new Set(
expired.map((interaction) => interaction.issueId),
)) {
await touchIssue(db, issueId);
for (const [issueId, companyId] of new Map(expired.map(interaction => [interaction.issueId, interaction.companyId]))) {
await touchIssue(db, issueId, companyId);
}
await emitResolvedInteractionsTelemetry(db, expired);
}
@@ -4052,7 +4049,7 @@ export function issueThreadInteractionService(
return interaction;
}
await touchIssue(db, issue.id);
await touchIssue(db, issue.id, issue.companyId);
if (superseded.length > 0) {
await emitResolvedInteractionsTelemetry(
db,
@@ -4357,7 +4354,7 @@ export function issueThreadInteractionService(
.where(eq(issueThreadInteractions.id, interactionId))
.returning();
await touchIssue(tx, issue.id);
await touchIssue(tx, issue.id, issue.companyId);
await mutationOptions.afterResolveInTransaction?.(tx, hydrateInteraction(updated));
await enqueueTerminalIssueInteractionChatPublications(tx as unknown as Db, hydrateInteraction(updated));
current.status = updated.status;
@@ -4549,7 +4546,7 @@ export function issueThreadInteractionService(
throw interactionAlreadyResolvedError();
}
await touchIssue(tx, issue.id);
await touchIssue(tx, issue.id, issue.companyId);
await mutationOptions.afterResolveInTransaction?.(tx, hydrateInteraction(updated));
if (complete) await enqueueTerminalIssueInteractionChatPublications(tx as unknown as Db, hydrateInteraction(updated));
return {
@@ -4622,7 +4619,7 @@ export function issueThreadInteractionService(
throw interactionAlreadyResolvedError();
}
await touchIssue(db, issue.id);
await touchIssue(db, issue.id, issue.companyId);
const rejected = hydrateInteraction(updated);
await emitInteractionResolvedTelemetry(db, rejected);
return rejected;
@@ -4720,7 +4717,7 @@ export function issueThreadInteractionService(
}
if (expired.length > 0) {
await touchIssue(db, issue.id);
await touchIssue(db, issue.id, issue.companyId);
await emitResolvedInteractionsTelemetry(db, expired);
}
return expired;
@@ -4952,7 +4949,7 @@ export function issueThreadInteractionService(
}
if (expired.length > 0) {
await touchIssue(db, issue.id);
await touchIssue(db, issue.id, issue.companyId);
await emitResolvedInteractionsTelemetry(db, expired);
}
return expired;
@@ -5049,7 +5046,7 @@ export function issueThreadInteractionService(
}
if (expired.length > 0) {
await touchIssue(db, issue.id);
await touchIssue(db, issue.id, issue.companyId);
await emitResolvedInteractionsTelemetry(db, expired);
}
return expired;
@@ -5189,7 +5186,7 @@ export function issueThreadInteractionService(
if (updated) expired.push(hydrateInteraction(updated));
}
if (expired.length > 0) {
await touchIssue(db, issue.id);
await touchIssue(db, issue.id, issue.companyId);
await emitResolvedInteractionsTelemetry(db, expired);
}
return expired;
@@ -5294,7 +5291,7 @@ export function issueThreadInteractionService(
return row;
});
await touchIssue(db, issue.id);
await touchIssue(db, issue.id, issue.companyId);
const withdrawn = hydrateInteraction(updated);
await emitInteractionResolvedTelemetry(db, withdrawn);
return withdrawn;
@@ -5417,7 +5414,7 @@ export function issueThreadInteractionService(
return row;
});
await touchIssue(db, issue.id);
await touchIssue(db, issue.id, issue.companyId);
const answered = hydrateInteraction(updated);
await emitInteractionResolvedTelemetry(db, answered);
return answered;
@@ -5553,7 +5550,7 @@ export function issueThreadInteractionService(
return row;
});
await touchIssue(db, issue.id);
await touchIssue(db, issue.id, issue.companyId);
const skipped = hydrateInteraction(updated);
await emitInteractionResolvedTelemetry(db, skipped);
return skipped;
@@ -5630,7 +5627,7 @@ export function issueThreadInteractionService(
return row;
});
await touchIssue(db, issue.id);
await touchIssue(db, issue.id, issue.companyId);
const cancelled = hydrateInteraction(updated);
await emitInteractionResolvedTelemetry(db, cancelled);
return cancelled;
+4
View File
@@ -1,3 +1,5 @@
import { notifyAttentionRetentionWork } from "./retention-work-notifications.js";
import { notifyStatusCardIssueChange } from "./status-card-notifications.js";
import { and, eq } from "drizzle-orm";
import { heartbeatRuns, issues, type Db } from "@paperclipai/db";
import { setIssueTitleSchema, type SetIssueTitle } from "@paperclipai/shared";
@@ -76,6 +78,8 @@ export async function setIssueTitle(
|| (issue.title === title && !issue.titleNeedsGeneration)) {
return { result: await saveReceipt({ id: issue.id, title: issue.title, titleNeedsGeneration: issue.titleNeedsGeneration, changed: false }), publication: null };
}
await notifyAttentionRetentionWork(tx, { companyId });
await notifyStatusCardIssueChange(tx, companyId);
await tx.update(issues).set({ title, titleNeedsGeneration: false, updatedAt: new Date() }).where(predicate);
await issueReferenceService(tx).syncIssue(issueId, tx);
const { publication } = await persistActivity(tx, {
+14 -1
View File
@@ -1,4 +1,7 @@
import { notifyRoutineExecutionStatus } from "./routine-execution-status-notifications.js";
import { notifyStatusCardIssueChange } from "./status-card-notifications.js";
import { notifyDeferredExecutionEligibility } from "./deferred-execution-notifications.js";
import { notifyIssueRecovery } from "./issue-recovery-notifications.js";
import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js";
import { notifyNativeStatusReassessment } from "./native-status-reassessment-notifications.js";
import { notifyUnsafeWorkspaceExportRecovery } from "./unsafe-workspace-export-notifications.js";
@@ -759,6 +762,8 @@ export function issueTreeControlService(db: Db) {
const releaseReason = input.reason ?? "Subtree resume applied.";
const { hold: resumeHold } = await db.transaction(async (tx) => {
await notifyIssueRecovery(tx, { companyId });
await notifyDeferredExecutionEligibility(tx, { companyId });
const [createdHold] = await tx
.insert(issueTreeHolds)
.values({
@@ -837,6 +842,8 @@ export function issueTreeControlService(db: Db) {
}
const { hold, members } = await db.transaction(async (tx) => {
await notifyIssueRecovery(tx, { companyId });
await notifyDeferredExecutionEligibility(tx, { companyId });
const [createdHold] = await tx
.insert(issueTreeHolds)
.values({
@@ -1001,6 +1008,7 @@ export function issueTreeControlService(db: Db) {
const now = new Date();
const releasedCancelHoldIds = activeCancelHolds.map((hold) => hold.id);
const updatedIssues = await db.transaction(async (tx) => {
await notifyDeferredExecutionEligibility(tx, { companyId });
await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors);
await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers);
const restored: TreeStatusUpdateResult["updatedIssues"] = [];
@@ -1013,6 +1021,7 @@ export function issueTreeControlService(db: Db) {
await notifyChatResponseAvailability(tx, { companyId, issueId });
if (status === "blocked" || status === "in_review") await notifyUnsafeWorkspaceExportRecovery(tx, { companyId, issueId });
}
await notifyStatusCardIssueChange(tx, companyId);
const rows = await tx
.update(issues)
.set({
@@ -1179,7 +1188,10 @@ export function issueTreeControlService(db: Db) {
throw conflict("Issue tree hold is already released");
}
const [updated] = await db
const [updated] = await db.transaction(async tx => {
await notifyIssueRecovery(tx, { companyId });
await notifyDeferredExecutionEligibility(tx, { companyId });
return tx
.update(issueTreeHolds)
.set({
status: "released",
@@ -1197,6 +1209,7 @@ export function issueTreeControlService(db: Db) {
})
.where(and(eq(issueTreeHolds.id, holdId), eq(issueTreeHolds.companyId, companyId)))
.returning();
});
const members = await db
.select()
+7 -1
View File
@@ -1,5 +1,5 @@
import { nonIdleSlackIssueCondition } from "./slack-conversation-state.js";
import { and, isNull, type SQL } from "drizzle-orm";
import { and, isNull, isNotNull, or, type SQL } from "drizzle-orm";
import { issues } from "@paperclipai/db";
export function visibleIssueCondition(): SQL {
@@ -14,3 +14,9 @@ export function visibleIssueSql(alias = "issues") {
export function executionIssueCondition(): SQL {
return and(visibleIssueCondition(), isNull(issues.conversationAgentId), nonIdleSlackIssueCondition())!;
}
export function attentionIssueCondition(includeConversations = false): SQL {
return includeConversations
? or(executionIssueCondition(), and(visibleIssueCondition(), isNotNull(issues.conversationAgentId)))!
: executionIssueCondition();
}
+259 -170
View File
@@ -1,4 +1,9 @@
import { notifyRoutineExecutionStatus } from "./routine-execution-status-notifications.js";
import { notifyIssueAttentionRetentionWork } from "./retention-work-notifications.js";
import { notifyStatusCardIssueChange } from "./status-card-notifications.js";
import { notifyDeferredExecutionEligibility } from "./deferred-execution-notifications.js";
import { notifyIssueRecovery } from "./issue-recovery-notifications.js";
import { notifyRunDispatch } from "./run-dispatch-notifications.js";
import { notifyExternalConversationSettlement } from "./external-conversation-settlement-notifications.js";
import { notifyNativeRecoveryNotices } from "./native-recovery-notice-notifications.js";
import { notifyNativeRunProjection } from "./native-run-projection-notifications.js";
@@ -2538,33 +2543,37 @@ async function listPendingFinalizeBlockerIssueIds(
),
);
if (lockSources) {
const rows: Array<{ issueId: string | null; executionWorkspaceId: string | null;
phase: string; status: string; startedAt: Date }> = [];
for (const chunk of chunkList(executionWorkspaceIds, ISSUE_LIST_RELATED_QUERY_CHUNK_SIZE)) {
if (lockSources) {
await dbOrTx.select({ id: executionWorkspaces.id }).from(executionWorkspaces)
.where(and(eq(executionWorkspaces.companyId, companyId), inArray(executionWorkspaces.id, executionWorkspaceIds)))
.for("update", { noWait: true });
}
const query = dbOrTx
.select({
issueId: workspaceOperations.issueId,
executionWorkspaceId: workspaceOperations.executionWorkspaceId,
phase: workspaceOperations.phase,
status: workspaceOperations.status,
startedAt: workspaceOperations.startedAt,
})
.from(workspaceOperations)
.where(
and(
eq(workspaceOperations.companyId, companyId),
inArray(
workspaceOperations.executionWorkspaceId,
executionWorkspaceIds,
await dbOrTx.select({ id: executionWorkspaces.id }).from(executionWorkspaces)
.where(and(eq(executionWorkspaces.companyId, companyId), inArray(executionWorkspaces.id, chunk)))
.for("update", { noWait: true });
}
const query = dbOrTx
.select({
issueId: workspaceOperations.issueId,
executionWorkspaceId: workspaceOperations.executionWorkspaceId,
phase: workspaceOperations.phase,
status: workspaceOperations.status,
startedAt: workspaceOperations.startedAt,
})
.from(workspaceOperations)
.where(
and(
eq(workspaceOperations.companyId, companyId),
inArray(
workspaceOperations.executionWorkspaceId,
chunk,
),
),
),
);
);
const rows = await (lockSources ? query.for("share", { noWait: true }) : query);
rows.push(...await (lockSources ? query.for("share", { noWait: true }) : query));
}
const latestAttributedByBlockerWorkspace = new Map<
string,
@@ -3432,6 +3441,7 @@ type IssueBlockerAttentionEdge = {
blockerIssueId: string;
};
type IssueBlockerAttentionQueryRow = IssueBlockerAttentionNode & {
executionWorkspaceId?: string | null;
issueId: string | null;
blockerIssueId: string;
};
@@ -3820,11 +3830,15 @@ async function terminalExplicitBlockersByRoot(
return terminalByRoot;
}
async function listIssueBlockerAttentionMap(
async function listIssueBlockerAttentionMap(dbOrTx: any, companyId: string, issueRows: IssueBlockerAttentionInputNode[]) {
return (await readIssueBlockerAttentionBatch(dbOrTx, companyId, issueRows)).attention;
}
async function readIssueBlockerAttentionBatch(
dbOrTx: any,
companyId: string,
issueRows: IssueBlockerAttentionInputNode[],
): Promise<Map<string, IssueBlockerAttention>> {
) {
const roots = issueRows.filter(
(row) => row.companyId === companyId && row.status === "blocked",
);
@@ -3834,36 +3848,42 @@ async function listIssueBlockerAttentionMap(
attentionMap.set(row.id, createIssueBlockerAttention());
}
}
if (roots.length === 0) return attentionMap;
const deferredRootIds = new Set<string>(roots.slice(BLOCKER_ATTENTION_MAX_NODES).map(root => root.id));
let edgeCount = 0, fetchedRows = 0;
const result = () => ({ attention: attentionMap, deferredRootIds, nodeCount: nodesById.size, edgeCount, fetchedRows });
if (roots.length === 0) return { attention: attentionMap, deferredRootIds, nodeCount: 0, edgeCount, fetchedRows };
const nodesById = new Map<string, IssueBlockerAttentionNode>();
const edgesByIssueId = new Map<string, IssueBlockerAttentionEdge[]>();
for (const root of roots) nodesById.set(root.id, { ...root });
// Returned rows are already bounded by the residual fetch budget. Keep their
// positive live-work evidence even when they cannot enter the traversal graph.
const observedNodes = new Map<string, IssueBlockerAttentionNode>();
const observedChildrenByParent = new Map<string, Set<string>>();
const admittedRoots = roots.slice(0, BLOCKER_ATTENTION_MAX_NODES);
for (const root of admittedRoots) nodesById.set(root.id, { ...root });
let frontier = roots.map((root) => root.id);
let truncated = false;
type Traversal = { rootId: string; nodes: Set<string>; expanded: Set<string>; frontier: string[]; truncated: boolean };
const traversals = new Map<string, Traversal>(admittedRoots.map(root => [root.id, {
rootId: root.id, nodes: new Set([root.id]), expanded: new Set<string>(), frontier: [root.id], truncated: false,
}]));
const expanded = new Set<string>();
const queried = new Set<string>();
const pendingFinalizeBlockerIssueIds = new Set<string>();
for (
let depth = 0;
frontier.length > 0 && depth < BLOCKER_ATTENTION_MAX_DEPTH;
depth += 1
) {
const nextFrontier = new Set<string>();
for (let depth = 0; depth < BLOCKER_ATTENTION_MAX_DEPTH; depth += 1) {
const active = [...traversals.values()].filter(traversal => !traversal.truncated && traversal.frontier.length > 0);
if (active.length === 0) break;
const unread = new Set<string>();
for (const traversal of active) for (const id of traversal.frontier) if (!expanded.has(id)) unread.add(id);
const frontier = [...unread];
for (const chunk of chunkList(
[...new Set(frontier)],
frontier,
ISSUE_LIST_RELATED_QUERY_CHUNK_SIZE,
)) {
const readinessByIssueId = await listIssueDependencyReadinessMap(
dbOrTx,
companyId,
chunk,
);
for (const readiness of readinessByIssueId.values()) {
for (const blockerIssueId of readiness.pendingFinalizeBlockerIssueIds) {
pendingFinalizeBlockerIssueIds.add(blockerIssueId);
}
}
const remaining = Math.min(BLOCKER_ATTENTION_MAX_NODES - nodesById.size,
BLOCKER_ATTENTION_MAX_NODES * 2 - edgeCount);
if (remaining <= 0) break;
const rowLimit = remaining + 1;
const explicitBlockerRowsPromise: Promise<
IssueBlockerAttentionQueryRow[]
> = dbOrTx
@@ -3877,6 +3897,7 @@ async function listIssueBlockerAttentionMap(
title: issues.title,
status: issues.status,
executionRunId: issues.executionRunId,
executionWorkspaceId: issues.executionWorkspaceId,
assigneeAgentId: issues.assigneeAgentId,
assigneeUserId: issues.assigneeUserId,
})
@@ -3889,7 +3910,7 @@ async function listIssueBlockerAttentionMap(
inArray(issueRelations.relatedIssueId, chunk),
eq(issues.companyId, companyId),
),
);
).limit(rowLimit);
const childRowsPromise: Promise<IssueBlockerAttentionQueryRow[]> = dbOrTx
.select({
issueId: issues.parentId,
@@ -3914,68 +3935,83 @@ async function listIssueBlockerAttentionMap(
BLOCKER_ATTENTION_CHILD_TERMINAL_STATUSES,
),
),
);
).limit(rowLimit);
const [explicitBlockerRows, childRows] = await Promise.all([
explicitBlockerRowsPromise,
childRowsPromise,
]);
for (const id of chunk) queried.add(id);
fetchedRows += explicitBlockerRows.length + childRows.length;
const pendingFinalize = await listPendingFinalizeBlockerIssueIds(dbOrTx, companyId,
explicitBlockerRows.flatMap(row => row.status === "done" && row.executionWorkspaceId
? [{ blockerIssueId: row.blockerIssueId, executionWorkspaceId: row.executionWorkspaceId }] : []));
for (const id of pendingFinalize) pendingFinalizeBlockerIssueIds.add(id);
let complete = explicitBlockerRows.length < rowLimit && childRows.length < rowLimit;
const unresolvedExplicitBlockerRows = explicitBlockerRows.filter(
(row) =>
row.status !== "done" ||
pendingFinalizeBlockerIssueIds.has(row.blockerIssueId),
);
appendBlockerAttentionEdges(edgesByIssueId, [
...unresolvedExplicitBlockerRows
.filter(
(row): row is IssueBlockerAttentionQueryRow & { issueId: string } =>
row.issueId !== null,
)
.map((row) => ({
issueId: row.issueId,
blockerIssueId: row.blockerIssueId,
})),
...childRows
.filter(
(row): row is IssueBlockerAttentionQueryRow & { issueId: string } =>
row.issueId !== null,
)
.map((row) => ({
issueId: row.issueId,
blockerIssueId: row.blockerIssueId,
})),
]);
for (const row of [...unresolvedExplicitBlockerRows, ...childRows]) {
if (!row.issueId || nodesById.has(row.blockerIssueId)) continue;
nodesById.set(row.blockerIssueId, {
id: row.blockerIssueId,
companyId: row.companyId,
parentId: row.parentId,
identifier: row.identifier,
title: row.title,
status: row.status,
executionRunId: row.executionRunId,
assigneeAgentId: row.assigneeAgentId,
assigneeUserId: row.assigneeUserId,
});
nextFrontier.add(row.blockerIssueId);
if (!row.issueId) continue;
observedNodes.set(row.blockerIssueId, row);
const observed = observedChildrenByParent.get(row.issueId) ?? new Set<string>();
observed.add(row.blockerIssueId);
observedChildrenByParent.set(row.issueId, observed);
if (!nodesById.has(row.blockerIssueId)) {
if (nodesById.size >= BLOCKER_ATTENTION_MAX_NODES) { complete = false; continue; }
nodesById.set(row.blockerIssueId, {
id: row.blockerIssueId, companyId: row.companyId, parentId: row.parentId, identifier: row.identifier,
title: row.title, status: row.status, executionRunId: row.executionRunId,
assigneeAgentId: row.assigneeAgentId, assigneeUserId: row.assigneeUserId,
});
}
const edges = edgesByIssueId.get(row.issueId) ?? [];
if (edges.some(edge => edge.blockerIssueId === row.blockerIssueId)) continue;
if (edgeCount >= BLOCKER_ATTENTION_MAX_NODES * 2) { complete = false; continue; }
edges.push({ issueId: row.issueId, blockerIssueId: row.blockerIssueId });
edgesByIssueId.set(row.issueId, edges);
edgeCount++;
}
if (complete) for (const id of chunk) expanded.add(id);
}
if (nodesById.size > BLOCKER_ATTENTION_MAX_NODES) {
truncated = true;
break;
for (const traversal of active) {
if (traversal.frontier.some(id => !expanded.has(id))) {
for (const id of traversal.frontier) if (queried.has(id)) {
traversal.expanded.add(id);
for (const edge of edgesByIssueId.get(id) ?? []) if (nodesById.has(edge.blockerIssueId)) traversal.nodes.add(edge.blockerIssueId);
}
traversal.truncated = true;
traversal.frontier = [];
deferredRootIds.add(traversal.rootId);
continue;
}
const nextFrontier = new Set<string>();
for (const id of traversal.frontier) {
traversal.expanded.add(id);
for (const edge of edgesByIssueId.get(id) ?? []) {
if (traversal.nodes.has(edge.blockerIssueId) || !nodesById.has(edge.blockerIssueId)) continue;
traversal.nodes.add(edge.blockerIssueId);
nextFrontier.add(edge.blockerIssueId);
}
}
traversal.truncated = traversal.nodes.size > BLOCKER_ATTENTION_MAX_NODES;
traversal.frontier = traversal.truncated ? [] : [...nextFrontier];
}
frontier = [...nextFrontier];
}
if (frontier.length > 0) truncated = true;
for (const traversal of traversals.values()) {
if (traversal.frontier.length > 0) traversal.truncated = true;
}
const nodeIds = [...nodesById.keys()];
const evidenceNodes = new Map([...nodesById, ...observedNodes]);
const nodeIds = [...evidenceNodes.keys()];
const activeIssueIds = new Set<string>();
const agentIds = new Set<string>();
const issueIdByExecutionRunId = new Map<string, string>();
for (const node of nodesById.values()) {
for (const node of evidenceNodes.values()) {
if (node.assigneeAgentId) agentIds.add(node.assigneeAgentId);
if (node.executionRunId)
issueIdByExecutionRunId.set(node.executionRunId, node.id);
@@ -4033,7 +4069,7 @@ async function listIssueBlockerAttentionMap(
}
}
const explicitWaitCandidateIds = [...nodesById.values()]
const explicitWaitCandidateIds = [...evidenceNodes.values()]
.filter((node) => node.status !== "done")
.map((node) => node.id);
const explicitWaitingIssueIds = new Set<string>();
@@ -4108,23 +4144,26 @@ async function listIssueBlockerAttentionMap(
ownerType: string;
ownerAgentId: string | null;
ownerUserId: string | null;
}> = await dbOrTx
.select({
id: issueRecoveryActions.id,
sourceIssueId: issueRecoveryActions.sourceIssueId,
status: issueRecoveryActions.status,
ownerType: issueRecoveryActions.ownerType,
ownerAgentId: issueRecoveryActions.ownerAgentId,
ownerUserId: issueRecoveryActions.ownerUserId,
})
.from(issueRecoveryActions)
.where(
and(
eq(issueRecoveryActions.companyId, companyId),
inArray(issueRecoveryActions.status, ["active", "escalated"]),
inArray(issueRecoveryActions.sourceIssueId, explicitWaitCandidateIds),
),
);
}> = [];
for (const chunk of chunkList(explicitWaitCandidateIds, ISSUE_LIST_RELATED_QUERY_CHUNK_SIZE)) {
recoveryActionRows.push(...await dbOrTx
.select({
id: issueRecoveryActions.id,
sourceIssueId: issueRecoveryActions.sourceIssueId,
status: issueRecoveryActions.status,
ownerType: issueRecoveryActions.ownerType,
ownerAgentId: issueRecoveryActions.ownerAgentId,
ownerUserId: issueRecoveryActions.ownerUserId,
})
.from(issueRecoveryActions)
.where(
and(
eq(issueRecoveryActions.companyId, companyId),
inArray(issueRecoveryActions.status, ["active", "escalated"]),
inArray(issueRecoveryActions.sourceIssueId, chunk),
),
));
}
const recoveryActionIds = recoveryActionRows.map((row) => row.id);
const liveRecoveryActionIds = new Set<string>();
for (const chunk of chunkList(
@@ -4187,22 +4226,11 @@ async function listIssueBlockerAttentionMap(
}
}
const agentRows: IssueBlockerAttentionAgentRow[] =
agentIds.size > 0
? await dbOrTx
.select({
id: agents.id,
companyId: agents.companyId,
status: agents.status,
})
.from(agents)
.where(
and(
eq(agents.companyId, companyId),
inArray(agents.id, [...agentIds]),
),
)
: [];
const agentRows: IssueBlockerAttentionAgentRow[] = [];
for (const chunk of chunkList([...agentIds], ISSUE_LIST_RELATED_QUERY_CHUNK_SIZE)) {
agentRows.push(...await dbOrTx.select({ id: agents.id, companyId: agents.companyId, status: agents.status })
.from(agents).where(and(eq(agents.companyId, companyId), inArray(agents.id, chunk))));
}
const agentsById = new Map(agentRows.map((agent) => [agent.id, agent]));
type PathClassification = {
@@ -4215,6 +4243,7 @@ async function listIssueBlockerAttentionMap(
const classifyPath = (
nodeId: string,
seen: Set<string>,
truncated: boolean,
): PathClassification => {
const sample = blockerSampleIdentifier(nodesById.get(nodeId));
if (truncated || seen.has(nodeId)) {
@@ -4319,7 +4348,7 @@ async function listIssueBlockerAttentionMap(
const nextSeen = new Set(seen);
nextSeen.add(nodeId);
const classified = downstream.map((edge) =>
classifyPath(edge.blockerIssueId, nextSeen),
classifyPath(edge.blockerIssueId, nextSeen, truncated),
);
const stalledChild = classified.find(
(result) => result.stalled || result.sampleStalledBlockerIdentifier,
@@ -4385,18 +4414,22 @@ async function listIssueBlockerAttentionMap(
};
};
const pathHasLiveWork = (nodeId: string, seen: Set<string>): boolean => {
if (seen.has(nodeId)) return false;
const nodeHasLiveWork = (node: IssueBlockerAttentionNode) =>
node.status === "in_progress" || activeIssueIds.has(node.id) ||
explicitWaitingIssueIds.has(node.id) || Boolean(node.assigneeUserId);
const observedChildHasLiveWork = (parentId: string) =>
[...observedChildrenByParent.get(parentId) ?? []].some(id => {
const node = observedNodes.get(id);
return node?.companyId === companyId && nodeHasLiveWork(node);
});
const pathHasLiveWork = (nodeId: string, seen: Set<string>, traversal: Traversal): boolean => {
if (seen.has(nodeId) || !traversal.nodes.has(nodeId)) return false;
const node = nodesById.get(nodeId);
if (!node || node.companyId !== companyId) return false;
if (
node.status === "in_progress" ||
activeIssueIds.has(node.id) ||
explicitWaitingIssueIds.has(node.id) ||
Boolean(node.assigneeUserId)
)
return true;
if (nodeHasLiveWork(node)) return true;
if (!traversal.expanded.has(nodeId)) return false;
if (observedChildHasLiveWork(nodeId)) return true;
const nextSeen = new Set(seen);
nextSeen.add(nodeId);
return (edgesByIssueId.get(node.id) ?? []).some((edge) => {
@@ -4406,7 +4439,7 @@ async function listIssueBlockerAttentionMap(
!pendingFinalizeBlockerIssueIds.has(edge.blockerIssueId)
)
return false;
return pathHasLiveWork(edge.blockerIssueId, nextSeen);
return pathHasLiveWork(edge.blockerIssueId, nextSeen, traversal);
});
};
@@ -4419,6 +4452,7 @@ async function listIssueBlockerAttentionMap(
};
for (const root of roots) {
const traversal = traversals.get(root.id)!;
const topLevelEdges = (edgesByIssueId.get(root.id) ?? []).filter((edge) => {
const blocker = nodesById.get(edge.blockerIssueId);
return (
@@ -4440,7 +4474,7 @@ async function listIssueBlockerAttentionMap(
const classified = topLevelEdges.map((edge) => ({
edge,
result: classifyPath(edge.blockerIssueId, new Set([root.id])),
result: classifyPath(edge.blockerIssueId, new Set([root.id]), traversal.truncated),
}));
const coveredBlockerCount = classified.filter(
(entry) => entry.result.covered,
@@ -4522,8 +4556,8 @@ async function listIssueBlockerAttentionMap(
stalledEntry?.result.sampleStalledBlockerIdentifier ??
sampleStalledFromChain ??
null,
blockingTreeLive: topLevelEdges.some((edge) =>
pathHasLiveWork(edge.blockerIssueId, new Set([root.id])),
blockingTreeLive: observedChildHasLiveWork(root.id) || topLevelEdges.some((edge) =>
pathHasLiveWork(edge.blockerIssueId, new Set([root.id]), traversal),
),
directBlockerIssueId: sampleEntry?.edge.blockerIssueId ?? null,
terminalBlockerIssueId,
@@ -4538,7 +4572,7 @@ async function listIssueBlockerAttentionMap(
);
}
return attentionMap;
return result();
}
type IssueReviewAttentionInput = Pick<IssueRow, "id" | "companyId" | "status">;
@@ -7598,7 +7632,7 @@ export function issueService(db: Db) {
companyId: string,
blockedByIssueIds: string[],
actor: { agentId?: string | null; userId?: string | null } = {},
dbOrTx: any = db,
dbOrTx: any,
) {
const deduped = [...new Set(blockedByIssueIds)];
if (deduped.some((candidate) => candidate === issueId)) {
@@ -7627,6 +7661,9 @@ export function issueService(db: Db) {
await assertNoBlockingCycles(companyId, issueId, deduped, dbOrTx);
}
await notifyIssueAttentionRetentionWork(dbOrTx, { companyId, issueId });
await notifyIssueRecovery(dbOrTx, { companyId });
await notifyDeferredExecutionEligibility(dbOrTx, { companyId });
await dbOrTx
.delete(issueRelations)
.where(
@@ -7730,6 +7767,7 @@ export function issueService(db: Db) {
}
const now = new Date();
await notifyStatusCardIssueChange(tx, lockedIssue.companyId);
const adopted = await tx
.update(issues)
.set({
@@ -7783,13 +7821,14 @@ export function issueService(db: Db) {
sql`select ${heartbeatRuns.id} from ${heartbeatRuns} where ${heartbeatRuns.id} = ${input.actorRunId} for update`,
);
const actorRun = await tx
.select({ status: heartbeatRuns.status })
.select({ status: heartbeatRuns.status, companyId: heartbeatRuns.companyId })
.from(heartbeatRuns)
.where(eq(heartbeatRuns.id, input.actorRunId))
.then((rows) => rows[0] ?? null);
if (!actorRun || TERMINAL_HEARTBEAT_RUN_STATUSES.has(actorRun.status))
return null;
await notifyStatusCardIssueChange(tx, actorRun.companyId);
const now = new Date();
const adopted = await tx
.update(issues)
@@ -9372,6 +9411,9 @@ export function issueService(db: Db) {
return listIssueDependencyReadinessMap(dbOrTx, companyId, issueIds);
},
listBlockerAttentionBatch: async (companyId: string, issueRows: IssueBlockerAttentionInputNode[], dbOrTx: any = db) =>
readIssueBlockerAttentionBatch(dbOrTx, companyId, issueRows),
listBlockerAttention: async (
companyId: string,
issueRows: IssueBlockerAttentionInputNode[],
@@ -9675,27 +9717,32 @@ export function issueService(db: Db) {
});
if (blockParentUntilDone) {
const existingBlockers = await db
.select({ blockerIssueId: issueRelations.issueId })
.from(issueRelations)
.where(
and(
eq(issueRelations.companyId, parent.companyId),
eq(issueRelations.relatedIssueId, parent.id),
eq(issueRelations.type, "blocks"),
),
await db.transaction(async tx => {
await tx.select({ id: issues.id }).from(issues).where(and(
eq(issues.companyId, parent.companyId), eq(issues.id, parent.id))).for("update");
const existingBlockers = await tx
.select({ blockerIssueId: issueRelations.issueId })
.from(issueRelations)
.where(
and(
eq(issueRelations.companyId, parent.companyId),
eq(issueRelations.relatedIssueId, parent.id),
eq(issueRelations.type, "blocks"),
),
);
await syncBlockedByIssueIds(
parent.id,
parent.companyId,
[
...new Set([
...existingBlockers.map((row) => row.blockerIssueId),
child.id,
]),
],
{ agentId: actorAgentId ?? null, userId: actorUserId ?? null },
tx,
);
await syncBlockedByIssueIds(
parent.id,
parent.companyId,
[
...new Set([
...existingBlockers.map((row) => row.blockerIssueId),
child.id,
]),
],
{ agentId: actorAgentId ?? null, userId: actorUserId ?? null },
);
});
[child] = await withIssueRelationSummaries(
parent.companyId,
[child],
@@ -10606,7 +10653,12 @@ export function issueService(db: Db) {
: values.parentId ? `issue:${values.parentId}` : undefined);
}
if (values.monitorNextCheckAt) await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors);
await notifyRunDispatch(tx, { companyId: values.companyId });
await notifyDeferredExecutionEligibility(tx, { companyId: values.companyId });
await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers);
values.id ??= randomUUID();
await notifyIssueAttentionRetentionWork(tx, { companyId, issueId: values.id, parentId: values.parentId, originKind: values.originKind, originId: values.originId });
await notifyStatusCardIssueChange(tx, companyId);
if (values.originKind === "routine_execution") await notifyRoutineExecutionStatus(tx, values.id, companyId);
const [issue] = await tx.insert(issues).values(values).returning();
await attachOwnedDraftImages(tx, issue, issue.description, { userId: issueData.createdByUserId, agentId: issueData.createdByAgentId });
@@ -10892,6 +10944,7 @@ export function issueService(db: Db) {
}
}
await notifyStatusCardIssueChange(tx, companyId);
await insertRowsInChunks(tx, issues, issueRows);
await insertRowsInChunks(tx, issueLabels, labelRows);
});
@@ -10908,6 +10961,7 @@ export function issueService(db: Db) {
const censorUsernameInLogs = (await instanceSettings.getGeneral())
.censorUsernameInLogs;
await db.transaction(async (tx) => {
for (const companyId of new Set(rows.map(row => row.companyId))) await notifyStatusCardIssueChange(tx, companyId);
const commentRows = rows.map((row) => {
const createdAt = row.createdAt ? new Date(row.createdAt) : null;
return {
@@ -11488,13 +11542,19 @@ export function issueService(db: Db) {
(blockedByIssueIds !== undefined || issueData.unblockDescriptor !== undefined)))) {
patch.statusVersion = sql`${issues.statusVersion} + 1` as unknown as number;
}
await notifyStatusCardIssueChange(tx, receiptExisting.companyId);
if ((patch.status !== undefined && patch.status !== receiptExisting.status)
|| patch.completedAt !== undefined || patch.cancelledAt !== undefined
|| issueData.parentId !== undefined || issueData.executionWorkspaceId !== undefined) {
await notifyTerminalWorkspaceWork(tx, `issue:${id}`);
}
await notifyIssueAttentionRetentionWork(tx, { companyId: receiptExisting.companyId, issueId: id, parentId: patch.parentId, originKind: patch.originKind, originId: patch.originId });
if (patch.status !== undefined || patch.assigneeAgentId !== undefined || patch.assigneeUserId !== undefined
|| patch.hiddenAt !== undefined || patch.conversationAgentId !== undefined) {
|| patch.hiddenAt !== undefined || patch.conversationAgentId !== undefined || patch.parentId !== undefined
|| patch.executionState !== undefined || patch.executionRunId !== undefined || patch.checkoutRunId !== undefined
|| blockedByIssueIds !== undefined) {
await notifyRunDispatch(tx, { companyId: receiptExisting.companyId });
await notifyDeferredExecutionEligibility(tx, { companyId: receiptExisting.companyId });
await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers);
await notifyRoutineExecutionStatus(tx, id);
}
@@ -11925,7 +11985,9 @@ export function issueService(db: Db) {
);
if (settings?.environmentId !== environmentId) continue;
await db
await db.transaction(async tx => {
await notifyStatusCardIssueChange(tx, companyId);
await tx
.update(issues)
.set({
executionWorkspaceSettings: {
@@ -11935,6 +11997,7 @@ export function issueService(db: Db) {
updatedAt: new Date(),
})
.where(eq(issues.id, row.id));
});
cleared += 1;
}
@@ -11958,11 +12021,15 @@ export function issueService(db: Db) {
await requestRestorePointDeletion(tx, { sourceIssueId: id });
const [owner] = await tx.select({ companyId: issues.companyId }).from(issues).where(eq(issues.id, id));
if (owner) {
await notifyIssueAttentionRetentionWork(tx, { companyId: owner.companyId, issueId: id });
await notifyExternalConversationSettlement(tx, { companyId: owner.companyId });
await notifyApprovalWork(tx, owner.companyId);
await notifyStatusCardIssueChange(tx, owner.companyId);
}
await notifyDecisionTargetIssueRemoval(tx, id);
await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors);
await notifyRunDispatch(tx, owner ? { companyId: owner.companyId } : undefined);
if (owner) await notifyDeferredExecutionEligibility(tx, owner);
await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers);
await notifyRoutineExecutionStatus(tx, id);
removedIssue = await tx
@@ -12091,7 +12158,11 @@ export function issueService(db: Db) {
)
: isNull(issues.executionRunId);
const updated = await db.transaction(async tx => {
await notifyIssueAttentionRetentionWork(tx, { companyId: issueCompany.companyId, issueId: id });
await notifyStatusCardIssueChange(tx, issueCompany.companyId);
await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors);
await notifyRunDispatch(tx, { companyId: issueCompany.companyId });
await notifyDeferredExecutionEligibility(tx, { companyId: issueCompany.companyId });
await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers);
await notifyRoutineExecutionStatus(tx, id);
return tx
@@ -12144,7 +12215,9 @@ export function issueService(db: Db) {
current.executionRunId === checkoutRunId) &&
checkoutRunId
) {
const adopted = await db
const adopted = await db.transaction(async tx => {
await notifyStatusCardIssueChange(tx, issueCompany.companyId);
return tx
.update(issues)
.set({
checkoutRunId,
@@ -12165,6 +12238,7 @@ export function issueService(db: Db) {
)
.returning()
.then((rows) => rows[0] ?? null);
});
if (adopted) return adopted;
}
@@ -12221,7 +12295,11 @@ export function issueService(db: Db) {
}
const staleExecutionRunId = current.executionRunId;
const adopted = await db.transaction(async tx => {
await notifyIssueAttentionRetentionWork(tx, { companyId: issueCompany.companyId, issueId: id });
await notifyStatusCardIssueChange(tx, issueCompany.companyId);
await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors);
await notifyRunDispatch(tx, { companyId: issueCompany.companyId });
await notifyDeferredExecutionEligibility(tx, { companyId: issueCompany.companyId });
await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers);
await notifyRoutineExecutionStatus(tx, id);
return tx
@@ -12477,7 +12555,10 @@ export function issueService(db: Db) {
existing.status === "in_progress" ? "todo" : existing.status;
await notifyTerminalWorkspaceWork(tx, `issue:${id}`);
await notifyNativeStatusReassessment(tx, { companyId: existing.companyId, issueId: id });
await notifyIssueAttentionRetentionWork(tx, { companyId: existing.companyId, issueId: id });
await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors);
await notifyRunDispatch(tx, { companyId: existing.companyId });
await notifyDeferredExecutionEligibility(tx, { companyId: existing.companyId });
await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers);
await notifyRoutineExecutionStatus(tx, id);
const updated = await tx
@@ -12530,6 +12611,8 @@ export function issueService(db: Db) {
patch.assigneeAgentId = null;
}
await notifyIssueAttentionRetentionWork(tx, { companyId: existing.companyId, issueId: id });
await notifyStatusCardIssueChange(tx, existing.companyId);
await notifyTerminalWorkspaceWork(tx, `issue:${id}`);
if (existing.checkoutRunId || existing.executionRunId || options.clearAssignee) {
await notifyExternalConversationSettlement(tx, { companyId: existing.companyId, issueId: id });
@@ -12539,6 +12622,8 @@ export function issueService(db: Db) {
await notifyUnsafeWorkspaceExportRecovery(tx, { companyId: existing.companyId, issueId: id });
}
await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors);
await notifyRunDispatch(tx, { companyId: existing.companyId });
await notifyDeferredExecutionEligibility(tx, { companyId: existing.companyId });
await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers);
await notifyRoutineExecutionStatus(tx, id);
const updated = await tx
@@ -12588,12 +12673,12 @@ export function issueService(db: Db) {
return created;
},
deleteLabel: async (id: string) =>
db
.delete(labels)
.where(eq(labels.id, id))
.returning()
.then((rows) => rows[0] ?? null),
deleteLabel: async (id: string) => db.transaction(async tx => {
const [label] = await tx.select({ companyId: labels.companyId }).from(labels).where(eq(labels.id, id));
if (!label) return null;
await notifyStatusCardIssueChange(tx, label.companyId);
return tx.delete(labels).where(eq(labels.id, id)).returning().then(rows => rows[0] ?? null);
}),
listComments: async (
issueId: string,
@@ -12737,6 +12822,7 @@ export function issueService(db: Db) {
.from(issueComments).where(eq(issueComments.id, commentId));
if (!source) return null;
await notifyExternalConversationSettlement(tx, source);
await notifyStatusCardIssueChange(tx, source.companyId);
const [comment] = await tx
.delete(issueComments)
.where(eq(issueComments.id, commentId))
@@ -12776,6 +12862,7 @@ export function issueService(db: Db) {
.for("update");
if (!source) return null;
await notifyExternalConversationSettlement(tx, source);
await notifyStatusCardIssueChange(tx, source.companyId);
await notifyChatInboundWakeNotices(tx, source);
const now = new Date();
const [comment] = await tx
@@ -12843,7 +12930,7 @@ export function issueService(db: Db) {
},
dbOrTx: any = db,
): Promise<IssueComment> {
if (dbOrTx === db && (actor.runId || actor.userId)) {
if (dbOrTx === db) {
const append = () =>
measureRunTransaction(db, "response", async (tx) => {
// Serialize run-authored comments on the issue so a provider retry
@@ -13086,6 +13173,7 @@ export function issueService(db: Db) {
currentUserRedactionOptions.enabled,
);
}
await notifyStatusCardIssueChange(dbOrTx, issue.companyId);
comment =
shouldUpgradeExternalAuthorization ||
shouldUpgradeAttachmentAuthorization
@@ -13800,6 +13888,7 @@ export function issueService(db: Db) {
.then((rows) => rows[0] ?? null);
if (!existing) return null;
await notifyIssueRecovery(tx, { companyId: existing.companyId });
await tx.delete(issueAttachments).where(eq(issueAttachments.id, id));
await tx.delete(assets).where(eq(assets.id, existing.assetId));
return existing;
@@ -77,14 +77,18 @@ async function renewOnLivenessPool(
});
}
export async function hasLiveLegacyController(db: Db, run: Run): Promise<boolean> {
if (run.runtimeMode === "native" || !run.controllerBootId) return false;
const [owner] = await db.select({ id: heartbeatRuns.id }).from(heartbeatRuns).where(and(
export async function liveLegacyControllerExpiresAt(db: Db, run: Run): Promise<Date | null> {
if (run.runtimeMode === "native" || !run.controllerBootId) return null;
const [owner] = await db.select({ expiresAt: heartbeatRuns.controllerLeaseExpiresAt }).from(heartbeatRuns).where(and(
eq(heartbeatRuns.id, run.id), eq(heartbeatRuns.companyId, run.companyId),
eq(heartbeatRuns.status, "running"),
gt(heartbeatRuns.controllerLeaseExpiresAt, sql`clock_timestamp()`),
));
return Boolean(owner);
return owner?.expiresAt ?? null;
}
export async function hasLiveLegacyController(db: Db, run: Run): Promise<boolean> {
return (await liveLegacyControllerExpiresAt(db, run)) !== null;
}
export type RevokedLegacyControllerState = Pick<Run, "executionStage" | "processPid" | "processGroupId">;
@@ -1,3 +1,4 @@
import { notifyAttentionRetentionWork } from "./retention-work-notifications.js";
import { notifyNativeEnvironmentSettlement } from "./native-environment-settlement-notifications.js";
import { notifyExternalAgentSettlement } from "../modules/external-agents/index.js";
import { notifyExternalConversationSettlement } from "./external-conversation-settlement-notifications.js";
@@ -14,6 +15,7 @@ import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notificatio
export type NativeFinalizationWorkScope = { companyId: string; runId?: string; issueId?: string };
export async function notifyNativeFinalizationWork(transaction: object, scope: NativeFinalizationWorkScope) {
await notifyAttentionRetentionWork(transaction, { companyId: scope.companyId });
if (scope.issueId || scope.runId) {
await notifyNativeEnvironmentSettlement(transaction, scope);
await notifyExternalAgentSettlement(transaction, scope);
@@ -73,7 +73,7 @@ describe("native same-conversation chat attachment reuse", () => {
beforeAll(async () => {
const external = resolveExternalTestDatabaseUrl("PAPERCLIP_CHAT_REUSE_TEST_DATABASE_URL");
if (!external) temporary = await startEmbeddedPostgresTestDatabase("native-chat-reuse-");
db = createDb(external ?? temporary!.connectionString);
db = createDb(external ?? temporary!.connectionString, { maxConnections: 6 });
const storageRoot = await mkdtemp(
path.join(tmpdir(), "paperclip-chat-reuse-"),
);
@@ -1,3 +1,4 @@
import { notifyIssueRecovery } from "../issue-recovery-notifications.js";
import { createHash } from "node:crypto";
import { and, desc, eq, inArray, isNull, sql } from "drizzle-orm";
@@ -1582,7 +1583,9 @@ export async function prepareReusedChatAttachment(input: {
if (!workProduct) {
throw new Error("paperclip_runner_chat_attachment_origin_not_persisted");
}
await input.db
await input.db.transaction(async tx => {
await notifyIssueRecovery(tx, { companyId: input.binding.companyId });
await tx
.update(issueWorkProducts)
.set({
title: input.title,
@@ -1602,6 +1605,7 @@ export async function prepareReusedChatAttachment(input: {
eq(issueWorkProducts.createdByRunId, input.binding.runId),
),
);
});
const comment = await issueService(input.db).addComment(
input.binding.issueId,
`Prepared ${input.title} for this response.`,
@@ -1,3 +1,4 @@
import { notifyStatusCardIssueChange } from "../status-card-notifications.js";
import { notifyExternalAgentSettlement } from "../../modules/external-agents/index.js";
import { pendingNativeRunProjectionCondition } from "./native-run-projection.js";
import { nativeStatusReassessmentSourceIsSuperseded, reassessNativeStatusFromSource } from "./native-status-reassessment.js";
@@ -334,6 +335,7 @@ export async function claimNativeSessionResumptions(input: {
nativePhaseUpdatedAt: now,
updatedAt: now,
}).where(eq(heartbeatRuns.id, row.run.id));
await notifyStatusCardIssueChange(tx, run.companyId);
await tx.update(issues).set({
executionRunId: run.id,
executionAgentNameKey: agent.name.trim().toLowerCase() || null,
@@ -1,3 +1,5 @@
import { notifyAttentionRetentionWork } from "../retention-work-notifications.js";
import { notifyRunDispatch } from "../run-dispatch-notifications.js";
import { randomUUID } from "node:crypto";
import { and, eq } from "drizzle-orm";
import { agentWakeupRequests, heartbeatRuns, type Db } from "@paperclipai/db";
@@ -49,6 +51,8 @@ export async function materializeNativeProviderCapacityRetry(input: {
scheduledRetryAt: dueAt.toISOString(),
executionRetryAccounting: accountingForScheduledRetry(run, NATIVE_PROVIDER_OVERLOADED_CODE, attempt),
};
await notifyAttentionRetentionWork(input.tx, { companyId: input.companyId });
await notifyRunDispatch(input.tx, { companyId: run.companyId });
const [wake] = await input.tx.insert(agentWakeupRequests).values({
companyId: run.companyId, agentId: run.agentId, source: "automation", triggerDetail: "system",
reason: NATIVE_PROVIDER_OVERLOADED_CODE, status: "queued", runId: retryId,
@@ -1,3 +1,5 @@
import { notifyAttentionRetentionWork } from "../retention-work-notifications.js";
import { notifyStatusCardIssueChange } from "../status-card-notifications.js";
import {
agents,
heartbeatRuns,
@@ -240,6 +242,8 @@ export async function claimNativeReviewExecutionLock(db: Db, input: {
if (!run || !["queued", "running"].includes(run.status) || !admitted || !expected
|| admitted.nativeReviewInteractionId !== expected.nativeReviewInteractionId
|| admitted.nativeReviewDecisionId !== expected.nativeReviewDecisionId) return false;
await notifyAttentionRetentionWork(tx, { companyId: input.companyId });
await notifyStatusCardIssueChange(tx, input.companyId);
const claimed = await tx.update(issues).set({
executionRunId: input.runId,
executionAgentNameKey: input.agentNameKey,
@@ -1,3 +1,4 @@
import { resolveExternalTestDatabaseUrl } from "../../__tests__/helpers/external-test-database.js";
import type { PrpStructuredRunResult } from "../../vendor/paperclip-runner/index.js";
import { nativeCompletionFeedback } from "./native-completion-feedback.js";
import { createHash, randomUUID } from "node:crypto";
@@ -61,10 +62,9 @@ describe("native runner file handoff", () => {
const runId = "00000000-0000-4000-8000-000000009104";
beforeAll(async () => {
temporary = await startEmbeddedPostgresTestDatabase(
"native-runner-file-handoff-",
);
db = createDb(temporary.connectionString);
const external = resolveExternalTestDatabaseUrl("PAPERCLIP_NATIVE_HANDOFF_TEST_DATABASE_URL");
if (!external) temporary = await startEmbeddedPostgresTestDatabase("native-runner-file-handoff-");
db = createDb(external ?? temporary!.connectionString, { maxConnections: 6 });
temporaryRoot = await mkdtemp(
path.join(tmpdir(), "paperclip-native-file-handoff-"),
);
@@ -1,3 +1,4 @@
import { notifyIssueRecovery } from "../issue-recovery-notifications.js";
import { createHash, randomUUID } from "node:crypto";
import { execFile } from "node:child_process";
import { constants } from "node:fs";
@@ -1265,7 +1266,9 @@ export async function prepareNativeRunnerFileHandoff(input: {
) {
throw new Error("paperclip_runner_file_handoff_origin_not_persisted");
}
await input.db
await input.db.transaction(async tx => {
await notifyIssueRecovery(tx, { companyId: input.binding.companyId });
await tx
.update(issueWorkProducts)
.set({ title: verified.title, updatedAt: new Date() })
.where(
@@ -1276,6 +1279,7 @@ export async function prepareNativeRunnerFileHandoff(input: {
eq(issueWorkProducts.createdByRunId, input.binding.runId),
),
);
});
const comment = await issueService(input.db).addComment(
input.binding.issueId,
`Prepared ${verified.title} for this response.`,
@@ -4302,6 +4302,17 @@ describe("retained native cleanup activation", () => {
releaseCommit = release;
});
const db = {
execute: async (query: SQL) => {
// Retention hints read the fixture's run/issue relationship inside the
// same cleanup transaction; this fixture has one issue and no ancestors.
const statement = new PgDialect().sqlToQuery(query);
expect(transactionOpen).toBe(true);
expect(statement.params).toContain(run.companyId);
if (statement.sql.includes("from heartbeat_runs")) return [{ id: run.nativeIssueId }];
if (statement.sql.includes("origin_kind")) return [{ id: run.nativeIssueId, originKind: null, originId: null }];
if (statement.sql.includes("with recursive affected")) return [{ issueId: run.nativeIssueId }];
throw new Error(`Unexpected cleanup fixture query: ${statement.sql}`);
},
select: () => ({
from: (table: unknown) => {
const rows =
@@ -1,3 +1,4 @@
import { notifyStatusCardIssueChange } from "../status-card-notifications.js";
import { notifyNativeFinalizationWork } from "../native-finalization-notifications.js";
import { alias } from "drizzle-orm/pg-core";
import { notifyUnsafeWorkspaceExportRecovery, type UnsafeWorkspaceExportScope } from "../unsafe-workspace-export-notifications.js";
@@ -107,6 +108,7 @@ async function recoverCandidate(db: Db, candidate: { runId: string; companyId: s
nativePhase: "result_accepted", nativePhaseUpdatedAt: now,
resultJson: { ...run.resultJson, finalizationPhase: "result_accepted", failureCode: null, originalFailureCode: null, nextAttemptAt: null }, updatedAt: now,
}).where(eq(heartbeatRuns.id, run.id)).returning();
await notifyStatusCardIssueChange(tx, run.companyId);
await tx.update(issues).set({ executionRunId: run.id, updatedAt: now }).where(eq(issues.id, issue.id));
await appendHeartbeatRunEvent(tx as unknown as Db, { companyId: run.companyId, agentId: run.agentId, runId: run.id,
eventType: "workspace_export_omitted", stream: "system", level: "info",
@@ -1,3 +1,4 @@
import { notifyStatusCardIssueChange } from "../status-card-notifications.js";
import { notifyNativeFinalizationWork } from "../native-finalization-notifications.js";
import { notifyEnvironmentCleanup } from "../environment-cleanup-notifications.js";
import { randomUUID } from "node:crypto";
@@ -141,6 +142,7 @@ export async function retryNativeWorkspaceExport(input: {
await tx.update(heartbeatRuns).set({ status: "running", finishedAt: null, error: null, errorCode: null,
nativePhase: "result_accepted", nativePhaseUpdatedAt: now,
resultJson: { ...current.run.resultJson, workspaceExportRetry: retry, finalizationPhase: "result_accepted", failureCode: null, originalFailureCode: null, nextAttemptAt: null }, updatedAt: now }).where(eq(heartbeatRuns.id, current.run.id));
await notifyStatusCardIssueChange(tx, current.run.companyId);
await tx.update(issues).set({ executionRunId: current.run.id, updatedAt: now }).where(eq(issues.id, current.issue.id));
// Reclaim the same lease only for control-plane copyback. The normal
// finalization settlement will stop/retain it again on success or failure.
@@ -131,7 +131,7 @@ describe("native workspace finalization recovery", () => {
const external = resolveExternalTestDatabaseUrl("PAPERCLIP_NATIVE_WORKSPACE_FINALIZER_TEST_DATABASE_URL");
if (!external) temporary = await startEmbeddedPostgresTestDatabase("paperclip-native-workspace-finalizer-");
connectionString = external ?? temporary!.connectionString;
db = createDb(connectionString);
db = createDb(connectionString, { maxConnections: 6 });
await db.insert(companies).values([
{ id: companyId, name: "Native workspace owner", issuePrefix: "NWO" },
@@ -452,7 +452,7 @@ describe("native workspace finalization recovery", () => {
const seeded = await seedRun({ executionWorkspaceId: randomUUID(), title: "Live copyback" });
const live = await withNativeWorkspaceFinalizationOwnership({ db, companyId, runId: seeded.runId }, async (ownership) => {
await ownership.assertHeld();
expect(await resumeNativeWorkspaceFinalization({ db: createDb(connectionString), runId: seeded.runId })).toBeNull();
expect(await resumeNativeWorkspaceFinalization({ db: createDb(connectionString, { maxConnections: 6 }), runId: seeded.runId })).toBeNull();
expect(await db.select().from(workspaceOperations).where(eq(workspaceOperations.heartbeatRunId, seeded.runId))).toEqual([]);
return "copied";
});
@@ -476,7 +476,7 @@ describe("native workspace finalization recovery", () => {
try {
await entered;
await db.execute(sql`select pg_terminate_backend(pid) from pg_locks where locktype = 'advisory' and database = (select oid from pg_database where datname = current_database()) and pid <> pg_backend_pid()`);
const second = await withNativeWorkspaceFinalizationOwnership({ db: createDb(connectionString), companyId, runId: seeded.runId }, async () => "must not run");
const second = await withNativeWorkspaceFinalizationOwnership({ db: createDb(connectionString, { maxConnections: 6 }), companyId, runId: seeded.runId }, async () => "must not run");
expect(second).toEqual({ acquired: false, reason: "busy" });
const [held] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, seeded.runId));
expect(held.runnerProfileJson?.nativeWorkspaceFinalizationOwner).toBeTruthy();
@@ -489,7 +489,7 @@ describe("native workspace finalization recovery", () => {
it("recovers its own joined copyback receipt after the application pool disconnects during cleanup", async () => {
const seeded = await seedRun({ executionWorkspaceId: randomUUID(), title: "Reconnect joined copyback" });
const interruptedDb = createDb(connectionString);
const interruptedDb = createDb(connectionString, { maxConnections: 6 });
let joined = false;
await expect(withNativeWorkspaceFinalizationOwnership({ db: interruptedDb, companyId, runId: seeded.runId }, async () => {
await fs.writeFile(path.join(workspaceRoot, "joined-copyback.txt"), "durable work");
@@ -507,7 +507,7 @@ describe("native workspace finalization recovery", () => {
.toEqual({ acquired: false, reason: "owner_unverified" });
}
await db.update(heartbeatRuns).set({ runnerProfileJson: beforeReconnect.runnerProfileJson }).where(eq(heartbeatRuns.id, seeded.runId));
expect(await withNativeWorkspaceFinalizationOwnership({ db: createDb(connectionString), companyId, runId: seeded.runId }, async () => "resumed"))
expect(await withNativeWorkspaceFinalizationOwnership({ db: createDb(connectionString, { maxConnections: 6 }), companyId, runId: seeded.runId }, async () => "resumed"))
.toEqual({ acquired: true, value: "resumed" });
});
@@ -1,4 +1,6 @@
import { notifyRoutineExecutionStatus } from "../routine-execution-status-notifications.js";
import { notifyAttentionRetentionWork } from "../retention-work-notifications.js";
import { notifyStatusCardIssueChange } from "../status-card-notifications.js";
import { DELIVERY_QUEUES, notifyDeliveryWork } from "../delivery-work-notifications.js";
import { notifyUnsafeWorkspaceExportRecovery } from "../unsafe-workspace-export-notifications.js";
import { and, eq, gt, inArray, ne, notInArray, or, sql } from "drizzle-orm";
@@ -180,6 +182,7 @@ export async function reconstructNativeWorkspace(input: {
...(source.interveningExecutions.length ? { interveningExecutions: source.interveningExecutions } : {}) } };
await markExecutionReconciliation(db, source.action, decision, input.actorId);
const now = new Date();
await notifyAttentionRetentionWork(tx, { companyId: input.companyId });
await notifyUnsafeWorkspaceExportRecovery(tx, { companyId: input.companyId, issueId: input.issueId });
await tx.update(issueRecoveryActions).set({ status: "resolved", outcome: "handed_back", resolvedAt: now, updatedAt: now,
resolutionNote: "workspace_reconstruction_authorized", nextAction: "Continue from saved sources. The original unexported edits are lost; verify the selected attachments before rebuilding. Do not repeat completed external actions or reset the original work limits.",
@@ -189,6 +192,7 @@ export async function reconstructNativeWorkspace(input: {
await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors);
await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers);
await notifyRoutineExecutionStatus(tx, input.issueId, input.companyId);
await notifyStatusCardIssueChange(tx, input.companyId);
await tx.update(issues).set({ status: "todo", unblockDescriptor: null, blockedTransitionAt: null,
blockedOwnerNotifiedAt: null, updatedAt: now }).where(eq(issues.id, issue.id));
const publication = await persistActivity(db, { companyId: input.companyId, actorType: "user", actorId: input.actorId,
@@ -1,4 +1,6 @@
import { notifyStatusCardIssueChange } from "../status-card-notifications.js";
import { nativeRunKeyScope, nativeSkillTestToolAllowed, assertNativeSkillTestTool } from "./native-skill-test-scope.js";
import { notifyAttentionRetentionWork } from "../retention-work-notifications.js";
import { requiredQualityReviewSchema } from "@paperclipai/shared";
import { priorRecoveryApiReceipt } from "../recovery-effect-receipts.js";
import { DELIVERY_QUEUES, notifyDeliveryWork } from "../delivery-work-notifications.js";
@@ -2067,8 +2069,10 @@ export class PaperclipRunnerToolAuthority {
replayMonitor = summarizeIssueMonitor(target, normalizeIssueExecutionPolicy(target.executionPolicy));
return { taskId: target.id, status: target.status, monitor: replayMonitor };
}
await notifyAttentionRetentionWork(tx, { companyId: this.binding.companyId });
const patch = prepareIssueMonitorUpdate(target, input.monitor, actor);
await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors);
await notifyStatusCardIssueChange(tx, this.binding.companyId);
const [updated] = await tx.update(issues).set({ ...patch, updatedAt: new Date() })
.where(and(eq(issues.id, target.id), eq(issues.companyId, this.binding.companyId))).returning();
if (!updated) throw notFound("Task not found");
@@ -497,7 +497,7 @@ else if(m.id!==undefined) send({id:m.id,result:{}});
const [agent] = await server.db.select().from(agents).where(eq(agents.id, fixture.agentId));
const context = structuredClone(run.contextSnapshot ?? {});
await prepareHeartbeatTaskContext(server.db, { run, agent,
task: { issueRef: issue, issueContext: null, wakeCommentContext: null, context, taskSession: null },
task: { issueRef: issue, issueContext: null, context, taskSession: null },
policy: { recovering: true, trustPreset: { kind: "standard" }, experimentalInstanceSettings: {} },
services: { issuesSvc: { getAncestors: async () => [] },
getPinnedSkillTestContext: createHeartbeatRunPreparation(server.db).getPinnedSkillTestContext },
@@ -1,3 +1,4 @@
import { notifyNativeStatusWakeWork } from "../deferred-execution-notifications.js";
import { notifyExternalAgentSettlement } from "../../modules/external-agents/index.js";
import { lockCurrentNativeWorkAssessmentEvidence } from "./work-assessments.js";
import { hasNativeWorkspaceFinalizationOwner } from "./native-workspace-finalization-recovery-state.js";
@@ -478,6 +479,7 @@ async function enqueueWake(input: {
.limit(1)
.then((rows) => rows[0] ?? null);
if (existing) return existing.id;
await notifyNativeStatusWakeWork(input.tx, { companyId: input.companyId });
const inserted = await input.tx
.insert(agentWakeupRequests)
.values({
@@ -1,3 +1,4 @@
import { notifyAttentionRetentionWork } from "../retention-work-notifications.js";
import { and, eq, isNull, ne, sql } from "drizzle-orm";
import { completionContracts, heartbeatRuns, issues, issueThreadInteractions, statusDecisionEffects, statusDecisions, workAssessments, type Db } from "@paperclipai/db";
import { hasPublishedGitHubReview } from "../chat-channels/github/review-wait.js";
@@ -70,6 +71,7 @@ export async function supersedeExternalReviewRequestsInTransaction(
publications: ActivityPublication[],
) {
for (const { interaction } of candidates) {
await notifyAttentionRetentionWork(tx, { companyId: binding.companyId });
const reason = "A later published review no longer requests human input.";
const [retired] = await tx.update(issueThreadInteractions).set({ status: "cancelled",
result: { version: 1, outcome: "withdrawn", reason }, resolvedAt: new Date(), updatedAt: new Date(),
@@ -1,8 +1,14 @@
import { notifyIssueAttentionRetentionWork } from "./retention-work-notifications.js";
import { notifyStatusCardIssueChange } from "./status-card-notifications.js";
import { notifyIssueRecovery } from "./issue-recovery-notifications.js";
import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js";
export type NativeStatusReassessmentScope = { companyId: string; issueId?: string; runId?: string; agentId?: string };
export function notifyNativeStatusReassessment(tx: object, scope: NativeStatusReassessmentScope) {
export async function notifyNativeStatusReassessment(tx: object, scope: NativeStatusReassessmentScope) {
await notifyIssueAttentionRetentionWork(tx, { companyId: scope.companyId, issueId: scope.issueId, runId: scope.runId });
await notifyStatusCardIssueChange(tx, scope.companyId);
await notifyIssueRecovery(tx, scope ? { companyId: scope.companyId } : undefined);
return notifyDeliveryWork(tx, DELIVERY_QUEUES.nativeStatusReassessment,
JSON.stringify({ companyId: scope.companyId, issueId: scope.issueId, runId: scope.runId, agentId: scope.agentId }));
}
@@ -1,3 +1,4 @@
import { notifyAttentionRetentionWork } from "./retention-work-notifications.js";
import { createAgentLifecycle } from "./agent-lifecycle.js";
import { and, eq, isNull, ne } from "drizzle-orm";
import type { Db } from "@paperclipai/db";
@@ -340,6 +341,7 @@ export function pluginManagedAgentService(
// and its canonical history. Transfer only its proven stable-key marker to
// the replacement installation, atomically with the new scoped bindings.
await upsertBinding(companyId, declaration, agentId, {}, adapterType, tx);
await notifyAttentionRetentionWork(tx, { companyId, sourceKind: "agent_error_alert" });
await tx.update(agents).set({
metadata: managedMetadata(options.pluginId, options.pluginKey, declaration, agent.metadata),
updatedAt: new Date(),
+2
View File
@@ -1,3 +1,4 @@
import { notifyStatusCardIssueChange } from "./status-card-notifications.js";
import { effectiveProjectWorkspacePolicy, isWorkspaceProvider } from "./project-workspace-provider.js";
import { releaseProviderRecoveryTransfers, stageProviderFreshWorkspace } from "./provider-fresh-workspace.js";
import { randomUUID, createHash } from "node:crypto";
@@ -418,6 +419,7 @@ export async function preparePluginWorkspace(
},
})
.returning();
await notifyStatusCardIssueChange(tx, task.companyId);
await tx
.update(issues)
.set({
+3
View File
@@ -1,3 +1,4 @@
import { notifyStatusCardIssueChange } from "./status-card-notifications.js";
import { notifyChatInboundWakeNotices } from "./chat-inbound-wake-notice-notifications.js";
import { listProjectWorkspaceProviders, resolveProjectWorkspaceProvider } from "./project-workspace-provider.js";
import { retireRemovedPluginWorkspaceDependents } from "./plugin-workspaces.js";
@@ -1013,6 +1014,7 @@ export function projectService(db: Db, options: { captureWorkspaceUpdates?: bool
const row = await db.transaction(async (tx) => {
if (updates.visibility !== undefined) {
await notifyStatusCardIssueChange(tx, existingProject.companyId);
await notifyChatInboundWakeNotices(tx, { companyId: existingProject.companyId });
await tx.execute(sql`select pg_advisory_xact_lock(hashtextextended(${`issue-privacy-tree:${existingProject.companyId}`}, 0))`);
}
@@ -1086,6 +1088,7 @@ export function projectService(db: Db, options: { captureWorkspaceUpdates?: bool
await tx.select({ id: projectWorkspaces.id }).from(projectWorkspaces).where(eq(projectWorkspaces.projectId, id)).for("update");
await recordProjectLifecycleEvent(tx as unknown as Db, project.companyId, id, "delete");
await notifyDeliveryWork(tx, DELIVERY_QUEUES.skillSourceSync);
await notifyStatusCardIssueChange(tx, project.companyId);
const [row] = await tx.delete(projects).where(eq(projects.id, id)).returning();
return row ? { ...row, urlKey: deriveProjectUrlKey(row.name, row.id) } : null;
}),
@@ -1,3 +1,4 @@
import { notifyStatusCardIssueChange } from "./status-card-notifications.js";
import { assertTaskWorkspaceExecutionAccess } from "./task-workspace-access.js";
import { isDeepStrictEqual } from "node:util";
import { createHash, randomUUID } from "node:crypto";
@@ -152,6 +153,7 @@ export async function reserveFreshPluginTaskWorkspace(input: {
providerType: "adapter_managed", cwd: `/workspaces/${input.fresh.attemptId}`,
metadata: { runnerTaskFreshWorkspace: input.fresh, runnerTaskWorkspaceOwner: input.owner },
});
await notifyStatusCardIssueChange(tx, input.companyId);
await tx.update(issues).set({ executionWorkspaceId: input.fresh.attemptId, executionWorkspacePreference: "reuse_existing",
workspaceBindingRevision: task.workspaceBindingRevision + 1,
workspacePendingSelection: task.workspacePendingSelection?.expectedBindingRevision === task.workspaceBindingRevision
+53 -19
View File
@@ -1,3 +1,4 @@
import { notifyRunDispatch } from "../run-dispatch-notifications.js";
import { notifyNativeEnvironmentSettlement } from "../native-environment-settlement-notifications.js";
import { notifyExternalAgentSettlement } from "../../modules/external-agents/index.js";
import { notifyNativeStatusReassessment } from "../native-status-reassessment-notifications.js";
@@ -26,7 +27,7 @@ import {
decideLegacyContinuation, legacyDispositionEpisode, legacyDispositionFingerprint,
LEGACY_DISPOSITION_REPAIR_INSTRUCTION, type LegacyDispositionEpisode,
} from "./legacy-continuation.js";
import { CONTROLLER_LEASE_LOST_FAILURE_CODE, hasLiveLegacyController } from "../legacy-controller-lease.js";
import { CONTROLLER_LEASE_LOST_FAILURE_CODE, hasLiveLegacyController, liveLegacyControllerExpiresAt } from "../legacy-controller-lease.js";
import { isCompletedOnboardingHandoffWake } from "../chat-completion-delivery.js";
import { instanceSettingsService } from "../instance-settings.js";
import { isWaitingConversation, settleConversationTurnForRun, deliverConversationComments } from "../agent-conversations.js";
@@ -244,6 +245,7 @@ type ResolvedDependencyWakeBackstopOptions = {
runId?: string | null;
companyId?: string | null;
blockerIssueId?: string | null;
issueId?: string;
source?: ResolvedDependencyWakeBackstopSource;
};
@@ -2201,7 +2203,7 @@ export function recoveryService(
return Boolean(budgetBlock);
}
async function reconcileUnassignedBlockingIssues() {
async function reconcileUnassignedBlockingIssues(opts?: { companyId?: string }) {
const candidates = await db
.select({
id: issues.id,
@@ -2215,6 +2217,8 @@ export function recoveryService(
.where(
and(
eq(issueRelations.type, "blocks"),
eq(issueRelations.companyId, issues.companyId),
opts?.companyId ? eq(issues.companyId, opts.companyId) : undefined,
inArray(issues.status, ["todo", "blocked"]),
isNull(issues.assigneeAgentId),
isNull(issues.assigneeUserId),
@@ -2716,6 +2720,7 @@ export function recoveryService(
const now = new Date();
const retryAt = readProviderQuotaRetryAt(input.latestRun, now);
return database.transaction(async (tx) => {
await notifyRunDispatch(tx, { companyId: input.issue.companyId });
const wakeup = await tx
.insert(agentWakeupRequests)
.values({
@@ -3316,6 +3321,7 @@ export function recoveryService(
created = Boolean(enqueuedRun);
} else {
scheduledRun = await db.transaction(async (tx) => {
await notifyRunDispatch(tx, { companyId: input.issue.companyId });
const wakeup = await tx
.insert(agentWakeupRequests)
.values({
@@ -3497,7 +3503,7 @@ export function recoveryService(
return Boolean(run || wake);
}
async function reconcileActiveRecoveryActions() {
async function reconcileActiveRecoveryActions(opts?: { companyId?: string }) {
const rows = await db
.select({
action: issueRecoveryActions,
@@ -3511,7 +3517,8 @@ export function recoveryService(
eq(issues.companyId, issueRecoveryActions.companyId),
),
)
.where(inArray(issueRecoveryActions.status, ["active", "escalated"]));
.where(and(inArray(issueRecoveryActions.status, ["active", "escalated"]),
opts?.companyId ? eq(issueRecoveryActions.companyId, opts.companyId) : undefined));
const result = {
requeued: 0,
@@ -4523,14 +4530,20 @@ export function recoveryService(
}
async function reconcileStrandedAssignedIssues(opts?: {
companyId?: string;
issueId?: string;
onDeferredDeadline?: (at: number) => void;
issueCreatedAtGte?: Date | null;
skipExternalConversationSettlement?: boolean;
skipRelatedRecovery?: boolean;
}) {
const candidates = await db
.select(recoveryIssueFields)
.from(issues)
.where(
and(
opts?.companyId ? eq(issues.companyId, opts.companyId) : undefined,
opts?.issueId ? eq(issues.id, opts.issueId) : undefined,
isNull(issues.assigneeUserId),
inArray(issues.status, ["todo", "in_progress", "in_review"]),
or(
@@ -5711,6 +5724,7 @@ export function recoveryService(
classification.baseBackoffMs *
Math.pow(2, Math.max(0, consecutive - 1));
if (elapsed < requiredDelay) {
opts?.onDeferredDeadline?.(latestFinishedAt.getTime() + requiredDelay);
result.skipped += 1;
continue;
}
@@ -5770,16 +5784,18 @@ export function recoveryService(
}
}
const orphanBlockerRecovery = await reconcileUnassignedBlockingIssues();
result.orphanBlockersAssigned = orphanBlockerRecovery.assigned;
result.skipped += orphanBlockerRecovery.skipped;
result.issueIds.push(...orphanBlockerRecovery.issueIds);
if (!opts?.skipRelatedRecovery) {
const orphanBlockerRecovery = await reconcileUnassignedBlockingIssues(opts);
result.orphanBlockersAssigned = orphanBlockerRecovery.assigned;
result.skipped += orphanBlockerRecovery.skipped;
result.issueIds.push(...orphanBlockerRecovery.issueIds);
const activeRecovery = await reconcileActiveRecoveryActions();
result.continuationRequeued += activeRecovery.requeued;
result.escalated += activeRecovery.escalated;
result.skipped += activeRecovery.skipped;
result.issueIds.push(...activeRecovery.issueIds);
const activeRecovery = await reconcileActiveRecoveryActions(opts);
result.continuationRequeued += activeRecovery.requeued;
result.escalated += activeRecovery.escalated;
result.skipped += activeRecovery.skipped;
result.issueIds.push(...activeRecovery.issueIds);
}
result.issueIds = [...new Set(result.issueIds)];
return result;
@@ -5812,7 +5828,7 @@ export function recoveryService(
source === "workspace.finalize"
? "workspace_finalize_reconciliation"
: "issue_graph_liveness_reconciliation";
const useCursor = !opts?.blockerIssueId;
const useCursor = !opts?.blockerIssueId && !opts?.issueId;
const queryCandidates = (afterIssueId: string | null) => {
const filters = [
@@ -5822,6 +5838,7 @@ export function recoveryService(
sql`${issues.assigneeAgentId} is not null`,
];
if (opts?.companyId) filters.push(eq(issues.companyId, opts.companyId));
if (opts?.issueId) filters.push(eq(issues.id, opts.issueId));
if (afterIssueId) filters.push(gt(issues.id, afterIssueId));
if (opts?.blockerIssueId) {
@@ -6112,6 +6129,7 @@ export function recoveryService(
// fallback below. Without it, a terminal issue named in the run context
// snapshot would still terminalize the shared run and defeat the guard.
runReferencedByActiveIssue?: boolean;
onDeferredDeadline?: (at: number) => void;
},
): Promise<{ terminalized: boolean; status: string }> {
// Act only on a run in "running" status. A "queued" run has no process yet,
@@ -6127,8 +6145,11 @@ export function recoveryService(
// Another controller may own a sandbox run whose PID has no meaning on
// this host. Its live lease owns both execution and finalization, even if
// the issue is already terminal or this process has no in-memory handle.
if (await hasLiveLegacyController(db, run))
const controllerDeadline = await liveLegacyControllerExpiresAt(db, run);
if (controllerDeadline) {
options?.onDeferredDeadline?.(controllerDeadline.getTime());
return { terminalized: false, status: run.status };
}
const pid = run.processPid ?? null;
const processGroupId = run.processGroupId ?? null;
@@ -6349,7 +6370,7 @@ export function recoveryService(
// still claims to be live but can no longer reach a terminal status on its
// own, so a stuck "running" run can no longer block the sweep. Idempotent and
// safe: clears at most one row's worth of lock columns per candidate.
async function sweepStaleIssueLocks() {
async function sweepStaleIssueLocks(opts?: { companyId?: string; issueId?: string; onDeferredDeadline?: (at: number) => void }) {
const result = {
cleared: 0,
issueIds: [] as string[],
@@ -6366,7 +6387,9 @@ export function recoveryService(
})
.from(issues)
.where(
sql`(${issues.checkoutRunId} is not null or ${issues.executionRunId} is not null)`,
and(sql`(${issues.checkoutRunId} is not null or ${issues.executionRunId} is not null)`,
opts?.companyId ? eq(issues.companyId, opts.companyId) : undefined,
opts?.issueId ? eq(issues.id, opts.issueId) : undefined),
);
const referencedRunIds = [
@@ -6381,7 +6404,8 @@ export function recoveryService(
? await db
.select()
.from(heartbeatRuns)
.where(inArray(heartbeatRuns.id, referencedRunIds))
.where(and(inArray(heartbeatRuns.id, referencedRunIds),
opts?.companyId ? eq(heartbeatRuns.companyId, opts.companyId) : undefined))
: [];
const runStatusById = new Map<string, string>();
for (const row of runRows) runStatusById.set(row.id, row.status);
@@ -6392,8 +6416,15 @@ export function recoveryService(
// the same run id in a stale lock column. The terminal reference alone must
// not terminalize a run that an active issue still owns, so exclude these
// runs from the issue-terminal authority below.
const ownershipReferences = opts?.issueId && referencedRunIds.length > 0
? await db.select({ companyId: issues.companyId, status: issues.status,
checkoutRunId: issues.checkoutRunId, executionRunId: issues.executionRunId }).from(issues).where(and(
opts.companyId ? eq(issues.companyId, opts.companyId) : undefined,
or(inArray(issues.checkoutRunId, referencedRunIds), inArray(issues.executionRunId, referencedRunIds)),
))
: candidates;
const runIdsReferencedByActiveIssue = new Set<string>();
for (const issue of candidates) {
for (const issue of ownershipReferences) {
if (issue.status === "done" || issue.status === "cancelled") continue;
for (const runId of [issue.checkoutRunId, issue.executionRunId]) {
if (runId) runIdsReferencedByActiveIssue.add(runId);
@@ -6434,6 +6465,7 @@ export function recoveryService(
referencingIssueTerminalStatus:
issueTerminalStatusByRunId.get(row.id) ?? null,
runReferencedByActiveIssue: runIdsReferencedByActiveIssue.has(row.id),
onDeferredDeadline: opts?.onDeferredDeadline,
});
runStatusById.set(row.id, outcome.status);
if (outcome.terminalized) result.terminalizedRunIds.push(row.id);
@@ -6537,6 +6569,8 @@ export function recoveryService(
recordWatchdogDecision,
scanSilentActiveRuns,
reconcileStrandedAssignedIssues,
reconcileUnassignedBlockingIssues,
reconcileActiveRecoveryActions,
reconcileLegacyContinuation,
legacyRepairDispatchBlock,
sweepStaleIssueLocks,
+113
View File
@@ -0,0 +1,113 @@
import { registerTaskWatchdogWork } from "../task-watchdog-work.js";
import { and, asc, eq, gt, inArray, isNotNull, isNull, or, sql } from "drizzle-orm";
import { issueRecoveryActions, issueRelations, issues, type Db } from "@paperclipai/db";
import type { createDeliveryWorkCoordinator } from "../delivery-work-coordinator.js";
import { DELIVERY_QUEUES, type DeliveryQueue } from "../delivery-work-notifications.js";
import type { IssueRecoveryScope } from "../issue-recovery-notifications.js";
import { registerScopedDeadlineWork } from "../scoped-deadline-work.js";
type Entry = { id: string; companyId: string; dueAt: number };
type Scope = { companyId: string; issueId: string; onDeferredDeadline: (at: number) => void };
const PAGE_SIZE = 100;
export function registerIssueRecoveryWork(
db: Db,
coordinator: ReturnType<typeof createDeliveryWorkCoordinator>,
owner: {
stranded(scope: Scope): Promise<unknown>;
orphanBlockers(scope: { companyId: string }): Promise<unknown>;
recoveryActions(scope: { companyId: string }): Promise<unknown>;
dependencies(scope: Scope): Promise<{ enqueueFailed: number }>;
locks(scope: Scope): Promise<unknown>;
watchdogs(scope: { companyId: string; watchdogId: string; onDeferredDeadline: (at: number) => void }): Promise<{ incomplete: number }>;
schedulingSuppressed(): Promise<boolean>;
},
canRun: () => boolean,
) {
function register(queue: DeliveryQueue, statuses: string[] | null, check: (scope: Scope) => Promise<unknown>) {
const cursors = new Map<string, string>();
let enabled = false;
return registerScopedDeadlineWork<Entry, IssueRecoveryScope>(coordinator, {
queue,
reconstructionIntervalMs: 30_000,
beforePass: async () => { enabled = !(await owner.schedulingSuppressed()); },
decodeScope: hint => hint === "*" ? undefined : JSON.parse(hint) as IssueRecoveryScope,
matches: (entry, scope) => !scope || (entry.companyId === scope.companyId && (!scope.issueId || entry.id === scope.issueId)),
load: async scope => {
const key = scope ? JSON.stringify([scope.companyId, scope.issueId]) : "*";
if (!enabled) { cursors.delete(key); return []; }
const cursor = cursors.get(key);
const rows = await db.select({ id: issues.id, companyId: issues.companyId }).from(issues).where(and(
scope ? eq(issues.companyId, scope.companyId) : undefined,
scope?.issueId ? eq(issues.id, scope.issueId) : undefined,
cursor ? gt(issues.id, cursor) : undefined,
statuses ? inArray(issues.status, statuses) : or(isNotNull(issues.checkoutRunId), isNotNull(issues.executionRunId)),
statuses ? isNull(issues.hiddenAt) : undefined,
statuses ? isNull(issues.assigneeUserId) : undefined,
statuses ? or(isNotNull(issues.assigneeAgentId), eq(issues.status, "in_review")) : undefined,
)).orderBy(asc(issues.id)).limit(PAGE_SIZE);
const complete = rows.length < PAGE_SIZE;
if (complete) cursors.delete(key); else cursors.set(key, rows.at(-1)!.id);
return { rows: rows.map(row => ({ ...row, dueAt: Date.now() })), complete };
},
check: async entry => {
if (!enabled) return null;
let next: number | null = null;
await check({ companyId: entry.companyId, issueId: entry.id,
onDeferredDeadline: at => { next = next === null ? at : Math.min(next, at); } });
return next === null ? null : { ...entry, dueAt: next };
},
}, canRun);
}
function registerCompany(queue: DeliveryQueue, kind: "blockers" | "actions", check: (scope: { companyId: string }) => Promise<unknown>) {
const cursors = new Map<string, string>();
let enabled = false;
return registerScopedDeadlineWork<Entry, IssueRecoveryScope>(coordinator, {
queue,
reconstructionIntervalMs: 30_000,
beforePass: async () => { enabled = !(await owner.schedulingSuppressed()); },
decodeScope: hint => hint === "*" ? undefined : JSON.parse(hint) as IssueRecoveryScope,
matches: (entry, scope) => !scope || entry.companyId === scope.companyId,
load: async scope => {
const key = scope?.companyId ?? "*";
if (!enabled) { cursors.delete(key); return []; }
const cursor = cursors.get(key);
const rows = kind === "actions"
? await db.selectDistinct({ id: issueRecoveryActions.companyId, companyId: issueRecoveryActions.companyId })
.from(issueRecoveryActions).where(and(inArray(issueRecoveryActions.status, ["active", "escalated"]),
scope ? eq(issueRecoveryActions.companyId, scope.companyId) : undefined,
cursor ? gt(issueRecoveryActions.companyId, cursor) : undefined))
.orderBy(asc(issueRecoveryActions.companyId)).limit(PAGE_SIZE)
: await db.selectDistinct({ id: issues.companyId, companyId: issues.companyId }).from(issues).where(and(
inArray(issues.status, ["todo", "blocked"]), isNull(issues.assigneeAgentId), isNull(issues.assigneeUserId),
isNotNull(issues.createdByAgentId),
scope ? eq(issues.companyId, scope.companyId) : undefined,
cursor ? gt(issues.companyId, cursor) : undefined,
sql`exists (select 1 from ${issueRelations} r inner join ${issues} dependent
on dependent.id = r.related_issue_id and dependent.company_id = r.company_id
where r.company_id = ${issues.companyId} and r.issue_id = ${issues.id}
and r.type = 'blocks' and dependent.status not in ('done', 'cancelled'))`,
)).orderBy(asc(issues.companyId)).limit(PAGE_SIZE);
const complete = rows.length < PAGE_SIZE;
if (complete) cursors.delete(key); else cursors.set(key, rows.at(-1)!.id);
return { rows: rows.map(row => ({ ...row, dueAt: Date.now() })), complete };
},
check: async entry => { if (enabled) await check({ companyId: entry.companyId }); return null; },
}, canRun);
}
const assignments = register(DELIVERY_QUEUES.strandedAssignments, ["todo", "in_progress", "in_review"], owner.stranded);
const orphanBlockers = registerCompany(DELIVERY_QUEUES.orphanBlockers, "blockers", owner.orphanBlockers);
const recoveryActions = registerCompany(DELIVERY_QUEUES.activeRecoveryActions, "actions", owner.recoveryActions);
const dependencies = register(DELIVERY_QUEUES.resolvedDependencies, ["blocked"], async scope => {
const result = await owner.dependencies(scope);
if (result.enqueueFailed > 0) throw new Error("Resolved dependency wake remains undelivered");
});
const locks = register(DELIVERY_QUEUES.staleIssueLocks, null, owner.locks);
const watchdogs = registerTaskWatchdogWork(db, coordinator, {
reconcile: owner.watchdogs, enabled: async () => !(await owner.schedulingSuppressed()),
}, canRun);
const workers = [assignments, orphanBlockers, recoveryActions, dependencies, locks, watchdogs];
return { ready: Promise.all(workers.map(worker => worker.ready)),
wake() { for (const worker of workers) worker.invalidate(); },
async stop() { await Promise.all(workers.map(worker => worker.stop())); } };
}
@@ -1,14 +1,89 @@
import { notifyApprovalWork } from "./approval-work-notifications.js";
import { RECOVERY_ORIGIN_KINDS, parseIssueGraphLivenessIncidentKey } from "./recovery/origins.js";
import { sql } from "drizzle-orm";
import type { Db } from "@paperclipai/db";
import type { AttentionSourceKind } from "@paperclipai/shared";
import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js";
export async function notifyRetentionWork(transaction: object, kind: AttentionSourceKind) {
export const ATTENTION_RETENTION_MAX_ISSUE_IDS = 100;
export type AttentionRetentionScope = { companyId?: string; sourceKind?: AttentionSourceKind;
issueId?: string; issueIds?: string[]; sourceId?: string };
export async function notifyAttentionRetentionWork(transaction: object, scope: AttentionRetentionScope = {}) {
const issueIds = [...new Set([...(scope.issueIds ?? []), ...(scope.issueId ? [scope.issueId] : [])])].sort();
await notifyDeliveryWork(transaction, DELIVERY_QUEUES.attentionRetention, JSON.stringify({
companyId: scope.companyId, sourceKind: scope.sourceKind, sourceId: scope.sourceId,
issueId: issueIds.length === 1 ? issueIds[0] : undefined,
issueIds: issueIds.length > 1 && issueIds.length <= ATTENTION_RETENTION_MAX_ISSUE_IDS ? issueIds : undefined,
}));
}
export async function notifyIssueAttentionRetentionWork(transaction: object,
scope: { companyId: string; issueId?: string; parentId?: string | null; runId?: string; originKind?: string | null; originId?: string | null }) {
const database = transaction as Db;
const seeds = [scope.issueId, scope.parentId].filter((id): id is string => Boolean(id));
if (!seeds.length && scope.runId) {
const rows = await database.execute(sql`select issue_id::text as id from heartbeat_runs
where company_id = ${scope.companyId}::uuid and id = ${scope.runId}::uuid and issue_id is not null
union select native_issue_id::text from heartbeat_runs
where company_id = ${scope.companyId}::uuid and id = ${scope.runId}::uuid and native_issue_id is not null
union select issue.id::text from heartbeat_runs run inner join issues issue
on issue.company_id = run.company_id
and issue.id::text = coalesce(run.context_snapshot->>'issueId', run.context_snapshot->>'taskId')
where run.company_id = ${scope.companyId}::uuid and run.id = ${scope.runId}::uuid`);
for (const row of rows) seeds.push(row.id as string);
}
if (!seeds.length) return notifyAttentionRetentionWork(transaction, { companyId: scope.companyId });
const origins = await database.execute(sql`select id::text as id, origin_kind as "originKind", origin_id as "originId" from issues
where company_id = ${scope.companyId}::uuid
and id in (${sql.join([...new Set(seeds)].map(id => sql`${id}::uuid`), sql`, `)})`);
const current = origins.find(source => source.id === scope.issueId);
origins.push({ originKind: scope.originKind ?? current?.originKind, originId: scope.originId ?? current?.originId });
const references: string[] = [];
for (const source of origins) {
if (source.originKind === RECOVERY_ORIGIN_KINDS.strandedIssueRecovery && typeof source.originId === "string") {
references.push(source.originId);
}
const incident = source.originKind === RECOVERY_ORIGIN_KINDS.issueGraphLivenessEscalation
? parseIssueGraphLivenessIncidentKey(source.originId as string | null) : null;
if (incident?.companyId === scope.companyId) references.push(incident.issueId, incident.leafIssueId);
}
if (references.length) {
const trusted = await database.execute(sql`select id::text as id from issues
where company_id = ${scope.companyId}::uuid
and id::text in (${sql.join([...new Set(references)].map(id => sql`${id}`), sql`, `)})`);
for (const row of trusted) seeds.push(row.id as string);
}
// UNION deduplicates visited UUIDs (including cycles). Each relation edge is
// unique by schema, so 101 neighbours guarantee overflow of a 100-ID scope.
// Bound that adjacency before materialization and stream at most 101 unique
// IDs to the caller; an inner UNION DISTINCT would consume the full fan-in.
const affected = await database.execute(sql`with recursive affected(id) as (
select unnest(array[${sql.join([...new Set(seeds)].map(id => sql`${id}::uuid`), sql`, `)}])
union
select parent.id from affected current cross join lateral (
(select related_issue_id as id from issue_relations where company_id = ${scope.companyId}::uuid
and issue_id = current.id and type = 'blocks' limit ${ATTENTION_RETENTION_MAX_ISSUE_IDS + 1})
union all select parent_id from issues where company_id = ${scope.companyId}::uuid
and id = current.id and parent_id is not null
) parent
) select id::text as "issueId" from affected limit ${ATTENTION_RETENTION_MAX_ISSUE_IDS + 1}`);
// The sentinel means the affected set is incomplete. Reconstruct the whole
// company through the existing paged authority instead of dropping ancestors.
await notifyAttentionRetentionWork(transaction, { companyId: scope.companyId,
issueIds: affected.length <= ATTENTION_RETENTION_MAX_ISSUE_IDS ? affected.map(row => row.issueId as string) : undefined,
});
}
export async function notifyRetentionWork(transaction: object, kind: AttentionSourceKind, scope: { companyId?: string; sourceId?: string } = {}) {
if (kind === "approval") await notifyDeliveryWork(transaction, DELIVERY_QUEUES.approvalRetention);
if (kind === "join_request") await notifyDeliveryWork(transaction, DELIVERY_QUEUES.joinRequestRetention);
if (kind !== "approval" && kind !== "join_request") await notifyAttentionRetentionWork(transaction, { ...scope, sourceKind: kind,
sourceId: kind === "blocker_attention" ? undefined : scope.sourceId });
}
export async function notifyCompanyRetentionWork(transaction: object, companyId: string) {
await notifyAttentionRetentionWork(transaction, { companyId });
await notifyApprovalWork(transaction, companyId);
await notifyRetentionWork(transaction, "join_request");
}
+2
View File
@@ -1,3 +1,4 @@
import { notifyStatusCardIssueChange } from "./status-card-notifications.js";
import { notifyApprovalWork } from "./approval-work-notifications.js";
import { notifyDecisionTargetIssueRemoval } from "./decision-targets.js";
import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js";
@@ -1949,6 +1950,7 @@ export function routineService(
if (createdIssue) {
await notifyApprovalWork(txDb, createdIssue.companyId);
await notifyDecisionTargetIssueRemoval(txDb, createdIssue.id);
await notifyStatusCardIssueChange(txDb, createdIssue.companyId);
await txDb.delete(issues).where(eq(issues.id, createdIssue.id));
}
const failureReason = error instanceof Error ? error.message : String(error);
@@ -0,0 +1,11 @@
import { notifyIssueRecovery } from "./issue-recovery-notifications.js";
import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js";
export type RunDispatchScope = { companyId: string };
export async function notifyRunDispatch(transaction: object, scope?: RunDispatchScope) {
await notifyIssueRecovery(transaction, scope ? { companyId: scope.companyId } : undefined);
const hint = scope ? JSON.stringify({ companyId: scope.companyId }) : "*";
await notifyDeliveryWork(transaction, DELIVERY_QUEUES.scheduledRetryDispatch, hint);
await notifyDeliveryWork(transaction, DELIVERY_QUEUES.queuedRunDispatch, hint);
}
@@ -1,6 +1,7 @@
import { createHmac, generateKeyPairSync, randomUUID, sign } from "node:crypto";
import { describe, expect, it, vi } from "vitest";
import { requestSandboxAbsenceProof, verifySandboxAbsenceReceipt } from "./sandbox-absence-proof.js";
import { inspectCloudSandboxCleanupAbsence, requestSandboxAbsenceProof, verifySandboxAbsenceReceipt,
verifySandboxCleanupAbsenceReceipt } from "./sandbox-absence-proof.js";
const { privateKey, publicKey } = generateKeyPairSync("ed25519");
const binding = { companyId: randomUUID(), issueId: randomUUID(), runId: randomUUID(), leaseId: randomUUID(),
@@ -9,10 +10,10 @@ const env = { PAPERCLIP_CLOUD_STACK_ID: "fixture-stack", PAPERCLIP_CLOUD_API_ORI
PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN: "fleet-token", PAPERCLIP_PRIMARY_HOST: "fixture.paperclip.test", PAPERCLIP_AUTH_SECRET: "fixture-secret",
PAPERCLIP_CLOUD_RUNTIME_IDENTITY_JWKS: JSON.stringify({ keys: [{ ...publicKey.export({ format: "jwk" }), kid: "fixture", alg: "EdDSA", use: "sig" }] }) };
const now = new Date("2099-01-01T00:00:00Z"), seconds = now.getTime() / 1000;
function receipt(overrides: Record<string, unknown> = {}) {
const header = Buffer.from(JSON.stringify({ alg: "EdDSA", kid: "fixture", typ: "paperclip-sandbox-absence+jwt" })).toString("base64url");
const payload = Buffer.from(JSON.stringify({ v: 1, iss: "paperclip-cloud", aud: "paperclip-sandbox-absence/v1", sub: "fixture-stack",
...binding, absent: true, iat: seconds, exp: seconds + 60, ...overrides })).toString("base64url");
function receipt(overrides: Record<string, unknown> = {}, purpose = "sandbox-absence", source: Record<string, unknown> = binding) {
const header = Buffer.from(JSON.stringify({ alg: "EdDSA", kid: "fixture", typ: `paperclip-${purpose}+jwt` })).toString("base64url");
const payload = Buffer.from(JSON.stringify({ v: 1, iss: "paperclip-cloud", aud: `paperclip-${purpose}/v1`, sub: "fixture-stack",
...source, absent: true, iat: seconds, exp: seconds + 60, ...overrides })).toString("base64url");
return `${header}.${payload}.${sign(null, Buffer.from(`${header}.${payload}`), privateKey).toString("base64url")}`;
}
describe("sandbox absence proof", () => {
@@ -45,3 +46,44 @@ describe("sandbox absence proof", () => {
expect(fetchImpl).not.toHaveBeenCalled();
});
});
describe("cleanup-only sandbox absence", () => {
const cleanup = { purpose: "cleanup" as const, companyId: binding.companyId, environmentId: randomUUID(),
leaseId: binding.leaseId, heartbeatRunId: null as string | null, provider: binding.provider, providerLeaseId: binding.providerLeaseId };
const lease = { id: cleanup.leaseId, ...cleanup };
const signed = (overrides: Record<string, unknown> = {}) => receipt(overrides, "sandbox-cleanup-absence", cleanup);
it.each([null, binding.runId])("accepts exact absence with nullable run %s without issuing a run receipt", async heartbeatRunId => {
const source = { ...cleanup, heartbeatRunId };
const fetchImpl = vi.fn().mockResolvedValue(Response.json({ receipt: receipt({}, "sandbox-cleanup-absence", source) }));
expect(await inspectCloudSandboxCleanupAbsence({ ...lease, heartbeatRunId }, { env, now, fetchImpl }))
.toEqual({ providerLeaseId: cleanup.providerLeaseId, state: "destroyed" });
const [url, init] = fetchImpl.mock.calls[0];
expect(url).toBe("https://cloud.example.test/v1/tenant/sandbox-absence");
expect(JSON.parse(init.body)).toEqual(source);
expect(init.headers.authorization).not.toContain(env.PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN);
expect(init.redirect).toBe("error");
});
it("requires every binding, current stack, signature and freshness, and rejects recovery receipts", () => {
expect(verifySandboxCleanupAbsenceReceipt(signed(), cleanup, env, now)).toMatchObject(cleanup);
for (const overrides of [
...Object.keys(cleanup).map(key => ({ [key]: randomUUID() })),
{ sub: "foreign-stack" }, { absent: false }, { aud: "paperclip-sandbox-absence/v1" }, { iss: "other" },
{ iat: seconds + 31 }, { exp: seconds }, { exp: seconds + 61 },
]) expect(verifySandboxCleanupAbsenceReceipt(signed(overrides), cleanup, env, now)).toBeNull();
expect(verifySandboxCleanupAbsenceReceipt(signed().slice(0, -8) + "tampered", cleanup, env, now)).toBeNull();
expect(verifySandboxCleanupAbsenceReceipt(receipt(), cleanup, env, now)).toBeNull();
expect(verifySandboxAbsenceReceipt(signed(), binding, env, now)).toBeNull();
});
it("keeps old Clouds, outages, invalid replies and unconfigured instances inconclusive", async () => {
for (const answer of [{ absent: false }, { absent: true }, { receipt: "forged" }, { receipt: signed({ heartbeatRunId: binding.runId }) }])
expect(await inspectCloudSandboxCleanupAbsence(lease, { env, now, fetchImpl: vi.fn().mockResolvedValue(Response.json(answer)) })).toBeUndefined();
for (const status of [400, 401, 403, 503]) expect(await inspectCloudSandboxCleanupAbsence(lease,
{ env, now, fetchImpl: vi.fn().mockResolvedValue(new Response(null, { status })) })).toBeUndefined();
expect(await inspectCloudSandboxCleanupAbsence(lease, { env, now, fetchImpl: vi.fn().mockRejectedValue(new Error("offline")) })).toBeUndefined();
const fetchImpl = vi.fn();
expect(await inspectCloudSandboxCleanupAbsence(lease, { env: {}, now, fetchImpl })).toBeUndefined();
for (const change of [{ provider: "other" }, { environmentId: null }, { providerLeaseId: "invalid" }])
expect(await inspectCloudSandboxCleanupAbsence({ ...lease, ...change }, { env, now, fetchImpl })).toBeUndefined();
expect(fetchImpl).not.toHaveBeenCalled();
});
});
+56 -12
View File
@@ -7,8 +7,18 @@ const bindingSchema = z.object({
companyId: z.string().uuid(), issueId: z.string().uuid(), runId: z.string().uuid(),
leaseId: z.string().uuid(), ownerToken: z.string().uuid(), provider: z.literal("daytona"),
providerLeaseId: z.string().uuid(),
});
}).strict();
const cleanupBindingSchema = z.object({
purpose: z.literal("cleanup"), companyId: z.string().uuid(), environmentId: z.string().uuid(),
leaseId: z.string().uuid(), heartbeatRunId: z.string().uuid().nullable(),
provider: z.literal("daytona"), providerLeaseId: z.string().uuid(),
}).strict();
export type SandboxAbsenceBinding = z.infer<typeof bindingSchema>;
export type SandboxCleanupAbsenceBinding = z.infer<typeof cleanupBindingSchema>;
export type SandboxCleanupAbsenceProof = SandboxCleanupAbsenceBinding & {
v: 1; iss: "paperclip-cloud"; aud: "paperclip-sandbox-cleanup-absence/v1"; sub: string;
absent: true; iat: number; exp: number;
};
export type SandboxAbsenceProof = SandboxAbsenceBinding & {
v: 1; iss: "paperclip-cloud"; aud: "paperclip-sandbox-absence/v1"; sub: string;
absent: true; iat: number; exp: number;
@@ -16,19 +26,30 @@ export type SandboxAbsenceProof = SandboxAbsenceBinding & {
export function verifySandboxAbsenceReceipt(receipt: string, binding: SandboxAbsenceBinding,
env: NodeJS.ProcessEnv = process.env, now = new Date()): SandboxAbsenceProof | null {
const parsed = bindingSchema.safeParse(binding);
return parsed.success ? verifyReceipt(receipt, parsed.data, "sandbox-absence", env, now) as SandboxAbsenceProof | null : null;
}
export function verifySandboxCleanupAbsenceReceipt(receipt: string, binding: SandboxCleanupAbsenceBinding,
env: NodeJS.ProcessEnv = process.env, now = new Date()): SandboxCleanupAbsenceProof | null {
const parsed = cleanupBindingSchema.safeParse(binding);
return parsed.success ? verifyReceipt(receipt, parsed.data, "sandbox-cleanup-absence", env, now) as SandboxCleanupAbsenceProof | null : null;
}
function verifyReceipt(receipt: string, binding: Record<string, unknown>, purpose: string,
env: NodeJS.ProcessEnv, now: Date): Record<string, unknown> | null {
try {
const parsed = bindingSchema.safeParse(binding);
if (receipt.length > 16_000 || !parsed.success) return null;
if (receipt.length > 16_000) return null;
const parts = receipt.split(".");
if (parts.length !== 3 || parts.some(part => !/^[A-Za-z0-9_-]+$/.test(part))) return null;
const header = JSON.parse(Buffer.from(parts[0], "base64url").toString("utf8"));
if (header.alg !== "EdDSA" || header.typ !== "paperclip-sandbox-absence+jwt" || typeof header.kid !== "string") return null;
if (header.alg !== "EdDSA" || header.typ !== `paperclip-${purpose}+jwt` || typeof header.kid !== "string") return null;
if (!verify(null, Buffer.from(`${parts[0]}.${parts[1]}`, "ascii"), cloudRuntimePublicKeyForKid(env, header.kid), Buffer.from(parts[2], "base64url"))) return null;
const proof = JSON.parse(Buffer.from(parts[1], "base64url").toString("utf8")) as SandboxAbsenceProof;
const proof = JSON.parse(Buffer.from(parts[1], "base64url").toString("utf8"));
const seconds = Math.floor(now.getTime() / 1000);
if (proof.v !== 1 || proof.iss !== "paperclip-cloud" || proof.aud !== "paperclip-sandbox-absence/v1"
if (proof.v !== 1 || proof.iss !== "paperclip-cloud" || proof.aud !== `paperclip-${purpose}/v1`
|| !env.PAPERCLIP_CLOUD_STACK_ID || proof.sub !== env.PAPERCLIP_CLOUD_STACK_ID || proof.absent !== true
|| Object.keys(parsed.data).some(key => proof[key as keyof SandboxAbsenceBinding] !== parsed.data[key as keyof SandboxAbsenceBinding])
|| Object.keys(binding).some(key => proof[key] !== binding[key])
|| !Number.isInteger(proof.iat) || !Number.isInteger(proof.exp) || proof.iat > seconds + 30
|| proof.exp <= seconds || proof.exp <= proof.iat || proof.exp - proof.iat > 60) return null;
return proof;
@@ -38,11 +59,35 @@ export function verifySandboxAbsenceReceipt(receipt: string, binding: SandboxAbs
export async function requestSandboxAbsenceProof(binding: SandboxAbsenceBinding, options: {
env?: NodeJS.ProcessEnv; fetchImpl?: typeof fetch; now?: Date;
} = {}): Promise<{ proof: SandboxAbsenceProof; receipt: string } | null> {
const parsed = bindingSchema.safeParse(binding);
if (!parsed.success) return null;
const receipt = await requestReceipt(parsed.data, options);
if (!receipt) return null;
const proof = verifySandboxAbsenceReceipt(receipt, parsed.data, options.env, options.now);
return proof ? { proof, receipt } : null;
}
/** Cleanup-only observation: never grants continuation authority to a run. */
export async function inspectCloudSandboxCleanupAbsence(lease: {
id: string; companyId: string; environmentId: string | null; heartbeatRunId: string | null;
provider: string | null; providerLeaseId: string | null;
}, options: { env?: NodeJS.ProcessEnv; fetchImpl?: typeof fetch; now?: Date } = {}) {
const parsed = cleanupBindingSchema.safeParse({ purpose: "cleanup", companyId: lease.companyId,
environmentId: lease.environmentId, leaseId: lease.id, heartbeatRunId: lease.heartbeatRunId,
provider: lease.provider, providerLeaseId: lease.providerLeaseId });
if (!parsed.success) return undefined;
const receipt = await requestReceipt(parsed.data, options);
if (!receipt || !verifySandboxCleanupAbsenceReceipt(receipt, parsed.data, options.env, options.now)) return undefined;
return { providerLeaseId: parsed.data.providerLeaseId, state: "destroyed" as const };
}
async function requestReceipt(binding: Record<string, unknown>, options: {
env?: NodeJS.ProcessEnv; fetchImpl?: typeof fetch;
}): Promise<string | null> {
const env = options.env ?? process.env;
const context = getCloudStackContext(env);
const secret = env.PAPERCLIP_AUTH_SECRET ?? env.BETTER_AUTH_SECRET;
const parsed = bindingSchema.safeParse(binding);
if (!context?.stackId || !context.primaryHost || !context.cloudOrigin || !secret?.trim() || !parsed.success) return null;
if (!context?.stackId || !context.primaryHost || !context.cloudOrigin || !secret?.trim()) return null;
try {
const origin = new URL(context.cloudOrigin);
const host = new URL(`https://${context.primaryHost}`);
@@ -51,12 +96,11 @@ export async function requestSandboxAbsenceProof(binding: SandboxAbsenceBinding,
const response = await (options.fetchImpl ?? fetch)(`${origin.origin}/v1/tenant/sandbox-absence`, {
method: "POST", redirect: "error", signal: AbortSignal.timeout(10_000),
headers: { authorization: `Bearer ${capability}`, "x-paperclip-cloud-tenant-host": host.hostname, "content-type": "application/json" },
body: JSON.stringify(parsed.data),
body: JSON.stringify(binding),
});
if (!response.ok) return null;
const payload = await response.json() as { receipt?: unknown };
if (typeof payload.receipt !== "string") return null;
const proof = verifySandboxAbsenceReceipt(payload.receipt, binding, env, options.now);
return proof ? { proof, receipt: payload.receipt } : null;
return payload.receipt;
} catch { return null; }
}
@@ -1,7 +1,9 @@
import { notifyRunDispatch } from "./run-dispatch-notifications.js";
import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js";
export type ScheduledRetryCancellationScope = { companyId: string; runId?: string };
export async function notifyScheduledRetryCancellation(transaction: object, scope?: ScheduledRetryCancellationScope) {
await notifyRunDispatch(transaction, scope);
await notifyDeliveryWork(transaction, DELIVERY_QUEUES.scheduledRetryCancellation, scope ? JSON.stringify(scope) : "*");
}
+18 -2
View File
@@ -12,6 +12,8 @@ export function registerScopedDeadlineWork<Entry extends ScheduledEntry, Scope>(
queue: DeliveryQueue;
batchSize?: number;
minimumPassIntervalMs?: number;
reconstructionIntervalMs?: number;
beforePass?(): Promise<void>;
decodeScope(hint: string): Scope | undefined;
matches(entry: Entry, scope?: Scope): boolean;
load(scope?: Scope, signal?: AbortSignal): Promise<Entry[] | { rows: Entry[] | null; complete: boolean }>;
@@ -25,6 +27,7 @@ export function registerScopedDeadlineWork<Entry extends ScheduledEntry, Scope>(
const entries = new Map<string, Entry>();
const failures = new Map<string, { count: number; retryAt: number }>();
const scopeFailures = new Map<string, { count: number; retryAt: number }>();
const reconstructAfter = new Map<string, number>();
let earliest = 0;
let lastCompany: string | null = null;
function failed(map: typeof failures, id: string) {
@@ -34,13 +37,14 @@ export function registerScopedDeadlineWork<Entry extends ScheduledEntry, Scope>(
function dirtyScopes() {
return [...dirty].filter(hint => hint !== "*" || !scopeFailures.has(hint))
.filter(hint => !scans.has(hint) || scopeFailures.has(hint))
.filter(hint => (reconstructAfter.get(hint) ?? 0) <= Date.now() || scopeFailures.has(hint))
.map(hint => { try { return options.decodeScope(hint); } catch { return undefined; } });
}
function next() {
if (!canRun()) return dirty.size > 0 || entries.size > 0 ? Date.now() + 30_000 : null;
let nextAt: number | null = null;
const take = (at: number) => { nextAt = nextAt === null ? at : Math.min(nextAt, at); };
for (const hint of dirty) take(Math.max(earliest, scopeFailures.get(hint)?.retryAt ?? 0));
for (const hint of dirty) take(Math.max(earliest, reconstructAfter.get(hint) ?? 0, scopeFailures.get(hint)?.retryAt ?? 0));
const unavailable = dirtyScopes();
for (const entry of entries.values()) if (!unavailable.some(scope => options.matches(entry, scope)))
take(Math.max(earliest, entry.dueAt, failures.get(entry.id)?.retryAt ?? 0));
@@ -54,9 +58,18 @@ export function registerScopedDeadlineWork<Entry extends ScheduledEntry, Scope>(
} },
run: async signal => {
if (!canRun() || Date.now() < earliest) return;
if ((options.reconstructionIntervalMs ?? 0) > 0) for (const [hint, until] of reconstructAfter) {
if (until <= Date.now()) reconstructAfter.delete(hint);
}
const ready = [...dirty].filter(hint => Math.max(reconstructAfter.get(hint) ?? 0, scopeFailures.get(hint)?.retryAt ?? 0) <= Date.now());
if (ready.length === 0 && ![...entries.values()].some(entry => Math.max(entry.dueAt, failures.get(entry.id)?.retryAt ?? 0) <= Date.now())) return;
earliest = Date.now() + (options.minimumPassIntervalMs ?? 1000);
try { await options.beforePass?.(); } catch (error) {
for (const hint of ready) failed(scopeFailures, hint);
for (const entry of entries.values()) if (entry.dueAt <= Date.now()) failed(failures, entry.id);
throw error;
}
const errors: unknown[] = [];
const ready = [...dirty].filter(hint => (scopeFailures.get(hint)?.retryAt ?? 0) <= Date.now());
const requested = ready.includes("*")
? ["*", ...ready.filter(hint => hint !== "*")].slice(0, 25)
: ready.slice(0, 25);
@@ -81,6 +94,9 @@ export function registerScopedDeadlineWork<Entry extends ScheduledEntry, Scope>(
entries.delete(entry.id); failures.delete(entry.id);
}
scans.delete(hint);
if ((options.reconstructionIntervalMs ?? 0) > 0) {
reconstructAfter.set(hint, Date.now() + options.reconstructionIntervalMs!);
}
if ((versions.get(hint) ?? 0) !== scan.version) dirty.add(hint); else versions.delete(hint);
if (!scope) scopeFailures.clear(); else scopeFailures.delete(hint);
} catch (error) {
@@ -1,3 +1,5 @@
import { notifyAttentionRetentionWork } from "./retention-work-notifications.js";
import { notifyStatusCardIssueChange } from "./status-card-notifications.js";
import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js";
import { notifyUnsafeWorkspaceExportRecovery } from "./unsafe-workspace-export-notifications.js";
import { and, asc, desc, eq, gt, inArray, isNull, sql } from "drizzle-orm";
@@ -126,6 +128,7 @@ export async function settleExternalConversation(db: Db, companyId: string, issu
const ready = await readSettlement(tx, companyId, issueId, true);
if (!ready) return false;
const { binding, issue, run } = ready;
await notifyAttentionRetentionWork(tx, { companyId });
const changed = await tx.update(chatConversations).set({ state: "waiting", updatedAt: new Date() })
.where(and(eq(chatConversations.id, binding.conversation.id), eq(chatConversations.companyId, companyId), eq(chatConversations.state, "active")))
.returning({ id: chatConversations.id });
@@ -133,6 +136,7 @@ export async function settleExternalConversation(db: Db, companyId: string, issu
await notifyUnsafeWorkspaceExportRecovery(tx, { companyId, issueId });
await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors);
await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers);
await notifyStatusCardIssueChange(tx, companyId);
await tx.update(issues).set({ status: "in_review", updatedAt: new Date() })
.where(and(eq(issues.id, issueId), eq(issues.companyId, companyId)));
activity = (await persistActivity(tx as unknown as Db, {
@@ -1,3 +1,5 @@
import { notifyAttentionRetentionWork } from "./retention-work-notifications.js";
import { notifyStatusCardIssueChange } from "./status-card-notifications.js";
import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js";
import { notifyExternalConversationSettlement } from "./external-conversation-settlement-notifications.js";
import { chatConversations, issues, type Db } from "@paperclipai/db";
@@ -45,10 +47,12 @@ export function nonIdleSlackIssueCondition() {
/** Called inside message admission while its transaction owns the issue lock. */
export async function resumeExternalConversation(tx: Db, companyId: string, issueId: string) {
await notifyAttentionRetentionWork(tx, { companyId });
await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers);
await notifyExternalConversationSettlement(tx, { companyId, issueId });
// Only the verified idle convention may leave In Review automatically.
// A real decision or delivery problem keeps the existing review semantics.
await notifyStatusCardIssueChange(tx, companyId);
await tx.update(issues).set({ status: "todo", updatedAt: new Date() })
.where(and(eq(issues.companyId, companyId), eq(issues.id, issueId),
sql`${externalConversationStateSql()} = 'waiting'`));
@@ -0,0 +1,98 @@
import { and, asc, eq, isNotNull, isNull } from "drizzle-orm";
import { companies, statusCards, type Db } from "@paperclipai/db";
import { logger } from "../middleware/logger.js";
import { registerDeadlineWork } from "./deadline-work.js";
import type { createDeliveryWorkCoordinator } from "./delivery-work-coordinator.js";
import { DELIVERY_QUEUES } from "./delivery-work-notifications.js";
import { instanceSettingsService } from "./instance-settings.js";
import { issueService } from "./issues.js";
import { queueIssueAssignmentWakeup, type IssueAssignmentWakeupDeps } from "./issue-assignment-wakeup.js";
import { statusCardService } from "./status-cards.js";
/** Feature-owned generation and assignment. Existing issue services retain
* execution authority, durable wake dedupe and stalled-generation cleanup. */
export function statusCardEvaluationService(db: Db, heartbeat: IssueAssignmentWakeupDeps) {
const cards = statusCardService(db);
const issues = issueService(db);
const settings = instanceSettingsService(db);
let reconstructReactive = true;
let allChanges = false;
const changes = new Set<string>();
let enabled: boolean | null = null;
let retryAt: number | null = null;
return {
changed(hints?: ReadonlySet<string>) {
for (const hint of hints ?? ["*"]) {
if (hint === "*" || hint === "changes:*") { allChanges = true; if (hint === "*") enabled = null; }
else if (hint.startsWith("changes:")) changes.add(hint.slice("changes:".length));
}
},
async run(signal: AbortSignal) {
if (enabled === null) enabled = (await settings.getExperimental()).enableStatusCards === true;
if (!enabled || signal.aborted) return;
const reconstruct = reconstructReactive;
const evaluateAllChanges = allChanges;
const companiesChanged = new Set(changes);
changes.clear();
reconstructReactive = false;
allChanges = false;
try {
const result = await cards.tickDueStatusCards(new Date(), evaluateAllChanges ? "changed" : companiesChanged, reconstruct);
retryAt = result.deferredCompanyIds.size ? Date.now() + 30_000 : null;
for (const companyId of result.deferredCompanyIds) changes.add(companyId);
const assignments = await Promise.allSettled(result.enqueued.map(async ({ cardId, generatingIssue }) => {
// Once generated, dispatch or cancel even if shutdown begins: leaving
// an unassigned claimed card is not a completed sweep.
try {
await queueIssueAssignmentWakeup({
heartbeat, issue: generatingIssue,
reason: "status_card_update_assigned",
mutation: "status_card.scheduler_update_requested",
contextSource: "status_card_scheduler",
requestedByActorType: "system", taskKey: `status-card:${cardId}`,
rethrowOnError: true,
});
} catch (error) {
await issues.update(generatingIssue.id, { status: "cancelled" });
throw error;
}
}));
const failures = assignments.filter(result => result.status === "rejected");
if (failures.length) throw new AggregateError(failures.map(result => result.reason), "Status-card assignment failed");
if (result.evaluated || result.enqueued.length) {
logger.info({ evaluated: result.evaluated, enqueued: result.enqueued.length }, "status-card evaluation complete");
}
} catch (error) {
reconstructReactive ||= reconstruct;
allChanges ||= evaluateAllChanges;
for (const companyId of companiesChanged) changes.add(companyId);
throw error;
}
},
async next() {
if (!enabled) return null;
const [next] = await db.select({ at: statusCards.nextEvalAt }).from(statusCards)
.innerJoin(companies, eq(companies.id, statusCards.companyId))
.where(and(eq(companies.status, "active"), isNull(statusCards.archivedAt),
isNull(statusCards.generatingIssueId), isNotNull(statusCards.nextEvalAt)))
.orderBy(asc(statusCards.nextEvalAt)).limit(1);
const deadline = next?.at?.getTime() ?? null;
return retryAt === null ? deadline : Math.min(deadline ?? Infinity, retryAt);
},
};
}
export function registerStatusCardEvaluationWork(
coordinator: ReturnType<typeof createDeliveryWorkCoordinator>,
service: ReturnType<typeof statusCardEvaluationService>,
canRun: () => boolean,
) {
return registerDeadlineWork(coordinator, DELIVERY_QUEUES.statusCardEvaluation, {
run: service.run, next: service.next, onSettled: service.changed, canRun, retryMs: 30_000,
// Batch source hints for at most one second after reconciliation. Do not
// use minimumSpacingMs: known policy/retry deadlines must preempt batching.
// This window alone arms no timer when the feature has no pending work.
notificationCoalesceMs: 1_000,
});
}
@@ -1,3 +1,4 @@
import { notifyStatusCardEvaluation } from "./status-card-notifications.js";
import { and, eq, isNull } from "drizzle-orm";
import type { Db } from "@paperclipai/db";
import { statusCards, statusCardUpdates } from "@paperclipai/db";
@@ -36,6 +37,7 @@ export async function finalizeStatusCardsForStalledGeneration(
) {
if (!STALLED_GENERATION_STATUSES.has(issue.status)) return [];
await notifyStatusCardEvaluation(dbOrTx, issue.companyId);
const now = new Date();
const failureReason = failureReasonForIssue(issue);
const cards = await dbOrTx
@@ -45,6 +47,7 @@ export async function finalizeStatusCardsForStalledGeneration(
failureReason,
generatingIssueId: null,
nextEvalAt: null,
nextEvalReason: null,
updatedAt: now,
})
.where(
Loaded 100 of 201 files, more files were not shown because too many files have changed in this diff. Show more