mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-11-03 09:37:19 +01:00
merge: preserve routine projections across retained execution producers
Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
commit
40bd4df17b
201 files changed
+8576
-1103
No files matched your search
@@ -1,3 +1,5 @@
|
||||
import { notifyStatusCardIssueChange } from "./status-card-notifications.js";
|
||||
import { notifyDeferredExecutionEligibility } from "./deferred-execution-notifications.js";
|
||||
import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js";
|
||||
import { notifyNativeStatusReassessment } from "./native-status-reassessment-notifications.js";
|
||||
import { notifyUnsafeWorkspaceExportRecovery } from "./unsafe-workspace-export-notifications.js";
|
||||
@@ -743,8 +745,10 @@ export function accessService(db: Db) {
|
||||
sql`${issues.status} not in ('done', 'cancelled')`,
|
||||
);
|
||||
await notifyNativeStatusReassessment(tx, { companyId });
|
||||
await notifyDeferredExecutionEligibility(tx, { companyId });
|
||||
await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors);
|
||||
await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers);
|
||||
await notifyStatusCardIssueChange(tx, companyId);
|
||||
const resetInProgress = await tx
|
||||
.update(issues)
|
||||
.set({
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { notifyAttentionRetentionWork } from "./retention-work-notifications.js";
|
||||
import { and, eq, sql } from "drizzle-orm";
|
||||
import { agents, agentRuntimeState, companies, heartbeatRuns, type Db } from "@paperclipai/db";
|
||||
import { normalizeCents, usdToCents, type AccountingFinding, type AccountingHealth, type AccountingInspection } from "@paperclipai/shared";
|
||||
@@ -75,6 +76,8 @@ export function accountingIntegrityService(db: Db, hooks: BudgetServiceHooks = {
|
||||
if (!reason.trim()) throw conflict("A repair reason is required");
|
||||
const before = await inspect(tx, companyId);
|
||||
if (before.fingerprint !== fingerprint) throw conflict("Accounting changed since inspection; inspect again before repairing");
|
||||
if (before.findings.some(f => f.repairable && f.kind === "agent_projection"))
|
||||
await notifyAttentionRetentionWork(tx, { companyId, sourceKind: "agent_error_alert" });
|
||||
for (const finding of before.findings.filter(f => f.repairable)) {
|
||||
if (finding.kind === "company_projection") await tx.update(companies).set({ spentMonthlyCents: sql`${finding.expected.cents}::numeric`, updatedAt: new Date() }).where(eq(companies.id, companyId));
|
||||
if (finding.kind === "agent_projection") await tx.update(agents).set({ spentMonthlyCents: sql`${finding.expected.cents}::numeric`, updatedAt: new Date() }).where(and(eq(agents.id, finding.entityId), eq(agents.companyId, companyId)));
|
||||
|
||||
@@ -1,3 +1,6 @@
|
||||
import { notifyAttentionRetentionWork } from "./retention-work-notifications.js";
|
||||
import { notifyDeferredExecutionEligibility } from "./deferred-execution-notifications.js";
|
||||
import { notifyRunDispatch } from "./run-dispatch-notifications.js";
|
||||
import { notifySkillVersionPinChange } from "./skill-version-retention-notifications.js";
|
||||
import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js";
|
||||
import { notifyExternalConversationSettlement } from "./external-conversation-settlement-notifications.js";
|
||||
@@ -16,6 +19,7 @@ import { clearPrimaryAgent } from "./primary-agent.js";
|
||||
import { budgetServiceInTransaction } from "./budgets.js";
|
||||
|
||||
export async function prepareAgentConfiguration(db: Db, existing: AgentConfigurationRecord, data: AgentConfigurationPatch, options?: UpdateAgentOptions): Promise<AgentConfigurationPatch> {
|
||||
await notifyAttentionRetentionWork(db, { companyId: existing.companyId });
|
||||
const id = existing.id;
|
||||
const { ensureManager, assertNoCycle, assertCompanyShortnameAvailable, assertBuiltInAgentMetadataMutationAllowed } = agentRecordQueries(db);
|
||||
if (existing.status === "pending_approval" && !options?.allowPendingApprovalConfigUpdate) {
|
||||
@@ -83,6 +87,8 @@ export async function prepareAgentConfiguration(db: Db, existing: AgentConfigura
|
||||
existing.adapterConfig,
|
||||
);
|
||||
}
|
||||
await notifyRunDispatch(db, { companyId: existing.companyId });
|
||||
await notifyDeferredExecutionEligibility(db, { companyId: existing.companyId });
|
||||
if (normalizedPatch.runtimeConfig !== undefined || normalizedPatch.reportsTo !== undefined) {
|
||||
await notifyDeliveryWork(db, DELIVERY_QUEUES.agentTimers);
|
||||
}
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
import { notifyStatusCardIssueChange } from "./status-card-notifications.js";
|
||||
import { notifyIssueRecovery } from "./issue-recovery-notifications.js";
|
||||
import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js";
|
||||
import { notifyUnsafeWorkspaceExportRecovery } from "./unsafe-workspace-export-notifications.js";
|
||||
import { FAST_RESPONSE_AGENT_GUIDANCE, fastResponseHistoryBody } from "@paperclipai/shared";
|
||||
@@ -269,6 +271,7 @@ export async function prepareConversationTurn(
|
||||
context.conversationReplayThroughCommentId = replayThrough?.id ?? commentId;
|
||||
}
|
||||
}
|
||||
await notifyStatusCardIssueChange(tx, issue.companyId);
|
||||
await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors);
|
||||
await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers);
|
||||
await tx
|
||||
@@ -389,6 +392,7 @@ export async function settleConversationTurn(
|
||||
issue.conversationState === conversationState
|
||||
)
|
||||
return true;
|
||||
await notifyStatusCardIssueChange(tx, issue.companyId);
|
||||
await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors);
|
||||
await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers);
|
||||
if (status === "in_review") await notifyUnsafeWorkspaceExportRecovery(tx, { companyId: issue.companyId, issueId: issue.id });
|
||||
@@ -544,6 +548,7 @@ export async function resumeConversationForReset(db: Db, comment: typeof issueCo
|
||||
eq(issues.id, comment.issueId), eq(issues.companyId, comment.companyId),
|
||||
)).for("update");
|
||||
if (!isConversation(issue) || comment.conversationSessionGeneration != null) return;
|
||||
await notifyIssueRecovery(tx, { companyId: issue.companyId });
|
||||
const released = await tx.update(issueTreeHolds).set({
|
||||
status: "released", releasedAt: new Date(), updatedAt: new Date(),
|
||||
releasedByActorType: "user", releasedByUserId: comment.authorUserId,
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
import { notifyAttentionRetentionWork } from "./retention-work-notifications.js";
|
||||
import { notifyStatusCardIssueChange } from "./status-card-notifications.js";
|
||||
import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js";
|
||||
import { notifyExternalConversationSettlement } from "./external-conversation-settlement-notifications.js";
|
||||
import { notifyUnsafeWorkspaceExportRecovery } from "./unsafe-workspace-export-notifications.js";
|
||||
@@ -34,10 +36,12 @@ export async function deleteAgentDependencies(tx: Db, companyId: string, id: str
|
||||
if (classifierHold) throw conflict("Wait for active prompt injection checks or resolve their unknown charges in Costs before deleting this agent", {
|
||||
code: "agent_prompt_injection_accounting_pending",
|
||||
});
|
||||
await notifyAttentionRetentionWork(tx, { companyId });
|
||||
await notifyExternalConversationSettlement(tx, { companyId });
|
||||
await issueThreadInteractionService(tx).cancelPendingForDeletedAddressee(companyId, id);
|
||||
await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors);
|
||||
await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers);
|
||||
await notifyStatusCardIssueChange(tx, companyId);
|
||||
await tx.update(agents).set({ reportsTo: null }).where(eq(agents.reportsTo, id));
|
||||
await tx
|
||||
.update(issues)
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { notifyAttentionRetentionWork } from "./retention-work-notifications.js";
|
||||
import { notifySkillVersionRetention, notifySkillVersionPinChange } from "./skill-version-retention-notifications.js";
|
||||
import { agentAppearanceSchema, randomAgentAppearance } from "@paperclipai/shared";
|
||||
import { normalizePaperclipRunnerAdapterConfig } from "@paperclipai/adapter-utils/server-utils";
|
||||
@@ -16,6 +17,7 @@ import { initializePrimaryAgent } from "./primary-agent.js";
|
||||
import { resolveAgentRunnerTaskDriverForCompany, resolveNewAgentRunnerForCompany } from "./agent-runner-selection.js";
|
||||
|
||||
export async function prepareAgentHire(db: Db, companyId: string, input: CreateAgentData, options?: CreateAgentOptions): Promise<CreateAgentData> {
|
||||
await notifyAttentionRetentionWork(db, { companyId });
|
||||
const { runner, ...data } = input;
|
||||
const selection = options?.runnerResolved
|
||||
? { adapterType: data.adapterType ?? "process", adapterConfig: isPlainRecord(data.adapterConfig) ? data.adapterConfig : {} }
|
||||
|
||||
@@ -1,3 +1,6 @@
|
||||
import { notifyAttentionRetentionWork } from "./retention-work-notifications.js";
|
||||
import { notifyDeferredExecutionEligibility } from "./deferred-execution-notifications.js";
|
||||
import { notifyRunDispatch } from "./run-dispatch-notifications.js";
|
||||
import { notifyNativeReviewResponse } from "./native-review-response-notifications.js";
|
||||
import { isAgentStatusInvokable } from "@paperclipai/shared";
|
||||
import { notifyNativeResponsePresentation } from "./native-response-presentation-notifications.js";
|
||||
@@ -38,7 +41,12 @@ export function createAgentLifecycleEffects(_hooks: BudgetServiceHooks = {}): li
|
||||
},
|
||||
notifyWork: tx => notifyDeliveryWork(tx, DELIVERY_QUEUES.agentLifecycle),
|
||||
async beforeStatusChange(tx, companyId, agentId, before, after) {
|
||||
if (before !== after) await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers);
|
||||
await notifyAttentionRetentionWork(tx, { companyId });
|
||||
if (before !== after) {
|
||||
await notifyRunDispatch(tx, { companyId });
|
||||
await notifyDeferredExecutionEligibility(tx, { companyId });
|
||||
await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers);
|
||||
}
|
||||
if (!isAgentStatusInvokable(before) && isAgentStatusInvokable(after)) {
|
||||
await notifyNativeResponsePresentation(tx, { companyId, agentId });
|
||||
await notifyNativeReviewResponse(tx, { companyId, agentId });
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { notifyAttentionRetentionWork } from "./retention-work-notifications.js";
|
||||
import { and, eq, inArray } from "drizzle-orm";
|
||||
import { agents, projects, type Db } from "@paperclipai/db";
|
||||
import { trustAuthorizationPolicySchema, type AgentProjectAccess } from "@paperclipai/shared";
|
||||
@@ -55,6 +56,7 @@ export async function addAgentProjectAccess(db: Db, actor: AuthorizationActor, c
|
||||
const added = requested.filter(id => !previous.includes(id));
|
||||
const next = [...previous, ...added];
|
||||
if (added.length) {
|
||||
await notifyAttentionRetentionWork(tx, { companyId, sourceKind: "agent_error_alert" });
|
||||
await tx.update(agents).set({ permissions: { ...agent.permissions, authorizationPolicy: {
|
||||
...policy, trustBoundary: { ...boundary, mode: "low_trust_review", companyId, projectIds: next },
|
||||
} }, updatedAt: new Date() }).where(and(eq(agents.id, agentId), eq(agents.companyId, companyId)));
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { notifyAttentionRetentionWork } from "./retention-work-notifications.js";
|
||||
import { type BudgetServiceHooks } from "./budgets.js";
|
||||
|
||||
import { and, desc, eq, inArray } from "drizzle-orm";
|
||||
@@ -52,7 +53,9 @@ export function agentService(db: Db, budgetHooks: BudgetServiceHooks = {}) {
|
||||
});
|
||||
}
|
||||
|
||||
const updated = await db
|
||||
const updated = await db.transaction(async tx => {
|
||||
await notifyAttentionRetentionWork(tx, { companyId: existing.companyId });
|
||||
return tx
|
||||
.update(agents)
|
||||
.set({
|
||||
permissions: normalizeAgentPermissions({ ...existing.permissions, ...permissions }),
|
||||
@@ -61,6 +64,7 @@ export function agentService(db: Db, budgetHooks: BudgetServiceHooks = {}) {
|
||||
.where(eq(agents.id, id))
|
||||
.returning()
|
||||
.then((rows) => rows[0] ?? null);
|
||||
});
|
||||
|
||||
return updated ? getById(updated.id) : null;
|
||||
},
|
||||
|
||||
@@ -0,0 +1,320 @@
|
||||
import { and, eq, getTableColumns, inArray, isNull, sql, type SQL } from "drizzle-orm";
|
||||
import { agents, decisionRetention, decisions, heartbeatRuns,
|
||||
issueApprovals, issueRecoveryActions, issueThreadInteractions, issues, approvals, type Db } from "@paperclipai/db";
|
||||
import { evaluateAgentInvokability } from "./agent-invokability.js";
|
||||
import { HUMAN_RECOVERY_OWNER_TYPES, ATTENTION_BUDGET_WARNING_PERCENT, ATTENTION_AGENT_ERROR_STATUS, collapsePendingConfirmationsToNewest, listFailedRunsWithNewerRun, readRunIssueId, requiresReviewAttention, terminalBlockerRequiresAttention } from "./attention-source-eligibility.js";
|
||||
import { decisionRetentionService, DEFAULT_DECISION_ARCHIVE_DAYS } from "./decision-retention.js";
|
||||
import type { createDeliveryWorkCoordinator } from "./delivery-work-coordinator.js";
|
||||
import { DELIVERY_QUEUES } from "./delivery-work-notifications.js";
|
||||
import { attentionIssueCondition } from "./issue-visibility.js";
|
||||
import { RETRY_EXHAUSTION_MESSAGE_PREFIX } from "./heartbeat-run-events.js";
|
||||
import { issueService } from "./issues.js";
|
||||
import { parseIssueExecutionState } from "./issue-execution-policy.js";
|
||||
import { isProspectiveBlockedTransition } from "./routable-blocked.js";
|
||||
import { registerScopedDeadlineWork } from "./scoped-deadline-work.js";
|
||||
import type { AttentionRetentionScope } from "./retention-work-notifications.js";
|
||||
|
||||
const archiveDelay = DEFAULT_DECISION_ARCHIVE_DAYS * 86_400_000;
|
||||
const kinds = ["decision", "issue_thread_interaction", "recovery_action", "blocker_attention", "review",
|
||||
"failed_run", "budget_alert", "agent_error_alert"] as const;
|
||||
type Kind = typeof kinds[number];
|
||||
type Scope = AttentionRetentionScope;
|
||||
type Candidate = { sourceKind: Kind; sourceId: string; companyId: string; activityAt: string; deadlineAt: number; issueId: string | null };
|
||||
type Entry = Candidate & { id: string; dueAt: number; rootId: string; variant?: "owner" | "terminal" };
|
||||
type Cursor = { sourceKind: string; sourceId: string };
|
||||
|
||||
type PageIssue = typeof issues.$inferSelect & { activityAt: string; deadlineAt: number;
|
||||
blockedActivityAt: string | null; blockedDeadlineAt: number | null };
|
||||
type PageEvidence = {
|
||||
decisions: Map<string, { originIssueId: string }>;
|
||||
recoveries: Map<string, typeof issueRecoveryActions.$inferSelect>;
|
||||
runs: Map<string, typeof heartbeatRuns.$inferSelect>;
|
||||
agents: Set<string>;
|
||||
visibleIssues: Map<string, PageIssue>;
|
||||
newerRuns: Set<string>;
|
||||
deferredRoots: Set<string>;
|
||||
blockers: Awaited<ReturnType<ReturnType<typeof issueService>["listBlockerAttention"]>>;
|
||||
};
|
||||
|
||||
const sourceTables: Record<Kind, SQL> = {
|
||||
decision: sql`decisions`, issue_thread_interaction: sql`issue_thread_interactions`,
|
||||
recovery_action: sql`issue_recovery_actions`, blocker_attention: sql`issues`, review: sql`issues`,
|
||||
failed_run: sql`heartbeat_runs`, budget_alert: sql`budget_incidents`, agent_error_alert: sql`agents`,
|
||||
};
|
||||
|
||||
const sources: Record<Kind, SQL> = {
|
||||
decision: sql`select 'decision' as source_kind, d.id::text as source_id, d.company_id, d.updated_at as activity_at, d.origin_issue_id::text as issue_id
|
||||
from decisions d where d.status = 'open'`,
|
||||
issue_thread_interaction: sql`select 'issue_thread_interaction', i.id::text, i.company_id, i.updated_at, i.issue_id::text
|
||||
from issue_thread_interactions i where i.status = 'pending' and i.addressee_user_id is null`,
|
||||
recovery_action: sql`select 'recovery_action', r.id::text, r.company_id, r.updated_at, r.source_issue_id::text
|
||||
from issue_recovery_actions r where r.status in ('active', 'escalated') and r.owner_type in (${sql.join(HUMAN_RECOVERY_OWNER_TYPES.map(owner => sql`${owner}`), sql`, `)})`,
|
||||
blocker_attention: sql`select 'blocker_attention', i.id::text, i.company_id, i.updated_at, i.id::text
|
||||
from issues i where i.status = 'blocked' and i.hidden_at is null and i.harness_kind is null`,
|
||||
review: sql`select 'review', i.id::text, i.company_id, i.updated_at, i.id::text
|
||||
from issues i where i.status = 'in_review' and i.hidden_at is null and i.harness_kind is null`,
|
||||
failed_run: sql`select 'failed_run', r.id::text, r.company_id, coalesce(r.finished_at, r.updated_at, r.created_at), coalesce(r.context_snapshot->>'issueId', r.context_snapshot->>'taskId')
|
||||
from heartbeat_runs r where r.status in ('failed', 'timed_out') and exists (
|
||||
select 1 from heartbeat_run_events e where e.run_id = r.id and e.company_id = r.company_id
|
||||
and e.event_type = 'lifecycle' and e.message like ${`${RETRY_EXHAUSTION_MESSAGE_PREFIX}%`})`,
|
||||
budget_alert: sql`select 'budget_alert', b.id::text, b.company_id, b.updated_at, null::text from budget_incidents b
|
||||
where b.status = 'open' and (b.threshold_type = 'hard' or (b.amount_limit > 0
|
||||
and round(b.amount_observed::numeric / b.amount_limit * 10000) / 100 >= ${ATTENTION_BUDGET_WARNING_PERCENT}))`,
|
||||
agent_error_alert: sql`select 'agent_error_alert', a.id::text, a.company_id, a.updated_at, null::text from agents a where a.status = ${ATTENTION_AGENT_ERROR_STATUS}`,
|
||||
};
|
||||
|
||||
export function attentionRetentionService(db: Db) {
|
||||
const cursors = new Map<string, Cursor>();
|
||||
async function candidates(database: Db, scope: Scope = {}, cursor?: Cursor, sourceId?: string): Promise<Candidate[]> {
|
||||
const issueIds = scope.issueIds ?? (scope.issueId ? [scope.issueId] : []);
|
||||
const selected = kinds.filter(kind => (!scope.sourceKind || scope.sourceKind === kind)
|
||||
&& (!issueIds.length || !["budget_alert", "agent_error_alert"].includes(kind)));
|
||||
if (!selected.length) return [];
|
||||
const rows = await database.execute(sql`with source_candidates(source_kind, source_id, company_id, activity_at, issue_id) as (${sql.join(selected.map(kind => sources[kind]), sql` union all `)})
|
||||
select s.source_kind as "sourceKind", s.source_id as "sourceId", s.company_id as "companyId",
|
||||
s.issue_id as "issueId", s.activity_at::text as "activityAt", ceil(extract(epoch from s.activity_at) * 1000 + ${archiveDelay}) as "deadlineAt"
|
||||
from source_candidates s inner join companies c on c.id = s.company_id
|
||||
left join decision_retention r on r.company_id = s.company_id and r.source_kind = s.source_kind and r.source_id = s.source_id
|
||||
where c.status = 'active'
|
||||
${issueIds.length ? sql`and s.issue_id in (${sql.join(issueIds.map(id => sql`${id}`), sql`, `)})` : sql``}
|
||||
and (${scope.companyId ?? null}::uuid is null or s.company_id = ${scope.companyId ?? null}::uuid)
|
||||
and (${sourceId ?? scope.sourceId ?? null}::text is null or s.source_id = ${sourceId ?? scope.sourceId ?? null})
|
||||
and (s.source_kind = 'blocker_attention' or ((r.id is null or r.keep = false) and r.archived_at is null))
|
||||
${cursor ? sql`and (s.source_kind, s.source_id) > (${cursor.sourceKind}, ${cursor.sourceId})` : sql``}
|
||||
order by s.source_kind, s.source_id limit 100`) as unknown as Candidate[];
|
||||
return rows.map(row => ({ ...row, deadlineAt: Number(row.deadlineAt) }));
|
||||
}
|
||||
|
||||
async function visibleIssue(database: Db, companyId: string, issueId: string, conversations = false) {
|
||||
const [row] = await database.select().from(issues).where(and(eq(issues.companyId, companyId), eq(issues.id, issueId),
|
||||
attentionIssueCondition(conversations)));
|
||||
return row;
|
||||
}
|
||||
|
||||
async function interactionEligibility(database: Db, candidates: Candidate[]) {
|
||||
const result = new Map<string, boolean>();
|
||||
const byCompany = new Map<string, string[]>();
|
||||
for (const candidate of candidates) {
|
||||
if (candidate.sourceKind !== "issue_thread_interaction") continue;
|
||||
const ids = byCompany.get(candidate.companyId) ?? [];
|
||||
ids.push(candidate.sourceId);
|
||||
byCompany.set(candidate.companyId, ids);
|
||||
}
|
||||
for (const [companyId, ids] of byCompany) {
|
||||
const rows = await database.select().from(issueThreadInteractions).where(and(
|
||||
eq(issueThreadInteractions.companyId, companyId), inArray(issueThreadInteractions.id, ids),
|
||||
eq(issueThreadInteractions.status, "pending"), isNull(issueThreadInteractions.addresseeUserId)));
|
||||
if (!rows.length) continue;
|
||||
const visible = await database.select({ id: issues.id }).from(issues).where(and(eq(issues.companyId, companyId),
|
||||
inArray(issues.id, rows.map(row => row.issueId)), attentionIssueCondition(true)));
|
||||
const visibleIds = new Set(visible.map(row => row.id));
|
||||
const roster = await database.select().from(agents).where(eq(agents.companyId, companyId));
|
||||
const boardAddressed = (row: typeof issueThreadInteractions.$inferSelect) => !row.addresseeAgentId
|
||||
|| !evaluateAgentInvokability(roster.find(agent => agent.id === row.addresseeAgentId), roster).invokable;
|
||||
const confirmationIssueIds = rows.filter(row => row.kind === "request_confirmation").map(row => row.issueId);
|
||||
const siblings = confirmationIssueIds.length ? await database.select().from(issueThreadInteractions).where(and(
|
||||
eq(issueThreadInteractions.companyId, companyId), inArray(issueThreadInteractions.issueId, confirmationIssueIds),
|
||||
eq(issueThreadInteractions.status, "pending"), eq(issueThreadInteractions.kind, "request_confirmation"),
|
||||
isNull(issueThreadInteractions.addresseeUserId))) : [];
|
||||
const newest = new Set(collapsePendingConfirmationsToNewest(siblings.filter(boardAddressed)).map(row => row.id));
|
||||
for (const row of rows) result.set(row.id, visibleIds.has(row.issueId) && boardAddressed(row)
|
||||
&& (row.kind !== "request_confirmation" || newest.has(row.id)));
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
async function reviewEligibility(database: Db, candidates: Candidate[]) {
|
||||
const result = new Map<string, boolean>();
|
||||
const byCompany = new Map<string, string[]>();
|
||||
for (const candidate of candidates) {
|
||||
if (candidate.sourceKind !== "review") continue;
|
||||
const ids = byCompany.get(candidate.companyId) ?? [];
|
||||
ids.push(candidate.sourceId);
|
||||
byCompany.set(candidate.companyId, ids);
|
||||
}
|
||||
for (const [companyId, ids] of byCompany) {
|
||||
const rows = await database.select().from(issues).where(and(eq(issues.companyId, companyId),
|
||||
inArray(issues.id, ids), attentionIssueCondition(false)));
|
||||
const pendingApprovals = await database.select({ issueId: issueApprovals.issueId }).from(issueApprovals)
|
||||
.innerJoin(approvals, eq(approvals.id, issueApprovals.approvalId)).where(and(
|
||||
eq(issueApprovals.companyId, companyId), eq(approvals.companyId, companyId),
|
||||
inArray(issueApprovals.issueId, ids), eq(approvals.status, "pending")));
|
||||
const approved = new Set(pendingApprovals.map(row => row.issueId));
|
||||
const stalled = await issueService(database).listReviewAttention(companyId, rows);
|
||||
for (const row of rows) {
|
||||
const state = parseIssueExecutionState(row.executionState);
|
||||
result.set(row.id, requiresReviewAttention({
|
||||
hasHumanParticipant: state?.status === "pending" && state.currentParticipant?.type === "user",
|
||||
hasUserAssignee: Boolean(row.assigneeUserId), hasPendingApproval: approved.has(row.id),
|
||||
stalled: stalled.get(row.id)?.state === "stalled",
|
||||
}));
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
async function pageEvidence(database: Db, candidates: Candidate[]) {
|
||||
const byCompany = new Map<string, Candidate[]>();
|
||||
for (const candidate of candidates) {
|
||||
if (!["decision", "recovery_action", "failed_run", "blocker_attention"].includes(candidate.sourceKind)) continue;
|
||||
const rows = byCompany.get(candidate.companyId) ?? [];
|
||||
rows.push(candidate);
|
||||
byCompany.set(candidate.companyId, rows);
|
||||
}
|
||||
const result = new Map<string, PageEvidence>();
|
||||
for (const [companyId, rows] of byCompany) {
|
||||
const ids = (kind: Kind) => rows.filter(row => row.sourceKind === kind).map(row => row.sourceId);
|
||||
const decisionIds = ids("decision"), recoveryIds = ids("recovery_action"), runIds = ids("failed_run"), blockerIds = ids("blocker_attention");
|
||||
const decisionRows = decisionIds.length ? await database.select({ id: decisions.id, originIssueId: decisions.originIssueId })
|
||||
.from(decisions).where(and(eq(decisions.companyId, companyId), inArray(decisions.id, decisionIds), eq(decisions.status, "open"))) : [];
|
||||
const recoveryRows = recoveryIds.length ? await database.select().from(issueRecoveryActions).where(and(
|
||||
eq(issueRecoveryActions.companyId, companyId), inArray(issueRecoveryActions.id, recoveryIds))) : [];
|
||||
const runRows = runIds.length ? await database.select().from(heartbeatRuns).where(and(
|
||||
eq(heartbeatRuns.companyId, companyId), inArray(heartbeatRuns.id, runIds))) : [];
|
||||
const runAgents = runRows.length ? await database.select({ id: agents.id }).from(agents).where(and(
|
||||
eq(agents.companyId, companyId), inArray(agents.id, [...new Set(runRows.map(row => row.agentId))]),
|
||||
sql`${agents.status} <> 'terminated'`)) : [];
|
||||
const readIssues = async (issueIds: string[]) => issueIds.length ? database.select({ ...getTableColumns(issues),
|
||||
activityAt: sql<string>`${issues.updatedAt}::text`,
|
||||
deadlineAt: sql<number>`ceil(extract(epoch from ${issues.updatedAt}) * 1000 + ${archiveDelay})`.mapWith(Number),
|
||||
blockedActivityAt: sql<string | null>`${issues.blockedTransitionAt}::text`,
|
||||
blockedDeadlineAt: sql<number | null>`ceil(extract(epoch from ${issues.blockedTransitionAt}) * 1000 + ${archiveDelay})`.mapWith(Number),
|
||||
}).from(issues).where(and(eq(issues.companyId, companyId), inArray(issues.id, [...new Set(issueIds)]),
|
||||
attentionIssueCondition(false))) : [];
|
||||
const issueIds = [...blockerIds, ...decisionRows.map(row => row.originIssueId), ...recoveryRows.map(row => row.sourceIssueId),
|
||||
...runRows.map(row => readRunIssueId(row.contextSnapshot))].filter((id): id is string => Boolean(id));
|
||||
const visibleRows = await readIssues(issueIds);
|
||||
const visible = new Map(visibleRows.map(row => [row.id, row]));
|
||||
const blockerRoots = visibleRows.filter(row => blockerIds.includes(row.id));
|
||||
const blockerBatch = blockerRoots.length ? await issueService(database).listBlockerAttentionBatch(companyId, blockerRoots)
|
||||
: { attention: new Map(), deferredRootIds: new Set<string>() };
|
||||
const blockers: PageEvidence["blockers"] = blockerBatch.attention;
|
||||
const terminalIds = [...blockers.values()].filter(terminalBlockerRequiresAttention)
|
||||
.map(attention => attention.terminalBlockerIssueId).filter((id): id is string => id != null && !visible.has(id));
|
||||
for (const row of await readIssues(terminalIds)) visible.set(row.id, row);
|
||||
result.set(companyId, {
|
||||
decisions: new Map(decisionRows.map(row => [row.id, row])), recoveries: new Map(recoveryRows.map(row => [row.id, row])),
|
||||
runs: new Map(runRows.map(row => [row.id, row])), agents: new Set(runAgents.map(row => row.id)), visibleIssues: visible,
|
||||
newerRuns: await listFailedRunsWithNewerRun(database, companyId,
|
||||
runRows.map(run => ({ ...run, issueId: readRunIssueId(run.contextSnapshot) }))), blockers, deferredRoots: blockerBatch.deferredRootIds,
|
||||
});
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
async function eligible(database: Db, candidate: Candidate, variant?: "owner" | "terminal", lock = false, reviewAttention?: Map<string, boolean>, interactionAttention?: Map<string, boolean>, evidence?: PageEvidence): Promise<Entry | null> {
|
||||
if (candidate.sourceKind === "blocker_attention" && variant === "terminal" && evidence?.deferredRoots.has(candidate.sourceId)) {
|
||||
return { ...candidate, rootId: candidate.sourceId, variant,
|
||||
id: `${candidate.sourceKind}:${candidate.sourceId}:terminal`, dueAt: Date.now() };
|
||||
}
|
||||
let sourceId = candidate.sourceId;
|
||||
let activityAt = candidate.activityAt;
|
||||
let dueAt = candidate.deadlineAt;
|
||||
if (candidate.sourceKind === "decision") {
|
||||
const row = evidence ? evidence.decisions.get(sourceId) : (await database.select({ originIssueId: decisions.originIssueId }).from(decisions)
|
||||
.where(and(eq(decisions.companyId, candidate.companyId), eq(decisions.id, sourceId), eq(decisions.status, "open"))))[0];
|
||||
if (!row || row.originIssueId && !(evidence ? evidence.visibleIssues.has(row.originIssueId)
|
||||
: await visibleIssue(database, candidate.companyId, row.originIssueId))) return null;
|
||||
} else if (candidate.sourceKind === "issue_thread_interaction") {
|
||||
const attention = interactionAttention ?? await interactionEligibility(database, [candidate]);
|
||||
if (!attention.get(sourceId)) return null;
|
||||
} else if (candidate.sourceKind === "recovery_action") {
|
||||
const row = evidence ? evidence.recoveries.get(sourceId) : (await database.select().from(issueRecoveryActions).where(and(eq(issueRecoveryActions.companyId, candidate.companyId), eq(issueRecoveryActions.id, sourceId))))[0];
|
||||
if (!row || !(evidence ? evidence.visibleIssues.has(row.sourceIssueId)
|
||||
: await visibleIssue(database, candidate.companyId, row.sourceIssueId))) return null;
|
||||
} else if (candidate.sourceKind === "review") {
|
||||
const attention = reviewAttention ?? await reviewEligibility(database, [candidate]);
|
||||
if (!attention.get(sourceId)) return null;
|
||||
} else if (candidate.sourceKind === "blocker_attention") {
|
||||
const row = evidence ? evidence.visibleIssues.get(sourceId) : await visibleIssue(database, candidate.companyId, sourceId);
|
||||
if (!row) return null;
|
||||
if (variant === "owner") {
|
||||
if (row.unblockDescriptor?.owner !== "board" || !isProspectiveBlockedTransition(row)) return null;
|
||||
const activity = evidence ? { at: evidence.visibleIssues.get(sourceId)!.blockedActivityAt!, due: evidence.visibleIssues.get(sourceId)!.blockedDeadlineAt! } : (await database.execute(sql`select blocked_transition_at::text as at,
|
||||
ceil(extract(epoch from blocked_transition_at) * 1000 + ${archiveDelay}) as due from issues
|
||||
where company_id = ${candidate.companyId}::uuid and id = ${row.id}::uuid`) as unknown as Array<{ at: string; due: string }>)[0];
|
||||
activityAt = activity.at; dueAt = Number(activity.due);
|
||||
} else {
|
||||
const attention = evidence ? evidence.blockers.get(row.id) : (await issueService(database).listBlockerAttention(candidate.companyId, [row])).get(row.id);
|
||||
if (!terminalBlockerRequiresAttention(attention)) return null;
|
||||
sourceId = attention.terminalBlockerIssueId ?? row.id;
|
||||
if (lock && sourceId !== candidate.sourceId) await database.execute(sql`select id from issues
|
||||
where company_id = ${candidate.companyId}::uuid and id = ${sourceId}::uuid for update`);
|
||||
const terminal = evidence ? evidence.visibleIssues.get(sourceId) : await visibleIssue(database, candidate.companyId, sourceId);
|
||||
if (!terminal) return null;
|
||||
const activity = evidence ? { at: evidence.visibleIssues.get(sourceId)!.activityAt, due: evidence.visibleIssues.get(sourceId)!.deadlineAt } : (await database.execute(sql`select updated_at::text as at,
|
||||
ceil(extract(epoch from updated_at) * 1000 + ${archiveDelay}) as due from issues
|
||||
where company_id = ${candidate.companyId}::uuid and id = ${sourceId}::uuid`) as unknown as Array<{ at: string; due: string }>)[0];
|
||||
activityAt = activity.at; dueAt = Number(activity.due);
|
||||
}
|
||||
const [retention] = await database.select().from(decisionRetention).where(and(eq(decisionRetention.companyId, candidate.companyId),
|
||||
eq(decisionRetention.sourceKind, candidate.sourceKind), eq(decisionRetention.sourceId, sourceId)));
|
||||
if (retention?.keep || retention?.archivedAt) return null;
|
||||
} else if (candidate.sourceKind === "failed_run") {
|
||||
const run = evidence ? evidence.runs.get(sourceId) : (await database.select().from(heartbeatRuns).where(and(eq(heartbeatRuns.companyId, candidate.companyId), eq(heartbeatRuns.id, sourceId))))[0];
|
||||
if (!run) return null;
|
||||
const agent = evidence ? evidence.agents.has(run.agentId) : (await database.select({ id: agents.id }).from(agents).where(and(eq(agents.companyId, candidate.companyId),
|
||||
eq(agents.id, run.agentId), sql`${agents.status} <> 'terminated'`)))[0];
|
||||
const issueId = readRunIssueId(run.contextSnapshot);
|
||||
if (!agent || issueId && !(evidence ? evidence.visibleIssues.has(issueId)
|
||||
: await visibleIssue(database, candidate.companyId, issueId))) return null;
|
||||
if ((evidence ? evidence.newerRuns : await listFailedRunsWithNewerRun(database, candidate.companyId, [{ ...run, issueId }])).has(run.id)) return null;
|
||||
}
|
||||
return { ...candidate, sourceId, activityAt, rootId: candidate.sourceId, variant,
|
||||
id: `${candidate.sourceKind}:${candidate.sourceId}:${variant ?? "source"}`, dueAt };
|
||||
}
|
||||
|
||||
async function load(scope?: Scope, signal?: AbortSignal) {
|
||||
const key = JSON.stringify(scope ?? {});
|
||||
const rows = await candidates(db, scope, cursors.get(key));
|
||||
const entries: Entry[] = [];
|
||||
const reviews = await reviewEligibility(db, rows);
|
||||
const interactions = await interactionEligibility(db, rows);
|
||||
const evidence = await pageEvidence(db, rows);
|
||||
for (const row of rows) {
|
||||
if (signal?.aborted) return { rows: null, complete: true };
|
||||
for (const variant of row.sourceKind === "blocker_attention" ? ["owner", "terminal"] as const : [undefined]) {
|
||||
const entry = await eligible(db, row, variant, false, reviews, interactions, evidence.get(row.companyId));
|
||||
if (entry) entries.push(entry);
|
||||
}
|
||||
}
|
||||
if (rows.length === 100) {
|
||||
const last = rows[rows.length - 1]!;
|
||||
cursors.set(key, { sourceKind: last.sourceKind, sourceId: last.sourceId });
|
||||
} else cursors.delete(key);
|
||||
return { rows: entries, complete: rows.length < 100 };
|
||||
}
|
||||
|
||||
async function check(entry: Entry) {
|
||||
return db.transaction(async transaction => {
|
||||
const database = transaction as unknown as Db;
|
||||
await database.execute(sql`select id from ${sourceTables[entry.sourceKind]}
|
||||
where company_id = ${entry.companyId}::uuid and id = ${entry.rootId}::uuid for update`);
|
||||
const [candidate] = await candidates(database, { companyId: entry.companyId, sourceKind: entry.sourceKind }, undefined, entry.rootId);
|
||||
if (!candidate) return null;
|
||||
const current = await eligible(database, candidate, entry.variant, true);
|
||||
if (!current || current.dueAt > Date.now()) return current;
|
||||
const retention = decisionRetentionService(database);
|
||||
const item = { sourceKind: current.sourceKind, subject: { id: current.sourceId }, activityAt: current.activityAt };
|
||||
const state = (await retention.syncItems(current.companyId, [item])).get(`${current.sourceKind}:${current.sourceId}`)!;
|
||||
await retention.autoArchive({ companyId: current.companyId, items: [{ ...item,
|
||||
keep: state.keep, archivedAt: state.archivedAt?.toISOString() ?? null, retentionVersion: state.version }] });
|
||||
return null;
|
||||
});
|
||||
}
|
||||
return { load, check };
|
||||
}
|
||||
|
||||
export function registerAttentionRetentionWork(coordinator: ReturnType<typeof createDeliveryWorkCoordinator>,
|
||||
service: ReturnType<typeof attentionRetentionService>, canRun: () => boolean) {
|
||||
return registerScopedDeadlineWork(coordinator, {
|
||||
queue: DELIVERY_QUEUES.attentionRetention,
|
||||
decodeScope: hint => hint === "*" ? undefined : JSON.parse(hint) as Scope,
|
||||
matches: (entry, scope) => (!scope?.companyId || scope.companyId === entry.companyId)
|
||||
&& (!scope?.sourceKind || scope.sourceKind === entry.sourceKind)
|
||||
&& (!scope?.sourceId || scope.sourceId === entry.sourceId)
|
||||
&& (!scope?.issueId || scope.issueId === entry.issueId)
|
||||
&& (!scope?.issueIds?.length || entry.issueId !== null && scope.issueIds.includes(entry.issueId)),
|
||||
load: service.load, check: service.check,
|
||||
}, canRun);
|
||||
}
|
||||
@@ -0,0 +1,112 @@
|
||||
import { and, eq, gt, inArray, sql, type SQL } from "drizzle-orm";
|
||||
import { heartbeatRuns, type Db } from "@paperclipai/db";
|
||||
|
||||
export const HUMAN_RECOVERY_OWNER_TYPES = ["user", "board"] as const;
|
||||
export const ATTENTION_BUDGET_WARNING_PERCENT = 85;
|
||||
export const ATTENTION_AGENT_ERROR_STATUS = "error";
|
||||
|
||||
export function collapsePendingConfirmationsToNewest<T extends {
|
||||
id: string;
|
||||
issueId: string;
|
||||
kind: string;
|
||||
createdAt: Date;
|
||||
payload: unknown;
|
||||
}>(rows: T[]) {
|
||||
const independent = (row: T) => Boolean(row.payload && typeof row.payload === "object" && "qualityReview" in row.payload && row.payload.qualityReview);
|
||||
const newestByGroup = new Map<string, T>();
|
||||
for (const row of rows) {
|
||||
if (row.kind !== "request_confirmation" || independent(row)) continue;
|
||||
const groupKey = `${row.issueId}:${row.kind}`;
|
||||
const newest = newestByGroup.get(groupKey);
|
||||
if (
|
||||
!newest
|
||||
|| row.createdAt.getTime() > newest.createdAt.getTime()
|
||||
|| (row.createdAt.getTime() === newest.createdAt.getTime() && row.id > newest.id)
|
||||
) {
|
||||
newestByGroup.set(groupKey, row);
|
||||
}
|
||||
}
|
||||
|
||||
return rows.filter((row) => (
|
||||
row.kind !== "request_confirmation"
|
||||
|| independent(row)
|
||||
|| newestByGroup.get(`${row.issueId}:${row.kind}`)?.id === row.id
|
||||
));
|
||||
}
|
||||
|
||||
export function readRunIssueId(contextSnapshot: Record<string, unknown> | null) {
|
||||
const issueId = contextSnapshot?.issueId ?? contextSnapshot?.taskId;
|
||||
return typeof issueId === "string" && issueId.length > 0 ? issueId : null;
|
||||
}
|
||||
|
||||
type FailedRunProbe = { id: string; agentId: string; issueId: string | null; createdAt: Date };
|
||||
|
||||
const FAILED_RUN_PROBE_BATCH_SIZE = 1000;
|
||||
|
||||
export async function listFailedRunsWithNewerRun(db: Db, companyId: string, failedRuns: FailedRunProbe[]) {
|
||||
const withNewerRun = new Set<string>();
|
||||
const keyed = failedRuns.filter(run => run.issueId !== null);
|
||||
for (let start = 0; start < keyed.length; start += FAILED_RUN_PROBE_BATCH_SIZE) {
|
||||
const batch = keyed.slice(start, start + FAILED_RUN_PROBE_BATCH_SIZE);
|
||||
for (const id of await listFailedRunBatchWithNewerRun(db, companyId, batch)) withNewerRun.add(id);
|
||||
}
|
||||
const unkeyed = failedRuns.filter(run => run.issueId === null);
|
||||
if (unkeyed.length === 0) return withNewerRun;
|
||||
const agentIds = [...new Set(unkeyed.map(run => run.agentId))];
|
||||
const oldestCreatedAt = unkeyed.reduce((oldest, run) => run.createdAt < oldest ? run.createdAt : oldest,
|
||||
unkeyed[0]!.createdAt);
|
||||
const latestByAgent = new Map<string, Date>();
|
||||
for (let start = 0; start < agentIds.length; start += FAILED_RUN_PROBE_BATCH_SIZE) {
|
||||
const rows = await db.select({
|
||||
agentId: heartbeatRuns.agentId,
|
||||
createdAt: sql<Date>`max(${heartbeatRuns.createdAt})`.mapWith(heartbeatRuns.createdAt),
|
||||
}).from(heartbeatRuns).where(and(
|
||||
eq(heartbeatRuns.companyId, companyId),
|
||||
inArray(heartbeatRuns.agentId, agentIds.slice(start, start + FAILED_RUN_PROBE_BATCH_SIZE)),
|
||||
gt(heartbeatRuns.createdAt, oldestCreatedAt),
|
||||
sql`coalesce(${heartbeatRuns.contextSnapshot} ->> 'issueId', ${heartbeatRuns.contextSnapshot} ->> 'taskId', '') = ''`,
|
||||
)).groupBy(heartbeatRuns.agentId);
|
||||
for (const row of rows) latestByAgent.set(row.agentId, row.createdAt);
|
||||
}
|
||||
for (const run of unkeyed) {
|
||||
if ((latestByAgent.get(run.agentId)?.getTime() ?? 0) > run.createdAt.getTime()) withNewerRun.add(run.id);
|
||||
}
|
||||
return withNewerRun;
|
||||
}
|
||||
|
||||
async function listFailedRunBatchWithNewerRun(db: Db, companyId: string, failedRuns: FailedRunProbe[]) {
|
||||
if (failedRuns.length === 0) return [];
|
||||
const failed = sql.join(
|
||||
failedRuns.map((run) => sql`(${run.id}::uuid, ${run.agentId}::uuid, ${run.issueId}::text, ${new Date(run.createdAt.getTime() + 1).toISOString()}::timestamptz)`),
|
||||
sql`, `,
|
||||
);
|
||||
const runIssueId = sql`(${heartbeatRuns.contextSnapshot} ->> 'issueId')`;
|
||||
const runTaskId = sql`(${heartbeatRuns.contextSnapshot} ->> 'taskId')`;
|
||||
const newerRunExists = (match: SQL) => sql`exists (
|
||||
select 1 from ${heartbeatRuns}
|
||||
where ${heartbeatRuns.companyId} = ${companyId}
|
||||
and ${heartbeatRuns.agentId} = failed.agent_id
|
||||
and ${heartbeatRuns.createdAt} >= failed.newer_from
|
||||
and ${match}
|
||||
)`;
|
||||
const rows = await db.execute(sql`
|
||||
select failed.id
|
||||
from (values ${failed}) as failed (id, agent_id, issue_id, newer_from)
|
||||
where ${newerRunExists(sql`${runIssueId} = failed.issue_id`)}
|
||||
or ${newerRunExists(sql`${runTaskId} = failed.issue_id and ${runIssueId} is null`)}
|
||||
`) as unknown as Array<{ id: string }>;
|
||||
return rows.map((row) => row.id);
|
||||
}
|
||||
|
||||
|
||||
export function requiresReviewAttention(input: {
|
||||
hasHumanParticipant: boolean; hasUserAssignee: boolean; hasPendingApproval: boolean; stalled: boolean;
|
||||
}) {
|
||||
return input.hasHumanParticipant || input.hasUserAssignee || input.hasPendingApproval || input.stalled;
|
||||
}
|
||||
|
||||
export function terminalBlockerRequiresAttention<T extends { state?: string; blockingTreeLive?: boolean }>(
|
||||
attention: T | null | undefined,
|
||||
): attention is T & { state: "stalled" | "needs_attention" } {
|
||||
return Boolean(attention && !attention.blockingTreeLive && ["stalled", "needs_attention"].includes(attention.state ?? ""));
|
||||
}
|
||||
@@ -1,5 +1,6 @@
|
||||
import { HUMAN_RECOVERY_OWNER_TYPES, ATTENTION_BUDGET_WARNING_PERCENT, ATTENTION_AGENT_ERROR_STATUS, collapsePendingConfirmationsToNewest, readRunIssueId, listFailedRunsWithNewerRun, requiresReviewAttention, terminalBlockerRequiresAttention } from "./attention-source-eligibility.js";
|
||||
import { approvalExecutionIssueCondition, firstApprovalIssueId } from "./approval-visibility.js";
|
||||
import { and, asc, desc, eq, gt, inArray, isNotNull, isNull, notInArray, or, sql, type SQL } from "drizzle-orm";
|
||||
import { and, asc, desc, eq, inArray, isNull, notInArray, or, sql, type SQL } from "drizzle-orm";
|
||||
import type { Db } from "@paperclipai/db";
|
||||
import {
|
||||
agents,
|
||||
@@ -58,7 +59,7 @@ import {
|
||||
BLOCKER_ATTENTION_MAX_NODES,
|
||||
issueService,
|
||||
} from "./issues.js";
|
||||
import { executionIssueCondition, visibleIssueCondition } from "./issue-visibility.js";
|
||||
import { attentionIssueCondition, executionIssueCondition } from "./issue-visibility.js";
|
||||
import { parseIssueExecutionState } from "./issue-execution-policy.js";
|
||||
import { isProspectiveBlockedTransition } from "./routable-blocked.js";
|
||||
import { issueReadSqlCondition, type AuthorizationActor } from "./authorization.js";
|
||||
@@ -107,7 +108,6 @@ const SOURCE_RANK: Record<AttentionSourceKind, number> = {
|
||||
|
||||
const PENDING_INTERACTION_STATUSES = ["pending"] as const;
|
||||
const OPEN_RECOVERY_STATUSES = ["active", "escalated"] as const;
|
||||
const HUMAN_RECOVERY_OWNER_TYPES = ["user", "board"] as const;
|
||||
const DETAIL_EXCERPT_LENGTH = 160;
|
||||
const DETAIL_IMAGE_LIMIT = 3;
|
||||
const OPEN_DECISION_DEFAULT_LIMIT = 500;
|
||||
@@ -766,37 +766,6 @@ export function interactionResolverAudience(
|
||||
};
|
||||
}
|
||||
|
||||
function collapsePendingConfirmationsToNewest<T extends {
|
||||
id: string;
|
||||
issueId: string;
|
||||
kind: string;
|
||||
createdAt: Date;
|
||||
payload: unknown;
|
||||
}>(rows: T[]) {
|
||||
// Quality gates bind independent outputs; a newer visual approval must not
|
||||
// hide a still-pending copy approval (or affect its badge count).
|
||||
const independent = (row: T) => Boolean(row.payload && typeof row.payload === "object" && "qualityReview" in row.payload && row.payload.qualityReview);
|
||||
const newestByGroup = new Map<string, T>();
|
||||
for (const row of rows) {
|
||||
if (row.kind !== "request_confirmation" || independent(row)) continue;
|
||||
const groupKey = `${row.issueId}:${row.kind}`;
|
||||
const newest = newestByGroup.get(groupKey);
|
||||
if (
|
||||
!newest
|
||||
|| row.createdAt.getTime() > newest.createdAt.getTime()
|
||||
|| (row.createdAt.getTime() === newest.createdAt.getTime() && row.id > newest.id)
|
||||
) {
|
||||
newestByGroup.set(groupKey, row);
|
||||
}
|
||||
}
|
||||
|
||||
return rows.filter((row) => (
|
||||
row.kind !== "request_confirmation"
|
||||
|| independent(row)
|
||||
|| newestByGroup.get(`${row.issueId}:${row.kind}`)?.id === row.id
|
||||
));
|
||||
}
|
||||
|
||||
function budgetObservedPercent(amountObserved: number, amountLimit: number) {
|
||||
return amountLimit > 0 ? Math.round((amountObserved / amountLimit) * 10_000) / 100 : 0;
|
||||
}
|
||||
@@ -866,9 +835,7 @@ async function issueSummaryMap(
|
||||
.where(and(
|
||||
eq(issues.companyId, companyId),
|
||||
inArray(issues.id, ids),
|
||||
includeConversations
|
||||
? or(executionIssueCondition(), and(visibleIssueCondition(), isNotNull(issues.conversationAgentId)))
|
||||
: executionIssueCondition(),
|
||||
attentionIssueCondition(includeConversations),
|
||||
readCondition,
|
||||
));
|
||||
return new Map(rows.map((row) => [row.id, {
|
||||
@@ -1108,70 +1075,6 @@ function resolveBlockingIssue(
|
||||
return null;
|
||||
}
|
||||
|
||||
function readRunIssueId(contextSnapshot: Record<string, unknown> | null) {
|
||||
const issueId = contextSnapshot?.issueId ?? contextSnapshot?.taskId;
|
||||
return typeof issueId === "string" && issueId.length > 0 ? issueId : null;
|
||||
}
|
||||
|
||||
type FailedRunProbe = { id: string; agentId: string; issueId: string | null; createdAt: Date };
|
||||
|
||||
const FAILED_RUN_PROBE_BATCH_SIZE = 1000;
|
||||
|
||||
async function listFailedRunsWithNewerRun(db: Db, companyId: string, failedRuns: FailedRunProbe[]) {
|
||||
const withNewerRun = new Set<string>();
|
||||
const keyed = failedRuns.filter(run => run.issueId !== null);
|
||||
for (let start = 0; start < keyed.length; start += FAILED_RUN_PROBE_BATCH_SIZE) {
|
||||
const batch = keyed.slice(start, start + FAILED_RUN_PROBE_BATCH_SIZE);
|
||||
for (const id of await listFailedRunBatchWithNewerRun(db, companyId, batch)) withNewerRun.add(id);
|
||||
}
|
||||
const unkeyed = failedRuns.filter(run => run.issueId === null);
|
||||
if (unkeyed.length === 0) return withNewerRun;
|
||||
const agentIds = [...new Set(unkeyed.map(run => run.agentId))];
|
||||
const oldestCreatedAt = unkeyed.reduce((oldest, run) => run.createdAt < oldest ? run.createdAt : oldest,
|
||||
unkeyed[0]!.createdAt);
|
||||
const latestByAgent = new Map<string, Date>();
|
||||
for (let start = 0; start < agentIds.length; start += FAILED_RUN_PROBE_BATCH_SIZE) {
|
||||
const rows = await db.select({
|
||||
agentId: heartbeatRuns.agentId,
|
||||
createdAt: sql<Date>`max(${heartbeatRuns.createdAt})`.mapWith(heartbeatRuns.createdAt),
|
||||
}).from(heartbeatRuns).where(and(
|
||||
eq(heartbeatRuns.companyId, companyId),
|
||||
inArray(heartbeatRuns.agentId, agentIds.slice(start, start + FAILED_RUN_PROBE_BATCH_SIZE)),
|
||||
gt(heartbeatRuns.createdAt, oldestCreatedAt),
|
||||
sql`coalesce(${heartbeatRuns.contextSnapshot} ->> 'issueId', ${heartbeatRuns.contextSnapshot} ->> 'taskId', '') = ''`,
|
||||
)).groupBy(heartbeatRuns.agentId);
|
||||
for (const row of rows) latestByAgent.set(row.agentId, row.createdAt);
|
||||
}
|
||||
for (const run of unkeyed) {
|
||||
if ((latestByAgent.get(run.agentId)?.getTime() ?? 0) > run.createdAt.getTime()) withNewerRun.add(run.id);
|
||||
}
|
||||
return withNewerRun;
|
||||
}
|
||||
|
||||
async function listFailedRunBatchWithNewerRun(db: Db, companyId: string, failedRuns: FailedRunProbe[]) {
|
||||
if (failedRuns.length === 0) return [];
|
||||
const failed = sql.join(
|
||||
failedRuns.map((run) => sql`(${run.id}::uuid, ${run.agentId}::uuid, ${run.issueId}::text, ${new Date(run.createdAt.getTime() + 1).toISOString()}::timestamptz)`),
|
||||
sql`, `,
|
||||
);
|
||||
const runIssueId = sql`(${heartbeatRuns.contextSnapshot} ->> 'issueId')`;
|
||||
const runTaskId = sql`(${heartbeatRuns.contextSnapshot} ->> 'taskId')`;
|
||||
const newerRunExists = (match: SQL) => sql`exists (
|
||||
select 1 from ${heartbeatRuns}
|
||||
where ${heartbeatRuns.companyId} = ${companyId}
|
||||
and ${heartbeatRuns.agentId} = failed.agent_id
|
||||
and ${heartbeatRuns.createdAt} >= failed.newer_from
|
||||
and ${match}
|
||||
)`;
|
||||
const rows = await db.execute(sql`
|
||||
select failed.id
|
||||
from (values ${failed}) as failed (id, agent_id, issue_id, newer_from)
|
||||
where ${newerRunExists(sql`${runIssueId} = failed.issue_id`)}
|
||||
or ${newerRunExists(sql`${runTaskId} = failed.issue_id and ${runIssueId} is null`)}
|
||||
`) as unknown as Array<{ id: string }>;
|
||||
return rows.map((row) => row.id);
|
||||
}
|
||||
|
||||
export function attentionService(db: Db, serviceOptions: AttentionServiceOptions = {}) {
|
||||
const openDecisionLimit = Math.min(
|
||||
Math.max(Math.trunc(serviceOptions.openDecisionLimit ?? OPEN_DECISION_DEFAULT_LIMIT), 1),
|
||||
@@ -1636,8 +1539,7 @@ export function attentionService(db: Db, serviceOptions: AttentionServiceOptions
|
||||
}));
|
||||
}
|
||||
const blockerAttention = issue.blockerAttention;
|
||||
if (blockerAttention?.state !== "stalled" && blockerAttention?.state !== "needs_attention") continue;
|
||||
if (blockerAttention.blockingTreeLive) continue;
|
||||
if (!terminalBlockerRequiresAttention(blockerAttention)) continue;
|
||||
const issueSummary = blockedIssueSummaries.get(issue.id) ?? null;
|
||||
const terminalIssueId = blockerAttention.terminalBlockerIssueId ?? issue.id;
|
||||
const terminalSummary = terminalBlockerSummaries.get(terminalIssueId)
|
||||
@@ -1739,7 +1641,8 @@ export function attentionService(db: Db, serviceOptions: AttentionServiceOptions
|
||||
const pendingApprovalId = pendingApprovalByIssueId.get(review.id) ?? null;
|
||||
const reviewAttention = reviewAttentionByIssueId.get(review.id);
|
||||
const stalled = reviewAttention?.state === "stalled";
|
||||
if (!hasHumanParticipant && !review.assigneeUserId && !pendingApprovalId && !stalled) continue;
|
||||
if (!requiresReviewAttention({ hasHumanParticipant, hasUserAssignee: Boolean(review.assigneeUserId),
|
||||
hasPendingApproval: Boolean(pendingApprovalId), stalled })) continue;
|
||||
const issue = reviewIssueMap.get(review.id);
|
||||
if (!issue) continue;
|
||||
const dedupKey = `review:${review.id}`;
|
||||
@@ -1858,7 +1761,7 @@ export function attentionService(db: Db, serviceOptions: AttentionServiceOptions
|
||||
const budgetOverview = await budgetService(db).overview(companyId);
|
||||
for (const incident of budgetOverview.activeIncidents) {
|
||||
const observedPercent = budgetObservedPercent(incident.amountObserved, incident.amountLimit);
|
||||
if (incident.thresholdType !== "hard" && observedPercent < 85) continue;
|
||||
if (incident.thresholdType !== "hard" && observedPercent < ATTENTION_BUDGET_WARNING_PERCENT) continue;
|
||||
const dedupKey = `budget:${incident.policyId}:${toIso(incident.windowStart)}:${incident.thresholdType}`;
|
||||
add(createItem({
|
||||
companyId,
|
||||
@@ -1921,7 +1824,7 @@ export function attentionService(db: Db, serviceOptions: AttentionServiceOptions
|
||||
updatedAt: agents.updatedAt,
|
||||
})
|
||||
.from(agents)
|
||||
.where(and(eq(agents.companyId, companyId), eq(agents.status, "error")))
|
||||
.where(and(eq(agents.companyId, companyId), eq(agents.status, ATTENTION_AGENT_ERROR_STATUS)))
|
||||
.orderBy(desc(agents.updatedAt), desc(agents.id));
|
||||
|
||||
for (const agent of erroredAgents) {
|
||||
|
||||
@@ -1,5 +1,8 @@
|
||||
import { notifyStatusCardIssueChange } from "./status-card-notifications.js";
|
||||
import { notifyDeferredExecutionEligibility } from "./deferred-execution-notifications.js";
|
||||
import { notifyRunDispatch } from "./run-dispatch-notifications.js";
|
||||
import { notifyApprovalWork } from "./approval-work-notifications.js";
|
||||
import { notifyCompanyRetentionWork } from "./retention-work-notifications.js";
|
||||
import { notifyAttentionRetentionWork, notifyCompanyRetentionWork } from "./retention-work-notifications.js";
|
||||
import { addCents, compareCents, normalizeCents } from "@paperclipai/shared";
|
||||
import { and, desc, eq, inArray, sql, or, isNull } from "drizzle-orm";
|
||||
import type { Db } from "@paperclipai/db";
|
||||
@@ -163,6 +166,19 @@ async function resolveScopeRecord(db: Db, scopeType: BudgetScopeType, scopeId: s
|
||||
};
|
||||
}
|
||||
|
||||
async function resolveSummaryScopeRecord(db: Db, companyId: string, scopeType: BudgetScopeType, scopeId: string): Promise<ScopeRecord> {
|
||||
let scope: ScopeRecord;
|
||||
try {
|
||||
scope = await resolveScopeRecord(db, scopeType, scopeId);
|
||||
} catch (error) {
|
||||
if (!(error instanceof HttpError) || error.status !== 404 || (scopeType !== "agent" && scopeType !== "project")) throw error;
|
||||
return { companyId, name: scopeType === "agent" ? "Deleted agent" : "Deleted project",
|
||||
paused: false, pauseReason: null, budgetHeld: false };
|
||||
}
|
||||
if (scope.companyId !== companyId) throw notFound("Budget scope not found");
|
||||
return scope;
|
||||
}
|
||||
|
||||
// Keep a large dashboard from filling the database pool with queued reads.
|
||||
async function mapBudgetReads<T, R>(rows: T[], read: (row: T) => Promise<R>): Promise<R[]> {
|
||||
const result: R[] = [];
|
||||
@@ -287,6 +303,8 @@ export function budgetServiceInTransaction(db: Db, publications: ActivityPublica
|
||||
}
|
||||
|
||||
if (policy.scopeType === "project") {
|
||||
await notifyRunDispatch(db, { companyId: policy.companyId });
|
||||
await notifyDeferredExecutionEligibility(db, { companyId: policy.companyId });
|
||||
await db
|
||||
.update(projects)
|
||||
.set({
|
||||
@@ -300,7 +318,10 @@ export function budgetServiceInTransaction(db: Db, publications: ActivityPublica
|
||||
|
||||
const scope = await resolveScopeRecord(db, "company", policy.scopeId);
|
||||
if (scope.paused) return;
|
||||
await notifyRunDispatch(db, { companyId: policy.companyId });
|
||||
await notifyDeferredExecutionEligibility(db, { companyId: policy.companyId });
|
||||
await notifyCompanyRetentionWork(db, policy.companyId);
|
||||
await notifyStatusCardIssueChange(db, policy.companyId);
|
||||
await notifyDeliveryWork(db, DELIVERY_QUEUES.agentTimers);
|
||||
await notifyDeliveryWork(db, DELIVERY_QUEUES.routineSchedules);
|
||||
await notifyDeliveryWork(db, DELIVERY_QUEUES.issueMonitors);
|
||||
@@ -346,6 +367,8 @@ export function budgetServiceInTransaction(db: Db, publications: ActivityPublica
|
||||
}
|
||||
|
||||
if (policy.scopeType === "project") {
|
||||
await notifyRunDispatch(db, { companyId: policy.companyId });
|
||||
await notifyDeferredExecutionEligibility(db, { companyId: policy.companyId });
|
||||
await db
|
||||
.update(projects)
|
||||
.set({
|
||||
@@ -360,7 +383,10 @@ export function budgetServiceInTransaction(db: Db, publications: ActivityPublica
|
||||
await notifyDeliveryWork(db, DELIVERY_QUEUES.skillSourceSync);
|
||||
const scope = await resolveScopeRecord(db, "company", policy.scopeId);
|
||||
if (scope.pauseReason !== "budget") return;
|
||||
await notifyRunDispatch(db, { companyId: policy.companyId });
|
||||
await notifyDeferredExecutionEligibility(db, { companyId: policy.companyId });
|
||||
await notifyCompanyRetentionWork(db, policy.companyId);
|
||||
await notifyStatusCardIssueChange(db, policy.companyId);
|
||||
await notifyDeliveryWork(db, DELIVERY_QUEUES.agentTimers);
|
||||
await notifyDeliveryWork(db, DELIVERY_QUEUES.routineSchedules);
|
||||
await notifyDeliveryWork(db, DELIVERY_QUEUES.issueMonitors);
|
||||
@@ -395,7 +421,7 @@ export function budgetServiceInTransaction(db: Db, publications: ActivityPublica
|
||||
}
|
||||
|
||||
async function buildPolicySummary(policy: PolicyRow): Promise<BudgetPolicySummary> {
|
||||
const scope = await resolveScopeRecord(db, policy.scopeType as BudgetScopeType, policy.scopeId);
|
||||
const scope = await resolveSummaryScopeRecord(db, policy.companyId, policy.scopeType as BudgetScopeType, policy.scopeId);
|
||||
const observed = await computeObservedSpend(db, policy);
|
||||
const { total: observedAmount, totalExact: observedAmountExact, unpricedEventCount, pendingRunCount } = observed;
|
||||
const { windowStart: start, windowEnd: end } = observed;
|
||||
@@ -466,6 +492,7 @@ export function budgetServiceInTransaction(db: Db, publications: ActivityPublica
|
||||
windowEnd: end,
|
||||
});
|
||||
|
||||
await notifyAttentionRetentionWork(db, { companyId: policy.companyId, sourceKind: "budget_alert" });
|
||||
if (thresholdType === "hard") await notifyApprovalWork(db, policy.companyId);
|
||||
const approval = thresholdType === "hard"
|
||||
? await db
|
||||
@@ -504,7 +531,8 @@ export function budgetServiceInTransaction(db: Db, publications: ActivityPublica
|
||||
return incident ? { incident, created: true } : null;
|
||||
}
|
||||
|
||||
async function resolveOpenSoftIncidents(policyId: string) {
|
||||
async function resolveOpenSoftIncidents(policyId: string, companyId: string) {
|
||||
await notifyAttentionRetentionWork(db, { companyId, sourceKind: "budget_alert" });
|
||||
await db
|
||||
.update(budgetIncidents)
|
||||
.set({
|
||||
@@ -523,10 +551,12 @@ export function budgetServiceInTransaction(db: Db, publications: ActivityPublica
|
||||
|
||||
async function resolveOpenIncidentsForPolicy(
|
||||
policyId: string,
|
||||
companyId: string,
|
||||
approvalStatus: "approved" | "rejected" | null,
|
||||
decidedByUserId: string | null,
|
||||
thresholdType?: BudgetThresholdType,
|
||||
) {
|
||||
await notifyAttentionRetentionWork(db, { companyId, sourceKind: "budget_alert" });
|
||||
const condition = and(eq(budgetIncidents.policyId, policyId), eq(budgetIncidents.status, "open"),
|
||||
thresholdType ? eq(budgetIncidents.thresholdType, thresholdType) : undefined);
|
||||
const openRows = await db
|
||||
@@ -560,7 +590,7 @@ export function budgetServiceInTransaction(db: Db, publications: ActivityPublica
|
||||
const approvalStatusById = new Map(approvalRows.map((row) => [row.id, row.status]));
|
||||
|
||||
return mapBudgetReads(rows, async (row) => {
|
||||
const scope = await resolveScopeRecord(db, row.scopeType as BudgetScopeType, row.scopeId);
|
||||
const scope = await resolveSummaryScopeRecord(db, row.companyId, row.scopeType as BudgetScopeType, row.scopeId);
|
||||
return {
|
||||
id: row.id,
|
||||
companyId: row.companyId,
|
||||
@@ -615,9 +645,9 @@ export function budgetServiceInTransaction(db: Db, publications: ActivityPublica
|
||||
if (policy.isActive && policy.hardStopEnabled && policy.amount > 0 && observed.recoveringRunCount > 0) recoveringPolicy ??= policy;
|
||||
// A closed UTC window, inactive policy, or budget raise releases only
|
||||
// budget-owned pauses. Manual pauses remain operator decisions.
|
||||
await resolveOpenIncidentsForPolicy(policy.id, "approved", "budget_service", "hard");
|
||||
await resolveOpenIncidentsForPolicy(policy.id, policy.companyId, "approved", "budget_service", "hard");
|
||||
if (!policy.isActive || !policy.notifyEnabled || compareCents(observed.totalExact, Math.ceil(policy.amount * policy.warnPercent / 100)) < 0) {
|
||||
await resolveOpenIncidentsForPolicy(policy.id, "approved", "budget_service", "soft");
|
||||
await resolveOpenIncidentsForPolicy(policy.id, policy.companyId, "approved", "budget_service", "soft");
|
||||
}
|
||||
}
|
||||
// Agent status has its own admission gate and error code. Preserve a
|
||||
@@ -674,7 +704,7 @@ export function budgetServiceInTransaction(db: Db, publications: ActivityPublica
|
||||
}
|
||||
|
||||
if (observedBlocks(policy, observed)) {
|
||||
await resolveOpenSoftIncidents(policy.id);
|
||||
await resolveOpenSoftIncidents(policy.id, policy.companyId);
|
||||
const hardIncident = await createIncidentIfNeeded(policy, "hard", observed);
|
||||
await pauseAndCancelScopeForBudget(policy);
|
||||
if (hardIncident?.created) {
|
||||
@@ -844,11 +874,11 @@ export function budgetServiceInTransaction(db: Db, publications: ActivityPublica
|
||||
const observed = await computeObservedSpend(db, row);
|
||||
if (!observedBlocks(row, observed)) {
|
||||
await resumeScopeFromBudget(row);
|
||||
await resolveOpenIncidentsForPolicy(row.id, "approved", actorUserId ?? "budget_service", "hard");
|
||||
await resolveOpenIncidentsForPolicy(row.id, row.companyId, "approved", actorUserId ?? "budget_service", "hard");
|
||||
if (row.notifyEnabled && compareCents(observed.totalExact, Math.ceil(row.amount * row.warnPercent / 100)) >= 0) {
|
||||
await createIncidentIfNeeded(row, "soft", observed);
|
||||
} else {
|
||||
await resolveOpenIncidentsForPolicy(row.id, null, null, "soft");
|
||||
await resolveOpenIncidentsForPolicy(row.id, row.companyId, null, null, "soft");
|
||||
}
|
||||
} else {
|
||||
const softThreshold = Math.ceil((row.amount * row.warnPercent) / 100);
|
||||
@@ -856,14 +886,14 @@ export function budgetServiceInTransaction(db: Db, publications: ActivityPublica
|
||||
await createIncidentIfNeeded(row, "soft", observed);
|
||||
}
|
||||
if (observedBlocks(row, observed)) {
|
||||
await resolveOpenSoftIncidents(row.id);
|
||||
await resolveOpenSoftIncidents(row.id, row.companyId);
|
||||
await createIncidentIfNeeded(row, "hard", observed);
|
||||
await pauseAndCancelScopeForBudget(row);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
await resumeScopeFromBudget(row);
|
||||
await resolveOpenIncidentsForPolicy(row.id, "approved", actorUserId ?? "budget_service");
|
||||
await resolveOpenIncidentsForPolicy(row.id, row.companyId, "approved", actorUserId ?? "budget_service");
|
||||
}
|
||||
|
||||
await recordActivity({
|
||||
@@ -925,6 +955,13 @@ export function budgetServiceInTransaction(db: Db, publications: ActivityPublica
|
||||
if (incident.companyId !== companyId) throw notFound("Budget incident not found");
|
||||
|
||||
const policy = await getPolicyRow(incident.policyId);
|
||||
if (policy.companyId !== companyId || policy.scopeType !== incident.scopeType || policy.scopeId !== incident.scopeId) {
|
||||
throw notFound("Budget scope not found");
|
||||
}
|
||||
const scope = input.action === "keep_paused"
|
||||
? await resolveSummaryScopeRecord(db, companyId, policy.scopeType as BudgetScopeType, policy.scopeId)
|
||||
: await resolveScopeRecord(db, policy.scopeType as BudgetScopeType, policy.scopeId);
|
||||
if (scope.companyId !== companyId) throw notFound("Budget scope not found");
|
||||
if (input.action === "raise_budget_and_resume") {
|
||||
const nextAmount = Math.max(0, Math.floor(input.amount ?? 0));
|
||||
const observed = await computeObservedSpend(db, policy);
|
||||
@@ -937,6 +974,7 @@ export function budgetServiceInTransaction(db: Db, publications: ActivityPublica
|
||||
}
|
||||
|
||||
const now = new Date();
|
||||
await notifyAttentionRetentionWork(db, { companyId, sourceKind: "budget_alert" });
|
||||
await notifyDeliveryWork(db, DELIVERY_QUEUES.budgetEnforcement, companyId);
|
||||
await db
|
||||
.update(budgetPolicies)
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { notifyDeferredExecutionWork, notifyTaskRetrySettlement } from "../deferred-execution-notifications.js";
|
||||
import { notifyNativeResponsePresentation } from "../native-response-presentation-notifications.js";
|
||||
import { notifyChatInboundWakeNotices } from "../chat-inbound-wake-notice-notifications.js";
|
||||
import { createFailedRetryNotices } from "./failed-retry-notices.js";
|
||||
@@ -1424,6 +1425,7 @@ export function createChatFailedRunRetry(db: Db, options: ChatFailedRunRetryOpti
|
||||
// Source authorization above checked every external receipt and its owner.
|
||||
// Reset only this certified unstarted batch under the task's admission lock.
|
||||
const skippedIds = source.taskRetryInput?.skippedActionIds ?? [];
|
||||
await notifyDeferredExecutionWork(tx, { companyId: input.companyId });
|
||||
await tx.update(agentWakeupRequests).set({ status: "deferred_issue_execution", runId: null, finishedAt: null,
|
||||
payload: sql`${agentWakeupRequests.payload} - 'coalescedIntoWakeupRequestId'`, updatedAt: new Date(),
|
||||
}).where(and(eq(agentWakeupRequests.companyId, input.companyId), eq(agentWakeupRequests.agentId, input.agentId),
|
||||
@@ -1784,6 +1786,9 @@ export function createChatFailedRunRetry(db: Db, options: ChatFailedRunRetryOpti
|
||||
const attemptCount = Number(candidate.result?.attemptCount ?? 0) + 1;
|
||||
const settle = async (status: string, result: Record<string, unknown>) => {
|
||||
const [current] = await db.transaction(async (tx) => {
|
||||
if (typeof claimed.payload.taskRetryIntentId === "string") await notifyTaskRetrySettlement(tx, {
|
||||
companyId: claimed.companyId, requestId: claimed.payload.taskRetryIntentId,
|
||||
});
|
||||
await notifyChatActionWork(tx, "failed_run_retry");
|
||||
if (status === "processed") await notifyNativeResponsePresentation(tx, {
|
||||
companyId: claimed.companyId, issueId: String(claimed.payload.issueId),
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { notifyDeferredExecutionWork } from "../deferred-execution-notifications.js";
|
||||
import { z } from "zod";
|
||||
import { and, asc, eq, gt, inArray, isNull, or, sql } from "drizzle-orm";
|
||||
import { agentWakeupRequests, chatActions, chatConversations, chatEndpoints, heartbeatRuns, issues, type Db } from "@paperclipai/db";
|
||||
@@ -91,6 +92,7 @@ export async function releaseUnadmittedTaskRetryInput(db: Db, input: { companyId
|
||||
)).for("update");
|
||||
if (receipt?.runId) return;
|
||||
await notifyChatInboundWakeNotices(tx, { companyId: input.companyId, issueId: issue.id });
|
||||
await notifyDeferredExecutionWork(tx, { companyId: input.companyId });
|
||||
await tx.update(agentWakeupRequests).set({ status: "deferred_issue_execution", finishedAt: null,
|
||||
payload: sql`${agentWakeupRequests.payload} - 'coalescedIntoWakeupRequestId'`, updatedAt: new Date(),
|
||||
}).where(and(eq(agentWakeupRequests.companyId, input.companyId), eq(agentWakeupRequests.status, "coalesced"),
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
import { notifyStatusCardIssueChange } from "../../status-card-notifications.js";
|
||||
import { notifyIssueRecovery } from "../../issue-recovery-notifications.js";
|
||||
import { createProviderEffects, type ProviderEffectTarget } from "../provider-effects.js";
|
||||
|
||||
import type { EndpointRow, RuntimeContext, InboundRuntimeContext } from "../runtime-types.js";
|
||||
@@ -78,10 +80,12 @@ export function createTelegramAttachmentFailure(db: Db, options: TelegramAttachm
|
||||
})(),
|
||||
);
|
||||
const effect = await db.transaction(async (tx) => {
|
||||
await notifyIssueRecovery(tx, { companyId: input.endpoint.companyId });
|
||||
// Replace the optimistic attachment placeholder with an explicit safe
|
||||
// Paperclip-visible failure. The message link stays authoritative for
|
||||
// exact provider redelivery, while no agent wakeup is enqueued for an
|
||||
// empty turn that contains no usable bytes.
|
||||
await notifyStatusCardIssueChange(tx, input.endpoint.companyId);
|
||||
await tx
|
||||
.update(issueComments)
|
||||
.set({ body: visibleFailure, updatedAt: new Date() })
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { notifyRunDispatch } from "./run-dispatch-notifications.js";
|
||||
import { notifyIssueReviewState } from "./issue-review-state-notifications.js";
|
||||
import { notifyNativeResponsePresentation } from "./native-response-presentation-notifications.js";
|
||||
import { notifyChatResponseAvailability } from "./chat-response-availability.js";
|
||||
@@ -11,6 +12,7 @@ import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notificatio
|
||||
export type ChatRunMilestoneScope = { companyId: string; issueId?: string; runId?: string };
|
||||
|
||||
export async function notifyChatRunMilestones(tx: object, scope: ChatRunMilestoneScope) {
|
||||
if (!scope.issueId && !scope.runId) await notifyRunDispatch(tx, scope);
|
||||
if (scope.issueId || scope.runId) {
|
||||
await notifyIssueReviewState(tx, { companyId: scope.companyId, issueId: scope.issueId,
|
||||
runId: scope.issueId ? undefined : scope.runId });
|
||||
|
||||
@@ -1,3 +1,6 @@
|
||||
import { notifyStatusCardIssueChange } from "./status-card-notifications.js";
|
||||
import { notifyDeferredExecutionEligibility } from "./deferred-execution-notifications.js";
|
||||
import { notifyRunDispatch } from "./run-dispatch-notifications.js";
|
||||
import { notifyChatInboundWakeNotices } from "./chat-inbound-wake-notice-notifications.js";
|
||||
import { notifyCompanyRetentionWork } from "./retention-work-notifications.js";
|
||||
import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js";
|
||||
@@ -332,10 +335,13 @@ export function companyService(db: Db, _budgetHooks: BudgetServiceHooks = {}) {
|
||||
const renamedPrefix = await resolveRenamedIssuePrefix(tx, id, companyPatch);
|
||||
|
||||
if (companyPatch.status !== undefined && companyPatch.status !== existing.status) {
|
||||
await notifyRunDispatch(tx, { companyId: id });
|
||||
await notifyDeferredExecutionEligibility(tx, { companyId: id });
|
||||
await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers);
|
||||
await notifyDeliveryWork(tx, DELIVERY_QUEUES.skillSourceSync);
|
||||
await notifyDeliveryWork(tx, DELIVERY_QUEUES.routineSchedules);
|
||||
await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors);
|
||||
await notifyStatusCardIssueChange(tx, id);
|
||||
await notifyCompanyRetentionWork(tx, id);
|
||||
await notifyChatInboundWakeNotices(tx, { companyId: id });
|
||||
await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentLifecycle, id);
|
||||
@@ -488,10 +494,13 @@ export function companyService(db: Db, _budgetHooks: BudgetServiceHooks = {}) {
|
||||
const wasAlreadyArchived = existing.status === "archived";
|
||||
|
||||
if (!wasAlreadyArchived) {
|
||||
await notifyRunDispatch(tx, { companyId: id });
|
||||
await notifyDeferredExecutionEligibility(tx, { companyId: id });
|
||||
await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers);
|
||||
await notifyDeliveryWork(tx, DELIVERY_QUEUES.skillSourceSync);
|
||||
await notifyDeliveryWork(tx, DELIVERY_QUEUES.routineSchedules);
|
||||
await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors);
|
||||
await notifyStatusCardIssueChange(tx, id);
|
||||
await notifyCompanyRetentionWork(tx, id);
|
||||
await notifyChatInboundWakeNotices(tx, { companyId: id });
|
||||
await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentLifecycle, id);
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
import { notifyStatusCardEvaluation } from "./status-card-notifications.js";
|
||||
import { notifyIssueRecovery } from "./issue-recovery-notifications.js";
|
||||
import { notifyUnsafeWorkspaceExportRecovery } from "./unsafe-workspace-export-notifications.js";
|
||||
import { notifyCancelledWorkspacePreservation } from "./cancelled-workspace-preservation-notifications.js";
|
||||
import { notifyInstructionCollection } from "./instruction-collection-notifications.js";
|
||||
@@ -71,12 +73,14 @@ export async function deleteCompany(db: Db, id: string) {
|
||||
const [existing] = await tx.select({ id: companies.id }).from(companies)
|
||||
.where(eq(companies.id, id)).for("no key update");
|
||||
if (!existing) return null;
|
||||
await notifyIssueRecovery(tx, { companyId: id });
|
||||
await notifyChatCompanyConfiguration(tx, id);
|
||||
await chatCompanyDeletion.deleteCompanyData(tx, id);
|
||||
await notifyDeliveryWork(tx, DELIVERY_QUEUES.skillSourceSync);
|
||||
await notifyDeliveryWork(tx, DELIVERY_QUEUES.skillPackages);
|
||||
await notifyDeliveryWork(tx, DELIVERY_QUEUES.routineSchedules);
|
||||
await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors);
|
||||
await notifyStatusCardEvaluation(tx, id);
|
||||
await notifyDeliveryWork(tx, DELIVERY_QUEUES.secretProposalExpiry, id);
|
||||
await notifyToolReviewChange(tx, id);
|
||||
await agentObservabilityService(tx as unknown as Db).removeForCompanyDeletion(id);
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
import { notifyAttentionRetentionWork } from "./retention-work-notifications.js";
|
||||
import { notifyIssueRecovery } from "./issue-recovery-notifications.js";
|
||||
import { createAgentLifecycle } from "./agent-lifecycle.js";
|
||||
import { agentAppearanceSchema, agentHarnessType, paperclipRunnerProfileForHarness, agentRunner, type AgentRunnerChoice } from "@paperclipai/shared";
|
||||
import { resolveNewAgentRunnerForCompany } from "./agent-runner-selection.js";
|
||||
@@ -6465,7 +6467,10 @@ export function companyPortabilityService(db: Db, storage?: StorageService) {
|
||||
}
|
||||
if (relationRows.length > 0) {
|
||||
const relationCompanyId = targetCompany.id;
|
||||
await db
|
||||
await db.transaction(async tx => {
|
||||
await notifyAttentionRetentionWork(tx, { companyId: relationCompanyId });
|
||||
await notifyIssueRecovery(tx, { companyId: relationCompanyId });
|
||||
await tx
|
||||
.insert(issueRelations)
|
||||
.values(relationRows.map((row) => ({
|
||||
companyId: relationCompanyId,
|
||||
@@ -6476,6 +6481,7 @@ export function companyPortabilityService(db: Db, storage?: StorageService) {
|
||||
createdByUserId: actorUserId ?? null,
|
||||
})))
|
||||
.onConflictDoNothing();
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -207,14 +207,20 @@ function priorityRank(priority: string | null | undefined) {
|
||||
return index >= 0 ? index : ISSUE_PRIORITIES.length;
|
||||
}
|
||||
|
||||
function updatedWithinStart(value: string | undefined, now = new Date()): Date | null {
|
||||
/** Duration used by saved search membership and its deadline consumers. */
|
||||
export function companySearchUpdatedWithinMs(value: string | undefined): number | null {
|
||||
if (!value) return null;
|
||||
const match = /^(\d+)(h|d|w|m)$/.exec(value);
|
||||
if (!match) return null;
|
||||
const amount = Number.parseInt(match[1]!, 10);
|
||||
const unit = match[2];
|
||||
const hours = unit === "h" ? amount : unit === "d" ? amount * 24 : unit === "w" ? amount * 24 * 7 : amount * 24 * 30;
|
||||
return new Date(now.getTime() - hours * 60 * 60 * 1000);
|
||||
return hours * 60 * 60 * 1000;
|
||||
}
|
||||
|
||||
function updatedWithinStart(value: string | undefined, now = new Date()): Date | null {
|
||||
const duration = companySearchUpdatedWithinMs(value);
|
||||
return duration === null ? null : new Date(now.getTime() - duration);
|
||||
}
|
||||
|
||||
function issueOnlyFiltersActive(query: CompanySearchQuery) {
|
||||
@@ -542,7 +548,7 @@ export function companySearchService(db: Db) {
|
||||
search: async (
|
||||
companyId: string,
|
||||
query: CompanySearchQuery,
|
||||
options?: { issueReadCondition?: SQL<boolean>; projectReadCondition?: SQL<boolean> },
|
||||
options?: { issueReadCondition?: SQL<boolean>; projectReadCondition?: SQL<boolean>; onNextMembershipChange?: (at: Date | null) => void },
|
||||
): Promise<CompanySearchResponse> => {
|
||||
const taskSearch = parseTaskSearch(query.q);
|
||||
const normalizedQuery = taskSearch.normalizedQuery;
|
||||
@@ -553,6 +559,7 @@ export function companySearchService(db: Db) {
|
||||
const limit = query.limit;
|
||||
const offset = query.offset;
|
||||
if (!hasSearchText && !issueOnlyFiltersActive(query)) {
|
||||
options?.onNextMembershipChange?.(null);
|
||||
return {
|
||||
query: query.q,
|
||||
normalizedQuery,
|
||||
@@ -721,6 +728,13 @@ export function companySearchService(db: Db) {
|
||||
}
|
||||
}
|
||||
|
||||
const membershipWindow = companySearchUpdatedWithinMs(query.updatedWithin);
|
||||
if (options?.onNextMembershipChange && membershipWindow !== null && scopeIncludesIssues(scope)) {
|
||||
// Includes rows outside the visible page: their expiry can move an
|
||||
// offset/priority-ranked result into the watched set without a write.
|
||||
branches.push(sql`SELECT 'membership-expiry' AS kind, min(m.updated_at)::text AS value,
|
||||
0::int AS count ${countTail} FROM matched m ${branchWhere([scopeCond, ...facetsAll])}`);
|
||||
}
|
||||
const resultRows = await db.execute(sql`
|
||||
${taskSearchCtes(companyId, taskSearch, scope !== "issues", and(...issueFilters), options?.issueReadCondition)}
|
||||
${sql.join(branches, sql` UNION ALL `)}
|
||||
@@ -732,6 +746,7 @@ export function companySearchService(db: Db) {
|
||||
totals: { current: 0, unfiltered: 0, omit: {} },
|
||||
};
|
||||
const issueRowsRaw: IssueSearchRow[] = [];
|
||||
let nextMembershipChange: Date | null = null;
|
||||
for (const row of resultRows) {
|
||||
if (row.kind === "result") {
|
||||
issueRowsRaw.push({
|
||||
@@ -756,6 +771,9 @@ export function companySearchService(db: Db) {
|
||||
});
|
||||
continue;
|
||||
}
|
||||
if (row.kind === "membership-expiry" && row.value && membershipWindow !== null) {
|
||||
nextMembershipChange = new Date(new Date(row.value).getTime() + membershipWindow + 1);
|
||||
}
|
||||
const count = Number(row.count ?? 0);
|
||||
if (row.kind === "type:issue") aggregates.typeCounts.issue = count;
|
||||
else if (row.kind === "type:comment") aggregates.typeCounts.comment = count;
|
||||
@@ -776,6 +794,7 @@ export function companySearchService(db: Db) {
|
||||
aggregates.totals.omit[row.kind.slice("total:omit:".length) as CompanySearchIssueFilterKey] = count;
|
||||
}
|
||||
}
|
||||
options?.onNextMembershipChange?.(nextMembershipChange);
|
||||
return { rows: await enrichIssueSnippets(issueRowsRaw), aggregates };
|
||||
}
|
||||
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { notifyStatusCardIssueChange } from "./status-card-notifications.js";
|
||||
import { skillExecutionContent } from "./skill-execution-content.js";
|
||||
import { notifySkillVersionRetention } from "./skill-version-retention-notifications.js";
|
||||
import { managedSkillContent, type SkillContentPublication } from "./managed-skill-content.js";
|
||||
@@ -7340,6 +7341,7 @@ export function companySkillService(db: Db, options: { actor?: AuthorizationActo
|
||||
for (const row of rows) {
|
||||
await db.transaction(async (tx) => {
|
||||
await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers);
|
||||
await notifyStatusCardIssueChange(tx, companyId);
|
||||
await tx
|
||||
.update(issues)
|
||||
.set({ hiddenAt: now, updatedAt: now })
|
||||
|
||||
@@ -26,7 +26,7 @@ export function registerDeadlineWork(
|
||||
pausePassiveMaintenance: true,
|
||||
onSettled: hints => { requested++; task.onSettled?.(hints); },
|
||||
run: async signal => {
|
||||
if (!task.canRun()) { requested++; next = null; return; }
|
||||
if (!task.canRun()) { requested++; return; }
|
||||
if (Date.now() < earliest) { next = earliest; return; }
|
||||
if (Date.now() < coalesceUntil && (next === null || next > Date.now())) {
|
||||
next = next === null ? coalesceUntil : Math.min(next, coalesceUntil);
|
||||
@@ -39,8 +39,6 @@ export function registerDeadlineWork(
|
||||
if (signal.aborted || !task.canRun()) { next = null; return; }
|
||||
const deadline = await task.next();
|
||||
|
||||
|
||||
|
||||
if (!task.canRun()) { next = null; return; }
|
||||
earliest = deadline === null || deadline > Date.now()
|
||||
? (task.minimumSpacingMs ? Date.now() + task.minimumSpacingMs : 0)
|
||||
@@ -49,8 +47,13 @@ export function registerDeadlineWork(
|
||||
coalesceUntil = Date.now() + (task.notificationCoalesceMs ?? 0);
|
||||
reconciled = generation;
|
||||
},
|
||||
hasPending: async () => requested !== reconciled,
|
||||
nextRunAt: () => next,
|
||||
// A settled producer must release its admission hold during drain. Keep
|
||||
// the unreconciled generation and an in-memory resume wake, without asking
|
||||
// the domain to probe its database while dispatch is closed.
|
||||
hasPending: async () => task.canRun() && requested !== reconciled,
|
||||
nextRunAt: () => !task.canRun() && requested !== reconciled
|
||||
? Date.now() + task.retryMs
|
||||
: next,
|
||||
});
|
||||
return { ...worker, wake: () => { requested++; worker.wake(); } };
|
||||
}
|
||||
@@ -23,7 +23,7 @@ export function decisionExpiryService(db: Db, deliverContinuation: (id: string)
|
||||
agentId: row.originAgentId, runId: row.originRunId, action: "decision.expired", entityType: "decision",
|
||||
entityId: row.id, details: { expiredReason: reason } }, publications);
|
||||
return row;
|
||||
});
|
||||
}, { companyId: decision.companyId, sourceId: decision.id });
|
||||
if (!updated) return 0;
|
||||
for (const publication of publications) publishActivity(publication);
|
||||
await deliverContinuation(updated.id);
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { isProspectiveBlockedTransition } from "./routable-blocked.js";
|
||||
import { notifyRetentionWork } from "./retention-work-notifications.js";
|
||||
import { decisionArchiveDeliveryService, type ArchiveNotificationBatch } from "./decision-archive-delivery.js";
|
||||
import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js";
|
||||
@@ -15,7 +16,6 @@ import {
|
||||
issueRecoveryActions,
|
||||
issueThreadInteractions,
|
||||
issues,
|
||||
joinRequests,
|
||||
} from "@paperclipai/db";
|
||||
import type {
|
||||
AttentionArchiveManifestEntry,
|
||||
@@ -145,7 +145,7 @@ export function decisionRetentionService(
|
||||
const existing = new Set(rows.map(row => sourceKey(row.sourceKind, row.sourceId)));
|
||||
const missing = unique.filter(item => !existing.has(itemSourceKey(item)));
|
||||
if (missing.length) {
|
||||
for (const kind of new Set(missing.map(item => item.sourceKind))) await notifyRetentionWork(tx, kind);
|
||||
for (const item of missing) await notifyRetentionWork(tx, item.sourceKind, { companyId, sourceId: item.subject.id });
|
||||
await tx.insert(decisionRetention).values(missing.map(item => ({
|
||||
companyId, sourceKind: item.sourceKind, sourceId: item.subject.id, sourceActivityAt: new Date(item.activityAt),
|
||||
}))).onConflictDoNothing({ target: [decisionRetention.companyId, decisionRetention.sourceKind, decisionRetention.sourceId] });
|
||||
@@ -155,7 +155,7 @@ export function decisionRetentionService(
|
||||
for (const item of unique) {
|
||||
const key = itemSourceKey(item), row = byKey.get(key), activityAt = new Date(item.activityAt);
|
||||
if (!row || row.sourceActivityAt.getTime() === activityAt.getTime()) continue;
|
||||
await notifyRetentionWork(tx, item.sourceKind);
|
||||
await notifyRetentionWork(tx, item.sourceKind, { companyId, sourceId: item.subject.id });
|
||||
const [updated] = await tx.update(decisionRetention).set({
|
||||
sourceActivityAt: activityAt, version: sql`${decisionRetention.version} + 1`, updatedAt: new Date(),
|
||||
}).where(and(eq(decisionRetention.id, row.id), ne(decisionRetention.sourceActivityAt, activityAt))).returning();
|
||||
@@ -169,9 +169,9 @@ export function decisionRetentionService(
|
||||
return sourceKey(item.sourceKind, item.subject.id);
|
||||
}
|
||||
|
||||
async function writeRetention<T>(kind: AttentionSourceKind, write: (tx: Db) => Promise<T>) {
|
||||
async function writeRetention<T>(companyId: string, kind: AttentionSourceKind, sourceId: string, write: (tx: Db) => Promise<T>) {
|
||||
return db.transaction(async tx => {
|
||||
await notifyRetentionWork(tx, kind);
|
||||
await notifyRetentionWork(tx, kind, { companyId, sourceId });
|
||||
return write(tx as unknown as Db);
|
||||
});
|
||||
}
|
||||
@@ -185,13 +185,28 @@ export function decisionRetentionService(
|
||||
}
|
||||
|
||||
async function materializeSourceState(database: Db, input: { companyId: string; sourceKind: AttentionSourceKind; sourceId: string }) {
|
||||
const table = input.sourceKind === "approval" ? approvals : input.sourceKind === "join_request" ? joinRequests : null;
|
||||
if (!table) return;
|
||||
const [source] = await database.select({ updatedAt: table.updatedAt }).from(table)
|
||||
.where(and(eq(table.companyId, input.companyId), eq(table.id, input.sourceId))).for("update");
|
||||
const tables: Record<AttentionSourceKind, string> = {
|
||||
approval: "approvals", join_request: "join_requests", decision: "decisions",
|
||||
issue_thread_interaction: "issue_thread_interactions", recovery_action: "issue_recovery_actions",
|
||||
failed_run: "heartbeat_runs", productivity_review: "issues", blocker_attention: "issues", review: "issues",
|
||||
budget_alert: "budget_incidents", agent_error_alert: "agents",
|
||||
};
|
||||
if (input.sourceKind === "blocker_attention") {
|
||||
const [source] = await database.select().from(issues).where(and(eq(issues.companyId, input.companyId), eq(issues.id, input.sourceId))).for("update");
|
||||
if (!source) throw notFound("Attention source not found");
|
||||
const activityAt = source.unblockDescriptor?.owner === "board" && isProspectiveBlockedTransition(source)
|
||||
? source.blockedTransitionAt : source.updatedAt;
|
||||
await decisionRetentionService(database).syncItems(input.companyId, [{
|
||||
sourceKind: input.sourceKind, subject: { id: input.sourceId }, activityAt: activityAt.toISOString(),
|
||||
}]);
|
||||
return;
|
||||
}
|
||||
const activity = input.sourceKind === "failed_run" ? sql`coalesce(finished_at, updated_at, created_at)` : sql`updated_at`;
|
||||
const [source] = await database.execute(sql`select ${activity} as "activityAt" from ${sql.identifier(tables[input.sourceKind])}
|
||||
where company_id = ${input.companyId}::uuid and id = ${input.sourceId}::uuid for update`) as unknown as Array<{ activityAt: Date | string }>;
|
||||
if (!source) throw notFound("Attention source not found");
|
||||
await decisionRetentionService(database).syncItems(input.companyId, [{
|
||||
sourceKind: input.sourceKind, subject: { id: input.sourceId }, activityAt: source.updatedAt.toISOString(),
|
||||
sourceKind: input.sourceKind, subject: { id: input.sourceId }, activityAt: new Date(source.activityAt).toISOString(),
|
||||
}]);
|
||||
}
|
||||
|
||||
@@ -206,7 +221,7 @@ export function decisionRetentionService(
|
||||
if (!(await canReadDecisionSource(db, input.authActor, input.companyId, input.sourceKind, input.sourceId))) {
|
||||
throw notFound("Attention source not found");
|
||||
}
|
||||
const updated = await writeRetention(input.sourceKind, async tx => {
|
||||
const updated = await writeRetention(input.companyId, input.sourceKind, input.sourceId, async tx => {
|
||||
await materializeSourceState(tx, input);
|
||||
return tx.update(decisionRetention).set({
|
||||
keep: input.keep,
|
||||
@@ -247,7 +262,7 @@ export function decisionRetentionService(
|
||||
throw notFound("Attention source not found");
|
||||
}
|
||||
const now = new Date();
|
||||
const updated = await writeRetention(input.sourceKind, async tx => {
|
||||
const updated = await writeRetention(input.companyId, input.sourceKind, input.sourceId, async tx => {
|
||||
await materializeSourceState(tx, input);
|
||||
return tx.update(decisionRetention).set(input.archived ? {
|
||||
archivedAt: now,
|
||||
@@ -324,7 +339,7 @@ export function decisionRetentionService(
|
||||
}
|
||||
const now = new Date();
|
||||
for (const entry of input.manifest) {
|
||||
await notifyRetentionWork(txDb, entry.sourceKind as AttentionSourceKind);
|
||||
await notifyRetentionWork(txDb, entry.sourceKind as AttentionSourceKind, { companyId: input.companyId, sourceId: entry.sourceId });
|
||||
const archived = await txDb.update(decisionRetention).set({
|
||||
archivedAt: now,
|
||||
archivedReason: "accepted_proposal",
|
||||
@@ -368,7 +383,7 @@ export function decisionRetentionService(
|
||||
await db.transaction(async (tx) => {
|
||||
const txDb = tx as unknown as Db;
|
||||
for (const item of candidates) {
|
||||
await notifyRetentionWork(txDb, item.sourceKind);
|
||||
await notifyRetentionWork(txDb, item.sourceKind, { companyId: input.companyId, sourceId: item.subject.id });
|
||||
const updated = await txDb.update(decisionRetention).set({
|
||||
archivedAt: now,
|
||||
archivedReason: "idle_ttl",
|
||||
|
||||
@@ -1,18 +1,23 @@
|
||||
import { notifyAttentionRetentionWork } from "./retention-work-notifications.js";
|
||||
import type { Db } from "@paperclipai/db";
|
||||
import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js";
|
||||
|
||||
export function notifyDecisionCreated(transaction: object) {
|
||||
type DecisionWorkScope = { companyId: string; sourceId?: string };
|
||||
|
||||
export async function notifyDecisionCreated(transaction: object, scope: DecisionWorkScope) {
|
||||
await notifyAttentionRetentionWork(transaction, { ...scope, sourceKind: "decision" });
|
||||
return notifyDeliveryWork(transaction, DELIVERY_QUEUES.decisionExpiry);
|
||||
}
|
||||
|
||||
export async function notifyDecisionChange(transaction: object) {
|
||||
export async function notifyDecisionChange(transaction: object, scope: DecisionWorkScope) {
|
||||
await notifyAttentionRetentionWork(transaction, { ...scope, sourceKind: "decision" });
|
||||
await notifyDeliveryWork(transaction, DELIVERY_QUEUES.decisionContinuation);
|
||||
await notifyDeliveryWork(transaction, DELIVERY_QUEUES.decisionExpiry);
|
||||
}
|
||||
|
||||
export function writeDecisionChange<T>(db: Db, write: (transaction: Db) => Promise<T>): Promise<T> {
|
||||
export function writeDecisionChange<T>(db: Db, write: (transaction: Db) => Promise<T>, scope: DecisionWorkScope): Promise<T> {
|
||||
return db.transaction(async transaction => {
|
||||
await notifyDecisionChange(transaction);
|
||||
await notifyDecisionChange(transaction, scope);
|
||||
await notifyDeliveryWork(transaction, DELIVERY_QUEUES.decisionEffectRecovery);
|
||||
return write(transaction as unknown as Db);
|
||||
});
|
||||
|
||||
@@ -239,7 +239,7 @@ export function decisionService(db: Db, options: DecisionServiceOptions) {
|
||||
...(input.additionalTargetSnapshots ?? {}),
|
||||
};
|
||||
const id = randomUUID();
|
||||
await notifyDecisionCreated(dbOrTx);
|
||||
await notifyDecisionCreated(dbOrTx, { companyId: input.companyId, sourceId: id });
|
||||
await notifyDecisionTargetWork(dbOrTx, `decision:${id}`);
|
||||
const [created] = await dbOrTx.insert(decisions).values({ id, companyId: input.companyId, bundleId: input.bundleId ?? null,
|
||||
originAgentId: input.agentId, originIssueId: provenance.issueId, originRunId: input.runId, ruleKey: input.ruleKey ?? null,
|
||||
@@ -581,7 +581,8 @@ export function decisionService(db: Db, options: DecisionServiceOptions) {
|
||||
return attempt;
|
||||
}
|
||||
|
||||
async function settleInvalidEffectOutcome(id: string) {
|
||||
async function settleInvalidEffectOutcome(decision: typeof decisions.$inferSelect) {
|
||||
const id = decision.id;
|
||||
const publications: ActivityPublication[] = [];
|
||||
await writeDecisionChange(db, async tx => {
|
||||
const [current] = await tx.select().from(decisions).where(eq(decisions.id, id)).for("update");
|
||||
@@ -601,14 +602,14 @@ export function decisionService(db: Db, options: DecisionServiceOptions) {
|
||||
action: "decision.recovery_failed", entityType: "decision", entityId: id,
|
||||
details: { reason, executionStatus, chosenOptionId: current.chosenOptionId, decidedByUserId: current.decidedByUserId },
|
||||
}, publications);
|
||||
});
|
||||
}, { companyId: decision.companyId, sourceId: id });
|
||||
for (const publication of publications) publishActivity(publication);
|
||||
return outcome(id);
|
||||
}
|
||||
|
||||
async function executeDecisionEffects(decision: typeof decisions.$inferSelect, userActor: AuthorizationActor) {
|
||||
const option = decision.options.find((item) => item.id === decision.chosenOptionId);
|
||||
if (!option || !decision.decidedByUserId) return settleInvalidEffectOutcome(decision.id);
|
||||
if (!option || !decision.decidedByUserId) return settleInvalidEffectOutcome(decision);
|
||||
const run = await db.select({ responsibleUserId: heartbeatRuns.responsibleUserId }).from(heartbeatRuns).where(eq(heartbeatRuns.id, decision.originRunId)).then((rows) => rows[0] ?? null);
|
||||
const metadata = decision.metadata as Record<string, unknown>;
|
||||
if (metadata.kind === "attention_archive_proposal") {
|
||||
@@ -618,7 +619,7 @@ export function decisionService(db: Db, options: DecisionServiceOptions) {
|
||||
targetSnapshots: decision.targetSnapshots as Record<string, Snapshot>,
|
||||
}), decision.signedSpec)) {
|
||||
await writeDecisionChange(db, async tx => tx.update(decisions).set({ executionStatus: "failed", updatedAt: new Date(), metadata: { ...metadata, archiveProposalError: "invalid_signature", ...(decision.continuationPolicy === "wake_origin_agent" ? { continuationPending: true } : {}) } })
|
||||
.where(and(eq(decisions.id, decision.id), eq(decisions.executionStatus, "running"))));
|
||||
.where(and(eq(decisions.id, decision.id), eq(decisions.executionStatus, "running"))), { companyId: decision.companyId, sourceId: decision.id });
|
||||
return outcome(decision.id);
|
||||
}
|
||||
if (option.id === "archive") {
|
||||
@@ -629,7 +630,7 @@ export function decisionService(db: Db, options: DecisionServiceOptions) {
|
||||
const manifestHash = manifest ? hashAttentionArchiveManifest(manifest) : null;
|
||||
if (!manifest || manifestHash !== metadata.manifestHash) {
|
||||
await writeDecisionChange(db, async tx => tx.update(decisions).set({ executionStatus: "failed", updatedAt: new Date(), metadata: { ...metadata, archiveProposalError: "manifest_mismatch", ...(decision.continuationPolicy === "wake_origin_agent" ? { continuationPending: true } : {}) } })
|
||||
.where(and(eq(decisions.id, decision.id), eq(decisions.executionStatus, "running"))));
|
||||
.where(and(eq(decisions.id, decision.id), eq(decisions.executionStatus, "running"))), { companyId: decision.companyId, sourceId: decision.id });
|
||||
return outcome(decision.id);
|
||||
}
|
||||
const originActor: AuthorizationActor = {
|
||||
@@ -651,7 +652,7 @@ export function decisionService(db: Db, options: DecisionServiceOptions) {
|
||||
} catch (error) {
|
||||
const message = error instanceof Error ? error.message : "archive_proposal_failed";
|
||||
await writeDecisionChange(db, async tx => tx.update(decisions).set({ executionStatus: "failed", updatedAt: new Date(), metadata: { ...metadata, archiveProposalError: message, ...(decision.continuationPolicy === "wake_origin_agent" ? { continuationPending: true } : {}) } })
|
||||
.where(and(eq(decisions.id, decision.id), eq(decisions.executionStatus, "running"))));
|
||||
.where(and(eq(decisions.id, decision.id), eq(decisions.executionStatus, "running"))), { companyId: decision.companyId, sourceId: decision.id });
|
||||
return outcome(decision.id);
|
||||
}
|
||||
}
|
||||
@@ -659,7 +660,7 @@ export function decisionService(db: Db, options: DecisionServiceOptions) {
|
||||
executionStatus: "succeeded",
|
||||
updatedAt: new Date(),
|
||||
metadata: { ...metadata, ...(decision.continuationPolicy === "wake_origin_agent" ? { continuationPending: true } : {}) },
|
||||
}).where(and(eq(decisions.id, decision.id), eq(decisions.executionStatus, "running"))));
|
||||
}).where(and(eq(decisions.id, decision.id), eq(decisions.executionStatus, "running"))), { companyId: decision.companyId, sourceId: decision.id });
|
||||
return outcome(decision.id);
|
||||
}
|
||||
for (let index = 0; index < option.effects.length; index += 1) await executeEffect(decision, option.effects[index]!, index, userActor, decision.decidedByUserId, run?.responsibleUserId ?? null);
|
||||
@@ -667,7 +668,7 @@ export function decisionService(db: Db, options: DecisionServiceOptions) {
|
||||
const successful = rows.filter((row) => row.status === "executed").length;
|
||||
const status = rows.every((row) => row.status === "executed") ? "succeeded" : successful ? "partial" : "failed";
|
||||
await writeDecisionChange(db, async tx => tx.update(decisions).set({ executionStatus: status, updatedAt: new Date(), metadata: { ...decision.metadata,
|
||||
...(decision.continuationPolicy === "wake_origin_agent" ? { continuationPending: true } : {}) } }).where(and(eq(decisions.id, decision.id), eq(decisions.executionStatus, "running"))));
|
||||
...(decision.continuationPolicy === "wake_origin_agent" ? { continuationPending: true } : {}) } }).where(and(eq(decisions.id, decision.id), eq(decisions.executionStatus, "running"))), { companyId: decision.companyId, sourceId: decision.id });
|
||||
return outcome(decision.id);
|
||||
}
|
||||
|
||||
@@ -706,7 +707,7 @@ export function decisionService(db: Db, options: DecisionServiceOptions) {
|
||||
decideIdempotencyKey: input.idempotencyKey ?? null,
|
||||
...(current.continuationPolicy === "wake_origin_agent" ? { continuationPending: true } : {}),
|
||||
...(input.dismissed ? { dismissed: true, dismissReason: input.dismissReason ?? null } : {}) } })
|
||||
.where(and(eq(decisions.id, current.id), eq(decisions.status, "open"))).returning());
|
||||
.where(and(eq(decisions.id, current.id), eq(decisions.status, "open"))).returning(), { companyId: current.companyId, sourceId: current.id });
|
||||
if (!claimed) throw conflict("decision_already_resolved", { code: "decision_already_resolved" });
|
||||
const run = await db.select({ responsibleUserId: heartbeatRuns.responsibleUserId }).from(heartbeatRuns).where(eq(heartbeatRuns.id, claimed.originRunId)).then((rows) => rows[0] ?? null);
|
||||
await logActivity(db, { companyId: claimed.companyId, actorType: "system", actorId: "decision-executor", agentId: claimed.originAgentId, runId: claimed.originRunId,
|
||||
@@ -724,7 +725,7 @@ export function decisionService(db: Db, options: DecisionServiceOptions) {
|
||||
return (await get(id))!;
|
||||
}
|
||||
const [updated] = await writeDecisionChange(db, async tx => tx.update(decisions).set({ status: "cancelled", updatedAt: new Date(), metadata: { ...current.metadata,
|
||||
...(current.continuationPolicy === "wake_origin_agent" ? { continuationPending: true } : {}) } }).where(and(eq(decisions.id, id), eq(decisions.status, "open"))).returning());
|
||||
...(current.continuationPolicy === "wake_origin_agent" ? { continuationPending: true } : {}) } }).where(and(eq(decisions.id, id), eq(decisions.status, "open"))).returning(), { companyId: current.companyId, sourceId: id });
|
||||
if (!updated) throw conflict("decision_already_resolved", { code: "decision_already_resolved" });
|
||||
await logActivity(db, { companyId: updated.companyId, actorType: actor.actorType, actorId: actor.actorId, runId: actor.runId, action: "decision.cancelled", entityType: "decision", entityId: id });
|
||||
await continuations.deliver(updated.id);
|
||||
@@ -738,7 +739,7 @@ export function decisionService(db: Db, options: DecisionServiceOptions) {
|
||||
if (empty) return decide({ id, optionId: empty.id, decidedByUserId: userId, userActor, dismissed: true, dismissReason: reason });
|
||||
const [updated] = await writeDecisionChange(db, async tx => tx.update(decisions).set({ status: "decided", executionStatus: "succeeded", chosenOptionId: "dismissed", decidedByUserId: userId,
|
||||
decidedAt: new Date(), updatedAt: new Date(), metadata: { ...current.metadata, dismissed: true, dismissReason: reason ?? null,
|
||||
...(current.continuationPolicy === "wake_origin_agent" ? { continuationPending: true } : {}) } }).where(and(eq(decisions.id, id), eq(decisions.status, "open"))).returning());
|
||||
...(current.continuationPolicy === "wake_origin_agent" ? { continuationPending: true } : {}) } }).where(and(eq(decisions.id, id), eq(decisions.status, "open"))).returning(), { companyId: current.companyId, sourceId: id });
|
||||
if (!updated) throw conflict("decision_already_resolved", { code: "decision_already_resolved" });
|
||||
await logActivity(db, { companyId: updated.companyId, actorType: "system", actorId: "decision-executor", agentId: updated.originAgentId,
|
||||
runId: updated.originRunId, responsibleUserIdOverride: userId, action: "decision.dismissed", entityType: "decision", entityId: updated.id,
|
||||
|
||||
@@ -0,0 +1,34 @@
|
||||
import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js";
|
||||
|
||||
export type DeferredExecutionScope = { companyId: string; requestId?: string };
|
||||
export type DeferredReceiptRecoveryOptions = {
|
||||
companyId?: string;
|
||||
requestId?: string;
|
||||
onError?: (error: unknown) => void;
|
||||
};
|
||||
|
||||
function hint(scope: DeferredExecutionScope) {
|
||||
return JSON.stringify({ companyId: scope.companyId, requestId: scope.requestId });
|
||||
}
|
||||
|
||||
export async function notifyDeferredExecutionWork(tx: object, scope: DeferredExecutionScope) {
|
||||
await notifyDeliveryWork(tx, DELIVERY_QUEUES.deferredExecutionReceipts, hint(scope));
|
||||
}
|
||||
|
||||
export async function notifyNativeStatusWakeWork(tx: object, scope: DeferredExecutionScope) {
|
||||
await notifyDeliveryWork(tx, DELIVERY_QUEUES.nativeStatusWakeIntents, hint(scope));
|
||||
}
|
||||
|
||||
export async function notifyTaskRetrySettlement(tx: object, scope: DeferredExecutionScope) {
|
||||
await notifyDeliveryWork(tx, DELIVERY_QUEUES.taskRetryReceiptSettlement, hint(scope));
|
||||
}
|
||||
|
||||
export async function notifyDeferredExecutionEligibility(tx: object, scope?: { companyId: string }) {
|
||||
if (scope) {
|
||||
await notifyDeferredExecutionWork(tx, scope);
|
||||
await notifyNativeStatusWakeWork(tx, scope);
|
||||
} else {
|
||||
await notifyDeliveryWork(tx, DELIVERY_QUEUES.deferredExecutionReceipts, "*");
|
||||
await notifyDeliveryWork(tx, DELIVERY_QUEUES.nativeStatusWakeIntents, "*");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
import { and, eq, sql } from "drizzle-orm";
|
||||
import { agentWakeupRequests, type Db } from "@paperclipai/db";
|
||||
import { notifyDeferredExecutionWork } from "./deferred-execution-notifications.js";
|
||||
|
||||
export async function writeDeferredExecutionWait(db: Db, scope: { companyId: string; requestId: string },
|
||||
condition: Record<string, unknown>, options: { replace?: boolean } = {}) {
|
||||
return db.transaction(async tx => {
|
||||
const [current] = await tx.select({ payload: agentWakeupRequests.payload }).from(agentWakeupRequests)
|
||||
.where(and(eq(agentWakeupRequests.companyId, scope.companyId), eq(agentWakeupRequests.id, scope.requestId),
|
||||
eq(agentWakeupRequests.status, "deferred_issue_execution"))).for("update");
|
||||
if (!current) return;
|
||||
const previous = current.payload?.executionWait;
|
||||
const previousReason = previous && typeof previous === "object" && !Array.isArray(previous)
|
||||
? (previous as Record<string, unknown>).reason : undefined;
|
||||
if (!previous || previousReason !== condition.reason) await notifyDeferredExecutionWork(tx, scope);
|
||||
await tx.update(agentWakeupRequests).set({
|
||||
payload: sql`jsonb_set(coalesce(${agentWakeupRequests.payload}, '{}'::jsonb), '{executionWait}',
|
||||
${options.replace ? sql`'{}'::jsonb` : sql`coalesce(${agentWakeupRequests.payload}->'executionWait', '{}'::jsonb)`}
|
||||
|| ${JSON.stringify(condition)}::jsonb)`,
|
||||
updatedAt: new Date(),
|
||||
}).where(and(eq(agentWakeupRequests.companyId, scope.companyId), eq(agentWakeupRequests.id, scope.requestId),
|
||||
eq(agentWakeupRequests.status, "deferred_issue_execution")));
|
||||
});
|
||||
}
|
||||
@@ -2,6 +2,18 @@ import { signalDatabaseWork, subscribeDatabaseWork } from "@paperclipai/db";
|
||||
|
||||
/** Topics for existing durable queues; register intent before their writes. */
|
||||
export const DELIVERY_QUEUES = {
|
||||
deferredExecutionReceipts: "deferred-execution-receipts",
|
||||
taskRetryReceiptSettlement: "task-retry-receipt-settlement",
|
||||
nativeStatusWakeIntents: "native-status-wake-intents",
|
||||
orphanBlockers: "orphan-blockers",
|
||||
activeRecoveryActions: "active-recovery-actions",
|
||||
strandedAssignments: "stranded-assignments",
|
||||
resolvedDependencies: "resolved-dependencies",
|
||||
taskWatchdogs: "task-watchdogs",
|
||||
staleIssueLocks: "stale-issue-locks",
|
||||
scheduledRetryDispatch: "scheduled-retry-dispatch",
|
||||
queuedRunDispatch: "queued-run-dispatch",
|
||||
statusCardEvaluation: "status-card-evaluation",
|
||||
runEvidenceMigration: "run-evidence-migration",
|
||||
providerTraceRetention: "provider-trace-retention",
|
||||
nativeRunnerEvidenceRetention: "native-runner-evidence-retention",
|
||||
@@ -35,6 +47,7 @@ export const DELIVERY_QUEUES = {
|
||||
executionDisposition: "execution-disposition",
|
||||
environmentCleanup: "environment-cleanup",
|
||||
approvalQueueSeeding: "approval-queue-seeding",
|
||||
attentionRetention: "attention-retention",
|
||||
approvalRetention: "approval-retention",
|
||||
joinRequestRetention: "join-request-retention",
|
||||
decisionTargets: "decision-targets",
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
import { notifyStatusCardIssueChange } from "./status-card-notifications.js";
|
||||
import { notifyIssueRecovery } from "./issue-recovery-notifications.js";
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { and, asc, desc, eq, sql } from "drizzle-orm";
|
||||
import type { Db } from "@paperclipai/db";
|
||||
@@ -238,6 +240,7 @@ export function documentService(db: Db) {
|
||||
for (let attempt = 0; attempt < maxAttempts; attempt += 1) {
|
||||
try {
|
||||
return await db.transaction(async (tx) => {
|
||||
await notifyStatusCardIssueChange(tx, issue.companyId);
|
||||
const now = new Date();
|
||||
const existing = await tx
|
||||
.select({
|
||||
@@ -275,6 +278,7 @@ export function documentService(db: Db) {
|
||||
.where(eq(issueDocuments.issueId, issue.id));
|
||||
const fallbackKey = nextAvailableDocumentKey(key, issueDocumentKeys.map((row) => row.key));
|
||||
|
||||
await notifyIssueRecovery(tx, { companyId: issue.companyId });
|
||||
const [document] = await tx
|
||||
.insert(documents)
|
||||
.values({
|
||||
@@ -376,6 +380,7 @@ export function documentService(db: Db) {
|
||||
});
|
||||
}
|
||||
|
||||
await notifyIssueRecovery(tx, { companyId: issue.companyId });
|
||||
const nextRevisionNumber = existing.latestRevisionNumber + 1;
|
||||
const [revision] = await tx
|
||||
.insert(documentRevisions)
|
||||
@@ -438,6 +443,7 @@ export function documentService(db: Db) {
|
||||
throw conflict("Document does not exist yet", { key });
|
||||
}
|
||||
|
||||
await notifyIssueRecovery(tx, { companyId: issue.companyId });
|
||||
const [document] = await tx
|
||||
.insert(documents)
|
||||
.values({
|
||||
@@ -592,6 +598,10 @@ export function documentService(db: Db) {
|
||||
});
|
||||
}
|
||||
await db.transaction(async (tx) => {
|
||||
for (const companyId of new Set(rows.map(row => row.companyId))) {
|
||||
await notifyStatusCardIssueChange(tx, companyId);
|
||||
await notifyIssueRecovery(tx, { companyId });
|
||||
}
|
||||
await insertRowsInChunks(tx, documents, documentRows);
|
||||
await insertRowsInChunks(tx, documentRevisions, revisionRows);
|
||||
await insertRowsInChunks(tx, issueDocuments, linkRows);
|
||||
@@ -644,6 +654,7 @@ export function documentService(db: Db) {
|
||||
});
|
||||
}
|
||||
|
||||
await notifyIssueRecovery(tx, { companyId: existing.companyId });
|
||||
const now = new Date();
|
||||
const nextRevisionNumber = existing.latestRevisionNumber + 1;
|
||||
const [restoredRevision] = await tx
|
||||
@@ -662,6 +673,7 @@ export function documentService(db: Db) {
|
||||
})
|
||||
.returning();
|
||||
|
||||
await notifyStatusCardIssueChange(tx, existing.companyId);
|
||||
await tx
|
||||
.update(documents)
|
||||
.set({
|
||||
@@ -722,6 +734,7 @@ export function documentService(db: Db) {
|
||||
};
|
||||
}
|
||||
|
||||
await notifyIssueRecovery(tx, { companyId: existing.companyId });
|
||||
const now = new Date();
|
||||
await tx
|
||||
.update(documents)
|
||||
@@ -769,6 +782,7 @@ export function documentService(db: Db) {
|
||||
};
|
||||
}
|
||||
|
||||
await notifyIssueRecovery(tx, { companyId: existing.companyId });
|
||||
const now = new Date();
|
||||
await tx
|
||||
.update(documents)
|
||||
@@ -817,7 +831,9 @@ export function documentService(db: Db) {
|
||||
});
|
||||
}
|
||||
|
||||
await notifyIssueRecovery(tx, { companyId: existing.companyId });
|
||||
await tx.delete(issueDocuments).where(eq(issueDocuments.documentId, existing.id));
|
||||
await notifyStatusCardIssueChange(tx, existing.companyId);
|
||||
await tx.delete(documents).where(eq(documents.id, existing.id));
|
||||
|
||||
return {
|
||||
|
||||
@@ -15,6 +15,7 @@ import { hasStopOnlyCleanup, prepareSandboxStopAndRetain, readStopOnlyCleanup, s
|
||||
import { JsonRpcCallError, readEnvironmentAcquisitionDiagnostic, readEnvironmentCreationCleanupError } from "@paperclipai/plugin-sdk";
|
||||
import { preserveEnvironmentSyncOutErrorDiagnostic } from "./environment-sync-out-error.js";
|
||||
import { remoteTerminationReceipt } from "./remote-execution-termination.js";
|
||||
import { inspectCloudSandboxCleanupAbsence } from "./sandbox-absence-proof.js";
|
||||
import { hasNativeWorkspaceExportResume, releaseCompletedNativeWorkspaceExportRetention } from "./native-runtime/native-workspace-export-resume.js";
|
||||
import { createHash, randomUUID } from "node:crypto";
|
||||
import { and, eq, inArray, ne, or, sql } from "drizzle-orm";
|
||||
@@ -4021,7 +4022,12 @@ export function environmentRuntimeService(
|
||||
const lease = await environmentsSvc.getLeaseById(leaseId);
|
||||
if (!lease || lease.status !== "failed" || lease.leasePolicy !== "ephemeral" || !lease.providerLeaseId) return undefined;
|
||||
const environment = lease.environmentId ? await environmentsSvc.getById(lease.environmentId) : null;
|
||||
return await getDriver(getLeaseDriverKey(lease, environment))?.inspectLeaseResource?.({ environment, lease });
|
||||
// Legacy Cloud leases can lack a credential version that permits a
|
||||
// provider-local observation. Cloud's manager verifies the historical
|
||||
// organization and signs only exact absence; no replacement key or
|
||||
// cleanup label is treated as proof. Explicit provider answers win.
|
||||
return await getDriver(getLeaseDriverKey(lease, environment))?.inspectLeaseResource?.({ environment, lease })
|
||||
?? await inspectCloudSandboxCleanupAbsence(lease);
|
||||
},
|
||||
|
||||
async retryPendingSandboxTeardown(input: {
|
||||
|
||||
@@ -1,4 +1,8 @@
|
||||
import { notifyRoutineExecutionStatus } from "./routine-execution-status-notifications.js";
|
||||
import { notifyAttentionRetentionWork } from "./retention-work-notifications.js";
|
||||
import { notifyStatusCardIssueChange } from "./status-card-notifications.js";
|
||||
import { notifyDeferredExecutionEligibility } from "./deferred-execution-notifications.js";
|
||||
import { notifyRunDispatch } from "./run-dispatch-notifications.js";
|
||||
import type { Db } from "@paperclipai/db";
|
||||
import { DELIVERY_QUEUES, notifyDeliveryWork, type DeliveryQueue } from "./delivery-work-notifications.js";
|
||||
|
||||
@@ -6,6 +10,12 @@ export type ExecutionWorkScope = { companyId: string; agentId?: string; runId?:
|
||||
|
||||
export async function notifyExecutionRunWork(transaction: object, scope?: ExecutionWorkScope) {
|
||||
if (scope) await notifyRoutineExecutionStatus(transaction, scope.issueId, scope.companyId);
|
||||
if (scope) {
|
||||
await notifyAttentionRetentionWork(transaction, scope);
|
||||
await notifyStatusCardIssueChange(transaction, scope.companyId);
|
||||
await notifyRunDispatch(transaction, scope);
|
||||
await notifyDeferredExecutionEligibility(transaction, scope);
|
||||
}
|
||||
for (const queue of [DELIVERY_QUEUES.executionStatus, DELIVERY_QUEUES.executionFinalization,
|
||||
DELIVERY_QUEUES.executionReplacement, DELIVERY_QUEUES.executionDisposition, DELIVERY_QUEUES.issueMonitors]) {
|
||||
await notifyDeliveryWork(transaction, queue);
|
||||
@@ -20,6 +30,10 @@ export function writeExecutionControl<T>(
|
||||
): Promise<T> {
|
||||
return db.transaction(async transaction => {
|
||||
if (scope) await notifyRoutineExecutionStatus(transaction, scope.issueId, scope.companyId);
|
||||
if (scope) await notifyAttentionRetentionWork(transaction, scope);
|
||||
if (scope) await notifyStatusCardIssueChange(transaction, scope.companyId);
|
||||
await notifyRunDispatch(transaction, scope);
|
||||
if (scope) await notifyDeferredExecutionEligibility(transaction, scope);
|
||||
await notifyDeliveryWork(transaction, DELIVERY_QUEUES.issueMonitors);
|
||||
for (const topic of typeof queue === "string" ? [queue] : queue) await notifyDeliveryWork(transaction, topic);
|
||||
return write(transaction as unknown as Db);
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { notifyStatusCardIssueChange } from "./status-card-notifications.js";
|
||||
import { effectiveProjectWorkspacePolicy } from "./project-workspace-provider.js";
|
||||
import { inspectPluginWorkspace, removePluginWorkspace, restorePluginWorkspace, type WorkspaceWorkers } from "./plugin-workspaces.js";
|
||||
import { acquireExecutionWorkspaceLifecycleLock, withExecutionWorkspaceLifecycleLock } from "./execution-workspace-lifecycle-lock.js";
|
||||
@@ -1590,6 +1591,7 @@ export function executionWorkspaceService(db: Db, opts: ExecutionWorkspaceServic
|
||||
await validateSelection(input, tx);
|
||||
const intent: TaskWorkspaceIntent = { version: 1, request: { key: input.requestKey, expectedBindingRevision: input.expectedBindingRevision }, selection: input.selection, source: input.source ?? "explicit" };
|
||||
const nextRevision = task.workspaceBindingRevision + 1;
|
||||
await notifyStatusCardIssueChange(tx, task.companyId);
|
||||
await tx.update(issues).set({ workspaceBindingRevision: nextRevision,
|
||||
workspacePendingSelection: { version: 1, requestKey: input.requestKey,
|
||||
expectedBindingRevision: nextRevision, intent }, updatedAt: new Date() }).where(eq(issues.id, task.id));
|
||||
@@ -1633,6 +1635,7 @@ export function executionWorkspaceService(db: Db, opts: ExecutionWorkspaceServic
|
||||
}
|
||||
}
|
||||
}
|
||||
await notifyStatusCardIssueChange(tx, task.companyId);
|
||||
await tx.update(issues).set({ ...patch, assigneeAdapterOverrides, workspaceSelection: pending.intent, workspacePendingSelection: null,
|
||||
workspaceBindingRevision: task.workspaceBindingRevision + 1, updatedAt: new Date() }).where(eq(issues.id, task.id));
|
||||
return true;
|
||||
@@ -1651,6 +1654,7 @@ export function executionWorkspaceService(db: Db, opts: ExecutionWorkspaceServic
|
||||
const pendingAfterFirstBinding = !task.executionWorkspaceId && pending?.expectedBindingRevision === task.workspaceBindingRevision
|
||||
? { ...pending, expectedBindingRevision: nextRevision }
|
||||
: pending;
|
||||
await notifyStatusCardIssueChange(tx, companyId);
|
||||
const [bound] = await tx.update(issues).set({ ...patch, executionWorkspaceId: workspaceId,
|
||||
workspaceBindingRevision: nextRevision, workspacePendingSelection: pendingAfterFirstBinding, updatedAt: new Date() })
|
||||
.where(eq(issues.id, issueId)).returning();
|
||||
@@ -2526,13 +2530,16 @@ export function executionWorkspaceService(db: Db, opts: ExecutionWorkspaceServic
|
||||
forceWorktreeRemoval: false,
|
||||
});
|
||||
if (cleanup.cleaned && workspace.mode === "shared_workspace") {
|
||||
await db
|
||||
await db.transaction(async tx => {
|
||||
await notifyStatusCardIssueChange(tx, workspace.companyId);
|
||||
await tx
|
||||
.update(issues)
|
||||
.set({ executionWorkspaceId: null, updatedAt: now() })
|
||||
.where(and(
|
||||
eq(issues.companyId, workspace.companyId),
|
||||
eq(issues.executionWorkspaceId, workspace.id),
|
||||
));
|
||||
});
|
||||
}
|
||||
const cleanupReason = [ISSUE_TERMINAL_WORKSPACE_CLEANUP_REASON, ...cleanup.warnings].join(" | ");
|
||||
if (!cleanup.cleaned || cleanup.warnings.length > 0) {
|
||||
|
||||
@@ -1,7 +1,9 @@
|
||||
import { notifyIssueRecovery } from "./issue-recovery-notifications.js";
|
||||
import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js";
|
||||
import type { ExternalConversationSettlementScope } from "./slack-conversation-lifecycle.js";
|
||||
|
||||
export function notifyExternalConversationSettlement(tx: object, scope?: ExternalConversationSettlementScope) {
|
||||
export async function notifyExternalConversationSettlement(tx: object, scope?: ExternalConversationSettlementScope) {
|
||||
await notifyIssueRecovery(tx, scope);
|
||||
return notifyDeliveryWork(tx, DELIVERY_QUEUES.externalConversationSettlement,
|
||||
scope ? JSON.stringify({ companyId: scope.companyId, issueId: scope.issueId }) : "*");
|
||||
}
|
||||
@@ -1,3 +1,4 @@
|
||||
import { notifyAttentionRetentionWork } from "../retention-work-notifications.js";
|
||||
import { and, eq, inArray, isNotNull, or, sql } from "drizzle-orm";
|
||||
import { dotRunnerAssignments, environmentLeases, heartbeatRuns, issueRecoveryActions, issues, nativeRunFinalizations, type Db } from "@paperclipai/db";
|
||||
import { authorizeExternalBotOperator } from "../external-bot-authority.js";
|
||||
@@ -74,6 +75,7 @@ export async function inspectGrokWorkspaceObligations(db: Db, companyId: string,
|
||||
return { ...rest, externalWorkspaceInspection: inspection };
|
||||
};
|
||||
await tx.update(heartbeatRuns).set({ resultJson: retireFileObligation(run.resultJson!) }).where(eq(heartbeatRuns.id, run.id));
|
||||
if (actions.length) await notifyAttentionRetentionWork(tx, { companyId, sourceKind: "recovery_action" });
|
||||
for (const action of actions) {
|
||||
await tx.update(issueRecoveryActions).set({ evidence: retireFileObligation(action.evidence), updatedAt: new Date(),
|
||||
nextAction: "Inspect Grok and verify its connection before sending a new task message. The old work will not be replayed.",
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { notifyAttentionRetentionWork } from "./retention-work-notifications.js";
|
||||
import { readNativeEventEvidenceOutcome } from "./native-runtime/evidence-classifier.js";
|
||||
import { notifyNativeStatusReassessment } from "./native-status-reassessment-notifications.js";
|
||||
import { isNativeResponseEvidence, notifyNativeResponsePresentation } from "./native-response-presentation-notifications.js";
|
||||
@@ -9,6 +10,8 @@ import type { Db } from "@paperclipai/db";
|
||||
import { heartbeatRunEvents, heartbeatRuns } from "@paperclipai/db";
|
||||
import { nativeSha256 } from "./native-runtime/canonical.js";
|
||||
|
||||
export const RETRY_EXHAUSTION_MESSAGE_PREFIX = "Bounded retry exhausted";
|
||||
|
||||
export interface AppendHeartbeatRunEventInput {
|
||||
companyId: string;
|
||||
runId: string;
|
||||
@@ -74,10 +77,11 @@ export async function appendHeartbeatRunEvent(
|
||||
db: Db,
|
||||
input: AppendHeartbeatRunEventInput,
|
||||
): Promise<AppendHeartbeatRunEventResult> {
|
||||
const projectsRetention = input.eventType === "lifecycle" && Boolean(input.message?.startsWith(RETRY_EXHAUSTION_MESSAGE_PREFIX));
|
||||
const projectsReassessment = readNativeEventEvidenceOutcome(input.payload?.prpEvent) !== null;
|
||||
const projectsProgress = isSafeNativeChatProgressEventType(input.eventType);
|
||||
const projectsResponse = isNativeResponseEvidence(input.eventType, input.payload?.prpEvent);
|
||||
if (!input.nativeSource && !input.retryExhaustion && !projectsProgress && !projectsResponse && !projectsReassessment) {
|
||||
if (!input.nativeSource && !input.retryExhaustion && !projectsProgress && !projectsResponse && !projectsReassessment && !projectsRetention) {
|
||||
const allocated = db.$with("allocated_run_event", {}).as(sql`
|
||||
update ${heartbeatRuns}
|
||||
set next_event_seq = ${heartbeatRuns.nextEventSeq} + 1,
|
||||
@@ -129,7 +133,7 @@ export async function appendHeartbeatRunEvent(
|
||||
eq(heartbeatRunEvents.runId, input.runId),
|
||||
eq(heartbeatRunEvents.agentId, input.agentId),
|
||||
eq(heartbeatRunEvents.eventType, "lifecycle"),
|
||||
sql`${heartbeatRunEvents.message} like 'Bounded retry exhausted%'`,
|
||||
sql`${heartbeatRunEvents.message} like ${`${RETRY_EXHAUSTION_MESSAGE_PREFIX}%`}`,
|
||||
sql`${heartbeatRunEvents.payload} @> ${JSON.stringify(input.retryExhaustion)}::jsonb`,
|
||||
))
|
||||
.limit(1)
|
||||
@@ -187,6 +191,7 @@ export async function appendHeartbeatRunEvent(
|
||||
if (projectsResponse && run.runtimeMode === "native") {
|
||||
await notifyNativeResponsePresentation(tx, { companyId: run.companyId, runId: input.runId });
|
||||
}
|
||||
if (projectsRetention) await notifyAttentionRetentionWork(tx, { companyId: input.companyId, sourceKind: "failed_run" });
|
||||
const seq = await allocateHeartbeatRunEventSeq(
|
||||
tx as unknown as Db,
|
||||
input.runId,
|
||||
|
||||
@@ -1,3 +1,7 @@
|
||||
import { registerDeferredReceiptWork } from "./heartbeat/deferred-receipt-work.js";
|
||||
import { reconcileTaskChatRetryReceipts } from "./task-retry.js";
|
||||
import { registerIssueRecoveryWork } from "./recovery/work.js";
|
||||
import { registerRunDispatchWork } from "./heartbeat/dispatch-work.js";
|
||||
import { externalConversationSettlementWork } from "./external-conversation-settlement-work.js";
|
||||
import { nativeRecoveryNoticeWork } from "./native-runtime/native-recovery-notice-work.js";
|
||||
import { nativeRunProjectionWork } from "./native-runtime/native-run-projection-work.js";
|
||||
@@ -999,6 +1003,7 @@ export function heartbeatService(
|
||||
finalizeAiConnectionBusyDeferral,
|
||||
finalizeWorkspaceBusyDeferral,
|
||||
promoteDueScheduledRetries,
|
||||
nextScheduledRetryAt,
|
||||
retryScheduledRetryNow,
|
||||
scheduleBoundedRetry,
|
||||
} = createHeartbeatRetries(db, {
|
||||
@@ -1333,8 +1338,8 @@ export function heartbeatService(
|
||||
return heartbeatQueue.dispatchPendingNativeStatusWakeups(...args);
|
||||
}
|
||||
|
||||
function resumeQueuedRunsAfterDrain() {
|
||||
const promise = heartbeatQueue.resumeQueuedRunsAfterDrain();
|
||||
function resumeQueuedRunsAfterDrain(...args: Parameters<typeof heartbeatQueue.resumeQueuedRunsAfterDrain>) {
|
||||
const promise = heartbeatQueue.resumeQueuedRunsAfterDrain(...args);
|
||||
activeWakeupPromises.add(promise);
|
||||
void promise.then(
|
||||
() => { activeWakeupPromises.delete(promise); },
|
||||
@@ -1609,15 +1614,15 @@ export function heartbeatService(
|
||||
void reportRunFailure(db, updated, failureReport);
|
||||
}
|
||||
|
||||
async function reconcileStrandedAssignedIssues(opts?: { skipExternalConversationSettlement?: boolean }) {
|
||||
async function reconcileStrandedAssignedIssues(opts?: Parameters<typeof recovery.reconcileStrandedAssignedIssues>[0]) {
|
||||
return recovery.reconcileStrandedAssignedIssues({
|
||||
skipExternalConversationSettlement: opts?.skipExternalConversationSettlement,
|
||||
...opts,
|
||||
issueCreatedAtGte: await getWorktreeExecutionCutoff(),
|
||||
});
|
||||
}
|
||||
|
||||
async function sweepStaleIssueLocks() {
|
||||
return recovery.sweepStaleIssueLocks();
|
||||
async function sweepStaleIssueLocks(opts?: Parameters<typeof recovery.sweepStaleIssueLocks>[0]) {
|
||||
return recovery.sweepStaleIssueLocks(opts);
|
||||
}
|
||||
|
||||
function issueIdFromRunContext(contextSnapshot: unknown) {
|
||||
@@ -1647,10 +1652,7 @@ export function heartbeatService(
|
||||
});
|
||||
}
|
||||
|
||||
async function reconcileTaskWatchdogs(opts?: {
|
||||
companyId?: string | null;
|
||||
runId?: string | null;
|
||||
}) {
|
||||
async function reconcileTaskWatchdogs(opts?: Parameters<typeof taskWatchdogs.reconcileTaskWatchdogs>[0]) {
|
||||
return taskWatchdogs.reconcileTaskWatchdogs({
|
||||
...opts,
|
||||
issueCreatedAtGte: await getWorktreeExecutionCutoff(),
|
||||
@@ -1675,10 +1677,7 @@ export function heartbeatService(
|
||||
return recovery.buildRunOutputSilence(run, now);
|
||||
}
|
||||
|
||||
async function reconcileResolvedDependencyWakes(opts?: {
|
||||
runId?: string | null;
|
||||
companyId?: string | null;
|
||||
}) {
|
||||
async function reconcileResolvedDependencyWakes(opts?: Parameters<typeof recovery.reconcileResolvedDependencyWakeBackstop>[0]) {
|
||||
return recovery.reconcileResolvedDependencyWakeBackstop(opts);
|
||||
}
|
||||
|
||||
@@ -2007,8 +2006,11 @@ export function heartbeatService(
|
||||
const dispatchIssueId = readNonEmptyString(parseObject(run.contextSnapshot).issueId);
|
||||
const resumingAdmittedConversationTurn = !!runOptions.nativeLeaseOwner
|
||||
&& typeof run.contextSnapshot?.conversationSessionGeneration === "number";
|
||||
if (dispatchIssueId && isConversation(await getIssueExecutionContext(run.companyId, dispatchIssueId))
|
||||
&& !resumingAdmittedConversationTurn && !(await instanceSettings.getExperimental()).enableAgentChat) {
|
||||
const conversationIdentity = dispatchIssueId && !resumingAdmittedConversationTurn
|
||||
? await db.select({ conversationAgentId: issues.conversationAgentId, conversationUserId: issues.conversationUserId })
|
||||
.from(issues).where(and(eq(issues.companyId, run.companyId), eq(issues.id, dispatchIssueId))).then(rows => rows[0] ?? null)
|
||||
: null;
|
||||
if (isConversation(conversationIdentity) && !(await instanceSettings.getExperimental()).enableAgentChat) {
|
||||
await setRunStatus(run.id, "cancelled", { finishedAt: new Date(), error: "Agent Chat is disabled", errorCode: "agent_chat_disabled" });
|
||||
await setWakeupStatus(run.wakeupRequestId, "cancelled", { finishedAt: new Date() });
|
||||
await releaseIssueExecutionAndPromote((await getRun(run.id))!, { suppressImmediateRecovery: true });
|
||||
@@ -2204,44 +2206,6 @@ export function heartbeatService(
|
||||
context[PAPERCLIP_EXTERNAL_CHAT_EXECUTION_BOUND_KEY] = true;
|
||||
}
|
||||
const wakeCommentId = deriveCommentId(context, null);
|
||||
const wakeCommentContext =
|
||||
issueContext && wakeCommentId
|
||||
? await db
|
||||
.select({
|
||||
id: issueComments.id,
|
||||
body: issueComments.body,
|
||||
authorType: issueComments.authorType,
|
||||
authorAgentId: issueComments.authorAgentId,
|
||||
authorUserId: issueComments.authorUserId,
|
||||
presentation: issueComments.presentation,
|
||||
metadata: issueComments.metadata,
|
||||
deletedAt: issueComments.deletedAt,
|
||||
deletedByType: issueComments.deletedByType,
|
||||
deletedByAgentId: issueComments.deletedByAgentId,
|
||||
deletedByUserId: issueComments.deletedByUserId,
|
||||
deletedByRunId: issueComments.deletedByRunId,
|
||||
sourceTrust: issueComments.sourceTrust,
|
||||
})
|
||||
.from(issueComments)
|
||||
.where(
|
||||
and(
|
||||
eq(issueComments.id, wakeCommentId),
|
||||
eq(issueComments.issueId, issueContext.id),
|
||||
eq(issueComments.companyId, agent.companyId),
|
||||
),
|
||||
)
|
||||
.then((rows) => {
|
||||
const row = rows[0] ?? null;
|
||||
return row?.deletedAt
|
||||
? {
|
||||
...row,
|
||||
body: "",
|
||||
presentation: null,
|
||||
metadata: null,
|
||||
}
|
||||
: row;
|
||||
})
|
||||
: null;
|
||||
let issueAssigneeOverrides =
|
||||
issueContext && issueContext.assigneeAgentId === agent.id
|
||||
? parseIssueAssigneeAdapterOverrides(
|
||||
@@ -2610,7 +2574,7 @@ export function heartbeatService(
|
||||
continuationSummary, executionContinuation, paperclipWakePayload, safeWakeComments, safeWakeCommentContext, pinnedSkillTest,
|
||||
} = await timePreparation("task_context", () => prepareHeartbeatTaskContext(db, {
|
||||
run, agent,
|
||||
task: { issueRef, issueContext, wakeCommentContext, context, taskSession },
|
||||
task: { issueRef, issueContext, context, taskSession },
|
||||
policy: { trustPreset, experimentalInstanceSettings, recovering: Boolean(runOptions.nativeRestartRecovery || runOptions.nativeLeaseOwner) },
|
||||
services: { issuesSvc, getPinnedSkillTestContext },
|
||||
}));
|
||||
@@ -4246,6 +4210,30 @@ export function heartbeatService(
|
||||
computeTaskDrain,
|
||||
applyTaskDrain,
|
||||
|
||||
registerDeferredReceiptWork: (coordinator: Parameters<typeof registerDeferredReceiptWork>[1], canRun: () => boolean) =>
|
||||
registerDeferredReceiptWork(db, coordinator, {
|
||||
resume: scope => heartbeatQueue.resumeDeferredExecutionReceipts(scope),
|
||||
settleTaskChat: scope => reconcileTaskChatRetryReceipts(db, { ...scope, settleOnly: true }),
|
||||
dispatchNative: dispatchPendingNativeStatusWakeups,
|
||||
schedulingSuppressed: async () => (await getSchedulingSuppression(true)).suppressed,
|
||||
}, () => canRun() && readTaskDrain(new Date()) === null),
|
||||
registerIssueStateRecoveryWork: (coordinator: Parameters<typeof registerIssueRecoveryWork>[1], canRun: () => boolean) =>
|
||||
registerIssueRecoveryWork(db, coordinator, {
|
||||
stranded: scope => reconcileStrandedAssignedIssues({ ...scope, skipExternalConversationSettlement: true, skipRelatedRecovery: true }),
|
||||
orphanBlockers: recovery.reconcileUnassignedBlockingIssues,
|
||||
recoveryActions: recovery.reconcileActiveRecoveryActions,
|
||||
dependencies: reconcileResolvedDependencyWakes,
|
||||
locks: sweepStaleIssueLocks,
|
||||
watchdogs: reconcileTaskWatchdogs,
|
||||
schedulingSuppressed: async () => (await getSchedulingSuppression(true)).suppressed,
|
||||
}, () => canRun() && readTaskDrain(new Date()) === null),
|
||||
registerRunDispatchWork: (coordinator: Parameters<typeof registerRunDispatchWork>[0], canRun: () => boolean) =>
|
||||
registerRunDispatchWork(coordinator, {
|
||||
promoteDueScheduledRetries,
|
||||
nextScheduledRetryAt,
|
||||
resumeQueuedRunsAfterDrain,
|
||||
schedulingSuppressed: async () => (await getSchedulingSuppression(true)).suppressed,
|
||||
}, () => canRun() && readTaskDrain(new Date()) === null),
|
||||
promoteDueScheduledRetries,
|
||||
retryScheduledRetryNow,
|
||||
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { notifyAttentionRetentionWork } from "../retention-work-notifications.js";
|
||||
import { DELIVERY_QUEUES, notifyDeliveryWork } from "../delivery-work-notifications.js";
|
||||
import { agents, heartbeatRuns, type Db } from "@paperclipai/db";
|
||||
import { and, asc, desc, eq, lte, sql } from "drizzle-orm";
|
||||
@@ -91,6 +92,7 @@ export function createHeartbeatAgentStatus(db: Db) {
|
||||
)).limit(1);
|
||||
const nextStatus = running ? "running" : settledStatus(outcome, options?.keepIdleOnFailure);
|
||||
const now = new Date();
|
||||
await notifyAttentionRetentionWork(tx, { companyId: existing.companyId });
|
||||
await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers);
|
||||
const [updated] = await tx.update(agents).set({
|
||||
status: nextStatus,
|
||||
@@ -140,6 +142,7 @@ export function createHeartbeatAgentStatus(db: Db) {
|
||||
eq(heartbeatRuns.agentId, candidate.id)))
|
||||
.orderBy(desc(heartbeatRuns.finishedAt), desc(heartbeatRuns.createdAt), desc(heartbeatRuns.id)).limit(1);
|
||||
const nextStatus = settledStatus(lastRun?.status);
|
||||
await notifyAttentionRetentionWork(tx, { companyId: candidate.companyId });
|
||||
const [agent] = await tx.update(agents).set({ status: nextStatus,
|
||||
errorReason: nextStatus === "error" ? errorReason(lastRun?.error) : null, updatedAt: now,
|
||||
}).where(and(eq(agents.id, candidate.id), eq(agents.companyId, candidate.companyId),
|
||||
|
||||
@@ -0,0 +1,118 @@
|
||||
import { and, asc, eq, gt, inArray, isNull, or, sql } from "drizzle-orm";
|
||||
import { agentWakeupRequests, type Db } from "@paperclipai/db";
|
||||
import type { createDeliveryWorkCoordinator } from "../delivery-work-coordinator.js";
|
||||
import { DELIVERY_QUEUES } from "../delivery-work-notifications.js";
|
||||
import type { DeferredExecutionScope, DeferredReceiptRecoveryOptions } from "../deferred-execution-notifications.js";
|
||||
import { registerScopedDeadlineWork } from "../scoped-deadline-work.js";
|
||||
|
||||
type Entry = { id: string; companyId: string; dueAt: number };
|
||||
type NativeIntentOptions = DeferredExecutionScope & {
|
||||
onDeferredDeadline(at: number): void;
|
||||
onError(error: unknown): void;
|
||||
};
|
||||
|
||||
export function registerDeferredReceiptWork(
|
||||
db: Db,
|
||||
coordinator: ReturnType<typeof createDeliveryWorkCoordinator>,
|
||||
owner: {
|
||||
resume(scope: DeferredReceiptRecoveryOptions): Promise<unknown>;
|
||||
settleTaskChat(scope: DeferredExecutionScope): Promise<unknown>;
|
||||
dispatchNative(scope: NativeIntentOptions): Promise<unknown>;
|
||||
schedulingSuppressed(): Promise<boolean>;
|
||||
},
|
||||
canRun: () => boolean,
|
||||
) {
|
||||
function register(kind: "deferred" | "task_chat" | "native") {
|
||||
const cursors = new Map<string, string>();
|
||||
let enabled = false;
|
||||
const queue = kind === "deferred" ? DELIVERY_QUEUES.deferredExecutionReceipts
|
||||
: kind === "task_chat" ? DELIVERY_QUEUES.taskRetryReceiptSettlement : DELIVERY_QUEUES.nativeStatusWakeIntents;
|
||||
return registerScopedDeadlineWork<Entry, DeferredExecutionScope>(coordinator, {
|
||||
queue,
|
||||
reconstructionIntervalMs: 30_000,
|
||||
beforePass: async () => { enabled = !(await owner.schedulingSuppressed()); },
|
||||
decodeScope: hint => hint === "*" ? undefined : JSON.parse(hint),
|
||||
matches: (entry, scope) => !scope || entry.companyId === scope.companyId && (!scope.requestId || entry.id === scope.requestId),
|
||||
load: async scope => {
|
||||
const key = scope ? JSON.stringify({ companyId: scope.companyId, requestId: scope.requestId }) : "*";
|
||||
if (!enabled) { cursors.delete(key); return []; }
|
||||
const cursor = cursors.get(key);
|
||||
const rows = await db.select({ id: agentWakeupRequests.id, companyId: agentWakeupRequests.companyId,
|
||||
status: agentWakeupRequests.status, updatedAt: agentWakeupRequests.updatedAt,
|
||||
claimedAt: agentWakeupRequests.claimedAt, error: agentWakeupRequests.error,
|
||||
delayed: sql<boolean>`${agentWakeupRequests.payload}->'queuedCommentInterrupt' is not null
|
||||
or ${agentWakeupRequests.payload}->'executionWait' is not null`,
|
||||
interruptedLegacy: sql<boolean>`exists (select 1 from heartbeat_runs r
|
||||
where r.company_id = ${agentWakeupRequests.companyId} and r.agent_id = ${agentWakeupRequests.agentId}
|
||||
and r.runtime_mode = 'legacy' and r.status = 'cancelled'
|
||||
and r.result_json->>'queuedCommentInterruptQueueId' = ${agentWakeupRequests.id}::text)`,
|
||||
}).from(agentWakeupRequests).where(and(
|
||||
scope ? eq(agentWakeupRequests.companyId, scope.companyId) : undefined,
|
||||
scope?.requestId ? eq(agentWakeupRequests.id, scope.requestId) : undefined,
|
||||
cursor ? gt(agentWakeupRequests.id, cursor) : undefined,
|
||||
kind === "native" ? and(
|
||||
eq(agentWakeupRequests.requestedByActorType, "system"),
|
||||
eq(agentWakeupRequests.requestedByActorId, "native-status-committer"),
|
||||
inArray(agentWakeupRequests.status, ["queued", "claimed"]), isNull(agentWakeupRequests.runId),
|
||||
) : and(eq(agentWakeupRequests.status, "deferred_issue_execution"),
|
||||
kind === "task_chat" ? and(eq(agentWakeupRequests.reason, "task_retry"),
|
||||
sql`${agentWakeupRequests.payload}#>>'{taskRetry,chatActionId}' is not null`) : or(
|
||||
eq(agentWakeupRequests.reason, "task_retry"),
|
||||
sql`${agentWakeupRequests.payload}->'queuedCommentInterrupt' is not null`,
|
||||
sql`${agentWakeupRequests.payload}->'executionWait' is not null`,
|
||||
sql`jsonb_typeof(${agentWakeupRequests.payload}#>'{_paperclipWakeContext,wakeCommentIds}') = 'array'
|
||||
and ${agentWakeupRequests.payload}#>'{_paperclipWakeContext,wakeCommentIds}' <> '[]'::jsonb`,
|
||||
sql`${agentWakeupRequests.payload}->>'mutation' = 'interaction'`,
|
||||
sql`exists (select 1 from heartbeat_runs r where r.company_id = ${agentWakeupRequests.companyId}
|
||||
and r.runtime_mode = 'legacy' and r.status = 'cancelled'
|
||||
and r.result_json->>'queuedCommentInterruptQueueId' = ${agentWakeupRequests.id}::text)`,
|
||||
)),
|
||||
)).orderBy(asc(agentWakeupRequests.id)).limit(100);
|
||||
const complete = rows.length < 100;
|
||||
if (complete) cursors.delete(key); else cursors.set(key, rows.at(-1)!.id);
|
||||
return { rows: rows.map(row => ({ id: row.id, companyId: row.companyId,
|
||||
dueAt: kind === "native" && row.status === "claimed" && row.claimedAt ? row.claimedAt.getTime() + 60_000
|
||||
: kind === "native" && row.error ? row.updatedAt.getTime() + 30_000
|
||||
: kind === "deferred" && row.delayed && !row.interruptedLegacy ? row.updatedAt.getTime() + 30_000
|
||||
: Date.now(),
|
||||
})), complete };
|
||||
},
|
||||
check: async entry => {
|
||||
if (!enabled) return null;
|
||||
if (kind !== "task_chat") {
|
||||
const [current] = await db.select({ status: agentWakeupRequests.status, payload: agentWakeupRequests.payload,
|
||||
updatedAt: agentWakeupRequests.updatedAt, claimedAt: agentWakeupRequests.claimedAt, error: agentWakeupRequests.error,
|
||||
interruptedLegacy: sql<boolean>`exists (select 1 from heartbeat_runs r
|
||||
where r.company_id = ${agentWakeupRequests.companyId} and r.agent_id = ${agentWakeupRequests.agentId}
|
||||
and r.runtime_mode = 'legacy' and r.status = 'cancelled'
|
||||
and r.result_json->>'queuedCommentInterruptQueueId' = ${agentWakeupRequests.id}::text)`,
|
||||
}).from(agentWakeupRequests).where(and(eq(agentWakeupRequests.companyId, entry.companyId),
|
||||
eq(agentWakeupRequests.id, entry.id)));
|
||||
if (!current) return null;
|
||||
const due = kind === "native" && current.status === "claimed" && current.claimedAt
|
||||
? current.claimedAt.getTime() + 60_000
|
||||
: kind === "native" && current.status === "queued" && current.error ? current.updatedAt.getTime() + 30_000
|
||||
: kind === "deferred" && current.status === "deferred_issue_execution" && !current.interruptedLegacy
|
||||
&& (current.payload?.queuedCommentInterrupt || current.payload?.executionWait)
|
||||
? current.updatedAt.getTime() + 30_000 : null;
|
||||
if (due !== null && due > Date.now()) return { ...entry, dueAt: due };
|
||||
}
|
||||
const errors: unknown[] = [];
|
||||
let next: number | null = null;
|
||||
const scope = { companyId: entry.companyId, requestId: entry.id };
|
||||
if (kind === "native") await owner.dispatchNative({ ...scope,
|
||||
onDeferredDeadline: at => { next = next === null ? at : Math.min(next, at); },
|
||||
onError: error => { errors.push(error); },
|
||||
});
|
||||
else if (kind === "task_chat") await owner.settleTaskChat(scope);
|
||||
else await owner.resume({ ...scope, onError: error => { errors.push(error); } });
|
||||
if (errors.length) throw new AggregateError(errors, `Deferred receipt ${entry.id} could not settle`);
|
||||
return next === null ? null : { ...entry, dueAt: next };
|
||||
},
|
||||
}, canRun);
|
||||
}
|
||||
const workers = [register("deferred"), register("task_chat"), register("native")];
|
||||
return { ready: Promise.all(workers.map(worker => worker.ready)),
|
||||
wake() { for (const worker of workers) worker.invalidate(); },
|
||||
async stop() { await Promise.all(workers.map(worker => worker.stop())); } };
|
||||
}
|
||||
@@ -0,0 +1,58 @@
|
||||
import type { RunDispatchScope } from "../run-dispatch-notifications.js";
|
||||
import type { createDeliveryWorkCoordinator } from "../delivery-work-coordinator.js";
|
||||
import { registerDeadlineWork } from "../deadline-work.js";
|
||||
import { DELIVERY_QUEUES } from "../delivery-work-notifications.js";
|
||||
|
||||
export function registerRunDispatchWork(
|
||||
coordinator: ReturnType<typeof createDeliveryWorkCoordinator>,
|
||||
owner: {
|
||||
promoteDueScheduledRetries(): Promise<unknown>;
|
||||
nextScheduledRetryAt(): Promise<number | null>;
|
||||
resumeQueuedRunsAfterDrain(scope?: RunDispatchScope): Promise<unknown>;
|
||||
schedulingSuppressed(): Promise<boolean>;
|
||||
},
|
||||
canRun: () => boolean,
|
||||
) {
|
||||
let suppressed = false;
|
||||
const retries = registerDeadlineWork(coordinator, DELIVERY_QUEUES.scheduledRetryDispatch, {
|
||||
retryMs: 30_000,
|
||||
continuationMs: 1_000,
|
||||
notificationCoalesceMs: 1_000,
|
||||
canRun,
|
||||
run: async signal => {
|
||||
suppressed = await owner.schedulingSuppressed();
|
||||
if (!signal.aborted && !suppressed && canRun()) await owner.promoteDueScheduledRetries();
|
||||
},
|
||||
next: () => suppressed ? Promise.resolve(null) : owner.nextScheduledRetryAt(),
|
||||
});
|
||||
const scopes = new Set<string>(["*"]);
|
||||
const queue = registerDeadlineWork(coordinator, DELIVERY_QUEUES.queuedRunDispatch, {
|
||||
retryMs: 30_000,
|
||||
notificationCoalesceMs: 1_000,
|
||||
canRun,
|
||||
onSettled: hints => { for (const hint of hints ?? ["*"]) scopes.add(hint); },
|
||||
run: async signal => {
|
||||
const requested = scopes.has("*") ? ["*"] : [...scopes];
|
||||
if (requested[0] === "*") scopes.clear();
|
||||
else for (const hint of requested) scopes.delete(hint);
|
||||
for (let index = 0; index < requested.length; index++) {
|
||||
if (signal.aborted || !canRun()) {
|
||||
for (const hint of requested.slice(index)) scopes.add(hint);
|
||||
return;
|
||||
}
|
||||
try {
|
||||
await owner.resumeQueuedRunsAfterDrain(requested[index] === "*" ? undefined : JSON.parse(requested[index]) as RunDispatchScope);
|
||||
} catch (error) {
|
||||
for (const hint of requested.slice(index)) scopes.add(hint);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
},
|
||||
next: async () => null,
|
||||
});
|
||||
return {
|
||||
ready: Promise.all([retries.ready, queue.ready]),
|
||||
wake() { scopes.add("*"); retries.wake(); queue.wake(); },
|
||||
async stop() { await Promise.all([retries.stop(), queue.stop()]); },
|
||||
};
|
||||
}
|
||||
@@ -139,28 +139,27 @@ export async function prepareHeartbeatExecutionConfiguration(db: Db, input: Hear
|
||||
stripWorkspaceRuntimeFromExecutionRunConfig(mergedConfig);
|
||||
const skillActor: AuthorizationActor = { type: "agent", agentId: agent.id, companyId: agent.companyId,
|
||||
runId: run.id, source: "agent_jwt", onBehalfOfUserId: responsibleUserId === "local-board" ? null : responsibleUserId };
|
||||
const runScopedMentionedSkillKeys =
|
||||
preconfiguredHome || pinnedSkillTest ? [] : await time("configuration_mentions", () => resolveRunScopedMentionedSkillKeys({
|
||||
const [mentionedSkills, githubIdentity] = await Promise.allSettled([
|
||||
preconfiguredHome || pinnedSkillTest ? Promise.resolve([]) : time("configuration_mentions", () => resolveRunScopedMentionedSkillKeys({
|
||||
db,
|
||||
companyId: agent.companyId,
|
||||
issueId,
|
||||
commentIds: extractWakeCommentIds(context),
|
||||
actor: skillActor,
|
||||
}));
|
||||
})),
|
||||
time("configuration_github_identity", () => resolveManagedGitHubIdentitySelection(db, agent.companyId, {
|
||||
agentId: agent.id, responsibleUserId, allowStandingDelegation: false,
|
||||
})),
|
||||
]);
|
||||
if (mentionedSkills.status === "rejected") throw mentionedSkills.reason;
|
||||
if (githubIdentity.status === "rejected") throw githubIdentity.reason;
|
||||
const runScopedMentionedSkillKeys = mentionedSkills.value;
|
||||
const runScopedSkillKeys =
|
||||
!pinnedSkillTest && acceptedPlanContinuationWake &&
|
||||
!acceptedPlanWakeRoutingDecision?.suppressAcceptedContinuation
|
||||
? [...runScopedMentionedSkillKeys, ACCEPTED_PLAN_CONVERSION_SKILL_KEY]
|
||||
: runScopedMentionedSkillKeys;
|
||||
const githubSelection = await time("configuration_github_identity", () => resolveManagedGitHubIdentitySelection(
|
||||
db,
|
||||
agent.companyId,
|
||||
{
|
||||
agentId: agent.id,
|
||||
responsibleUserId,
|
||||
allowStandingDelegation: false,
|
||||
},
|
||||
));
|
||||
const githubSelection = githubIdentity.value;
|
||||
const useHostGitHub =
|
||||
!githubSelection.configured &&
|
||||
trustPreset.kind === "standard" &&
|
||||
@@ -274,6 +273,7 @@ export async function prepareHeartbeatExecutionConfiguration(db: Db, input: Hear
|
||||
);
|
||||
const nativeRunnerPreparationSpans: NativeRunHistoricalSpan[] = [];
|
||||
const skillsPrepareStartedAtMs = Date.now();
|
||||
let skillsPrepareEndedAtMs = skillsPrepareStartedAtMs;
|
||||
let skillDeliveryConfig = effectiveResolvedConfig;
|
||||
const prepareSkills = async () => {
|
||||
|
||||
@@ -346,24 +346,35 @@ export async function prepareHeartbeatExecutionConfiguration(db: Db, input: Hear
|
||||
throw error;
|
||||
}
|
||||
};
|
||||
const runtimeSkillEntries = preconfiguredHome ? [] : await time("configuration_skills", prepareSkills);
|
||||
const skillsPreparation = (preconfiguredHome ? Promise.resolve([]) : time("configuration_skills", prepareSkills))
|
||||
.finally(() => { skillsPrepareEndedAtMs = Date.now(); });
|
||||
const assignmentsPreparation = preconfiguredHome ? Promise.resolve([]) : time("configuration_connector_assignments", () => resolveConnectorAssignments(db, { companyId: agent.companyId, agentId: agent.id, runId: run.id, issueId: typeof context.issueId === "string" ? context.issueId : undefined }));
|
||||
const [preparedSkills, preparedAssignments, preparedInstructions] = await Promise.allSettled([
|
||||
skillsPreparation,
|
||||
assignmentsPreparation,
|
||||
assignmentsPreparation.then(() => time("configuration_connection_instructions", () => resolveAssignedConnectionInstructionsForRun(db, { companyId: agent.companyId, agentId: agent.id, runId: run.id }))),
|
||||
]);
|
||||
if (preparedSkills.status === "rejected") throw preparedSkills.reason;
|
||||
if (preparedAssignments.status === "rejected") throw preparedAssignments.reason;
|
||||
if (preparedInstructions.status === "rejected") throw preparedInstructions.reason;
|
||||
const runtimeSkillEntries = preparedSkills.value;
|
||||
nativeRunnerPreparationSpans.push({
|
||||
name: "skills.prepare",
|
||||
parentName: "task.prepare",
|
||||
startedAtMs: skillsPrepareStartedAtMs,
|
||||
endedAtMs: Date.now(),
|
||||
endedAtMs: skillsPrepareEndedAtMs,
|
||||
});
|
||||
const connectorAssignments = preconfiguredHome ? [] : await time("configuration_connector_assignments", () => resolveConnectorAssignments(db, { companyId: agent.companyId, agentId: agent.id, runId: run.id, issueId: typeof context.issueId === "string" ? context.issueId : undefined }));
|
||||
const connectorAssignments = preparedAssignments.value;
|
||||
const connectorSkillConfig = await time("configuration_connector_skills", () => applyConnectorSkills(skillDeliveryConfig, runtimeSkillEntries, connectorAssignments));
|
||||
// Both CLI adapters and native context materialization use the same resolved set.
|
||||
runtimeSkillEntries.splice(0, runtimeSkillEntries.length, ...connectorSkillConfig.paperclipRuntimeSkills);
|
||||
const connectorDelivery = await time("configuration_connector_delivery", () => prepareConnectorSkillDelivery(connectorSkillConfig, agent.adapterType));
|
||||
// Always replace this runtime-only field; caller wake data cannot supply skills.
|
||||
context.paperclipWake = { ...parseObject(context.paperclipWake), connectorSkillInstructions: connectorDelivery.instructions };
|
||||
let runtimeConfig = await time("configuration_connection_instructions", () => prepareConnectionInstructionDelivery({
|
||||
resolve: () => resolveAssignedConnectionInstructionsForRun(db, { companyId: agent.companyId, agentId: agent.id, runId: run.id }),
|
||||
let runtimeConfig = await prepareConnectionInstructionDelivery({
|
||||
resolve: async () => preparedInstructions.value,
|
||||
context, config: connectorDelivery.config, native: agent.adapterType === "paperclip_runner",
|
||||
}));
|
||||
});
|
||||
const agentIdentity = supportsManagedAgentIdentity(agent.adapterType, agent.adapterConfig, run.runtimeMode === "native" ? run.driverKind : undefined)
|
||||
? await time("configuration_agent_identity", () => agentIdentityService(db).ensureAgentIdentity(agent.companyId, agent.id))
|
||||
: undefined;
|
||||
|
||||
@@ -1,4 +1,9 @@
|
||||
import { notifyRoutineExecutionStatus } from "../routine-execution-status-notifications.js";
|
||||
import { notifyStatusCardIssueChange } from "../status-card-notifications.js";
|
||||
import { writeDeferredExecutionWait } from "../deferred-execution-wait.js";
|
||||
import { notifyDeferredExecutionWork } from "../deferred-execution-notifications.js";
|
||||
import type { DeferredReceiptRecoveryOptions } from "../deferred-execution-notifications.js";
|
||||
import { notifyRunDispatch } from "../run-dispatch-notifications.js";
|
||||
import { DELIVERY_QUEUES, notifyDeliveryWork } from "../delivery-work-notifications.js";
|
||||
import { notifyExternalAgentSettlement } from "../../modules/external-agents/index.js";
|
||||
import { notifyNativeStatusReassessment } from "../native-status-reassessment-notifications.js";
|
||||
@@ -222,7 +227,7 @@ export interface HeartbeatQueueDependencies extends Pick<HeartbeatRetryDependenc
|
||||
publishRunLifecyclePluginEvent: (run: HeartbeatRun) => void;
|
||||
cancelRunInternal: (runId: string, reason?: string, options?: { errorCode?: string }) => Promise<unknown>;
|
||||
cancelActiveForAgentInternal: (agentId: string, reason: string) => Promise<unknown>;
|
||||
resumeExecutionWaitComments: () => Promise<unknown>;
|
||||
resumeExecutionWaitComments: (scope?: DeferredReceiptRecoveryOptions) => Promise<unknown>;
|
||||
resumeQueuedCommentInterrupt: (companyId: string, queueId: string) => Promise<unknown>;
|
||||
resumeSavedLegacyComments: (companyId: string, queueId: string) => Promise<unknown>;
|
||||
formatIssueIdentifierLink: (identifier: string | null, fallback: string) => string;
|
||||
@@ -1268,6 +1273,7 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
|
||||
}
|
||||
async function bindClaimedIssueExecution(tx: Db, ownsIssue: boolean, claimedRun: typeof heartbeatRuns.$inferSelect | null | undefined) {
|
||||
if (!claimedRun || !issueId || !ownsIssue) return;
|
||||
await notifyStatusCardIssueChange(tx, run.companyId);
|
||||
await tx.update(issues).set({
|
||||
executionRunId: claimedRun.id,
|
||||
executionAgentNameKey: normalizeAgentNameKey(agent.name),
|
||||
@@ -1851,13 +1857,15 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
|
||||
return cancelled;
|
||||
}
|
||||
|
||||
async function resumeQueuedRuns() {
|
||||
async function resumeDeferredExecutionReceipts(scope: DeferredReceiptRecoveryOptions = {}) {
|
||||
if ((await getSchedulingSuppression()).suppressed) return;
|
||||
await resumeExecutionWaitComments();
|
||||
await resumeExecutionWaitComments(scope);
|
||||
const cutoff = await getWorktreeExecutionCutoff();
|
||||
const pendingInterrupts = await db.select({ id: agentWakeupRequests.id, companyId: agentWakeupRequests.companyId })
|
||||
.from(agentWakeupRequests).innerJoin(companies, eq(companies.id, agentWakeupRequests.companyId))
|
||||
.where(and(eq(agentWakeupRequests.status, "deferred_issue_execution"),
|
||||
.where(and(scope.companyId ? eq(agentWakeupRequests.companyId, scope.companyId) : undefined,
|
||||
scope.requestId ? eq(agentWakeupRequests.id, scope.requestId) : undefined,
|
||||
eq(agentWakeupRequests.status, "deferred_issue_execution"),
|
||||
eq(companies.status, "active"),
|
||||
sql`${agentWakeupRequests.payload}->'queuedCommentInterrupt' is not null`,
|
||||
lte(agentWakeupRequests.updatedAt, new Date(Date.now() - 30_000)),
|
||||
@@ -1868,6 +1876,7 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
|
||||
eq(agentWakeupRequests.id, wake.id), eq(agentWakeupRequests.status, "deferred_issue_execution"),
|
||||
));
|
||||
await resumeQueuedCommentInterrupt(wake.companyId, wake.id).catch(err => {
|
||||
scope.onError?.(err);
|
||||
logger.warn({ err, queueId: wake.id }, "failed to resume interrupted comment queue");
|
||||
});
|
||||
}
|
||||
@@ -1881,7 +1890,9 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
|
||||
or(eq(issues.assigneeAgentId, agentWakeupRequests.agentId),
|
||||
sql`${agentWakeupRequests.payload} #>> '{_paperclipWakeContext,wakeReason}' = 'issue_comment_mentioned'`)))
|
||||
.innerJoin(companies, and(eq(companies.id, issues.companyId), eq(companies.status, "active")))
|
||||
.where(and(eq(agentWakeupRequests.status, "deferred_issue_execution"),
|
||||
.where(and(scope.companyId ? eq(agentWakeupRequests.companyId, scope.companyId) : undefined,
|
||||
scope.requestId ? eq(agentWakeupRequests.id, scope.requestId) : undefined,
|
||||
eq(agentWakeupRequests.status, "deferred_issue_execution"),
|
||||
isNull(issues.executionRunId),
|
||||
or(and(
|
||||
sql`jsonb_typeof(${agentWakeupRequests.payload} #> '{_paperclipWakeContext,wakeCommentIds}') = 'array'`,
|
||||
@@ -1920,7 +1931,8 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
|
||||
if (latest.runtimeMode !== "legacy" && latest.runtimeMode !== "native") continue;
|
||||
if (wake.idempotencyKey?.startsWith("chat-inbound:")) {
|
||||
await resumeDeferredChatWake(db, { companyId: wake.companyId, agentId: wake.agentId,
|
||||
issueId: String(wake.payload?.issueId), requestId: wake.id }).catch(() => {
|
||||
issueId: String(wake.payload?.issueId), requestId: wake.id }).catch(err => {
|
||||
scope.onError?.(err);
|
||||
logger.warn({ requestId: wake.id }, "saved chat source could not resume after recovery");
|
||||
});
|
||||
continue;
|
||||
@@ -1932,6 +1944,7 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
|
||||
latest.errorCode === "execution_reconciliation_required";
|
||||
if ((latest.status !== "cancelled" || cancelledAdmission) && await getExecutionBlocker(db, wake.companyId, String(wake.payload?.issueId))) {
|
||||
await resumeSavedLegacyComments(wake.companyId, wake.id).catch(err => {
|
||||
scope.onError?.(err);
|
||||
logger.warn({ err, queueId: wake.id }, "failed to deliver saved legacy comment after recovery stopped");
|
||||
});
|
||||
}
|
||||
@@ -1939,6 +1952,7 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
|
||||
await releaseIssueExecutionAndPromote(latest, { suppressImmediateRecovery: true, resumeStrandedQueue: true }).then(() => {
|
||||
if (latest.runtimeMode === "native") releasedNativeSources.add(releaseSource);
|
||||
}).catch(err => {
|
||||
scope.onError?.(err);
|
||||
logger.warn({ err, queueId: wake.id }, "failed to promote stranded comments");
|
||||
});
|
||||
}
|
||||
@@ -1956,6 +1970,8 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
|
||||
))
|
||||
.innerJoin(companies, eq(companies.id, heartbeatRuns.companyId))
|
||||
.where(and(
|
||||
scope.companyId ? eq(agentWakeupRequests.companyId, scope.companyId) : undefined,
|
||||
scope.requestId ? eq(agentWakeupRequests.id, scope.requestId) : undefined,
|
||||
eq(agentWakeupRequests.status, "deferred_issue_execution"),
|
||||
eq(heartbeatRuns.status, "cancelled"),
|
||||
eq(heartbeatRuns.runtimeMode, "legacy"),
|
||||
@@ -1964,16 +1980,21 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
|
||||
));
|
||||
for (const run of interruptedQueues) {
|
||||
await releaseIssueExecutionAndPromote(run, { suppressImmediateRecovery: true }).catch((err) => {
|
||||
scope.onError?.(err);
|
||||
logger.error({ err, runId: run.id }, "failed to retry interrupted comment queue");
|
||||
});
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
async function resumeQueuedRuns() {
|
||||
await resumeDeferredExecutionReceipts();
|
||||
await resumeQueuedRunsAfterDrain();
|
||||
}
|
||||
|
||||
// Release admission promptly without first scanning unrelated recovery work.
|
||||
// Every run still passes the ordinary per-agent claim and dispatch gates.
|
||||
async function resumeQueuedRunsAfterDrain() {
|
||||
async function resumeQueuedRunsAfterDrain(scope?: { companyId: string }) {
|
||||
if ((await getSchedulingSuppression()).suppressed) return;
|
||||
const cutoff = await getWorktreeExecutionCutoff();
|
||||
const queuedRuns = await db
|
||||
@@ -1983,6 +2004,7 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
|
||||
.where(
|
||||
and(
|
||||
eq(heartbeatRuns.status, "queued"),
|
||||
scope ? eq(heartbeatRuns.companyId, scope.companyId) : undefined,
|
||||
eq(companies.status, "active"),
|
||||
cutoff ? gte(heartbeatRuns.createdAt, cutoff) : undefined,
|
||||
),
|
||||
@@ -2505,17 +2527,11 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
|
||||
// its one durable receipt ID on a skipped wake.
|
||||
if (caseRequest) return { created: false };
|
||||
if (executionWaitRequestId) {
|
||||
await db.update(agentWakeupRequests).set({
|
||||
payload: sql`jsonb_set(coalesce(${agentWakeupRequests.payload}, '{}'::jsonb), '{executionWait}',
|
||||
coalesce(${agentWakeupRequests.payload}->'executionWait', '{}'::jsonb) || ${JSON.stringify({
|
||||
reason: skipReason, message: patch.error ?? (skipReason === "issue_tree_hold_active"
|
||||
? "This task is paused. Resume it to send your saved message."
|
||||
: "Waiting for task execution to be enabled. Your message is saved."),
|
||||
})}::jsonb)`,
|
||||
updatedAt: new Date(),
|
||||
}).where(and(eq(agentWakeupRequests.id, executionWaitRequestId),
|
||||
eq(agentWakeupRequests.companyId, agent.companyId), eq(agentWakeupRequests.agentId, agentId),
|
||||
eq(agentWakeupRequests.status, "deferred_issue_execution")));
|
||||
await writeDeferredExecutionWait(db, { companyId: agent.companyId, requestId: executionWaitRequestId }, {
|
||||
reason: skipReason, message: patch.error ?? (skipReason === "issue_tree_hold_active"
|
||||
? "This task is paused. Resume it to send your saved message."
|
||||
: "Waiting for task execution to be enabled. Your message is saved."),
|
||||
});
|
||||
return { created: false };
|
||||
}
|
||||
const request = {
|
||||
@@ -3403,15 +3419,14 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
|
||||
opts.issueStateGuard?.onExecutionDeferred?.(executionBlocker.cause);
|
||||
const condition = { recoveryActionId: executionBlocker.recoveryActionId, ...continuationWait };
|
||||
if (executionWaitRequestId) {
|
||||
await tx.update(agentWakeupRequests).set({
|
||||
payload: sql`jsonb_set(coalesce(${agentWakeupRequests.payload}, '{}'::jsonb), '{executionWait}', ${JSON.stringify(condition)}::jsonb)`,
|
||||
updatedAt: new Date(),
|
||||
}).where(eq(agentWakeupRequests.id, executionWaitRequestId));
|
||||
await writeDeferredExecutionWait(tx as unknown as Db, { companyId: agent.companyId,
|
||||
requestId: executionWaitRequestId }, condition, { replace: true });
|
||||
return { kind: "deferred" as const, admitted: true };
|
||||
}
|
||||
if (durableRequest || wakeCommentId ||
|
||||
hasInteractionContinuationWakeContext(enrichedContextSnapshot) ||
|
||||
readNonEmptyString(enrichedContextSnapshot.nativeStatusWakeIntentId)) {
|
||||
await notifyDeferredExecutionWork(tx, { companyId: agent.companyId });
|
||||
await tx.insert(agentWakeupRequests).values({
|
||||
...durableReceiptFields,
|
||||
companyId: agent.companyId, agentId, source, triggerDetail, reason,
|
||||
@@ -3759,6 +3774,7 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
|
||||
.from(agents)
|
||||
.where(eq(agents.id, legacyRun.agentId))
|
||||
.then((rows) => rows[0] ?? null);
|
||||
await notifyStatusCardIssueChange(tx, issue.companyId);
|
||||
await tx
|
||||
.update(issues)
|
||||
.set({
|
||||
@@ -4030,6 +4046,7 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
|
||||
}));
|
||||
}
|
||||
if (followUp) {
|
||||
if (!followUp.consumed) await notifyDeferredExecutionWork(tx, { companyId: agent.companyId });
|
||||
const assignmentId = followUp.assignmentId;
|
||||
const pending = followUp.consumed ? undefined : pendingFollowUps.find(wake =>
|
||||
wake.payload?.[followUpPayloadKey] === assignmentId &&
|
||||
@@ -4634,6 +4651,7 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
|
||||
}
|
||||
|
||||
const queueOutcome = await db.transaction(async (tx) => {
|
||||
await notifyRunDispatch(tx, { companyId: agent.companyId });
|
||||
await tx.execute(
|
||||
sql`select id from agents where id = ${agentId} and company_id = ${agent.companyId} for update`,
|
||||
);
|
||||
@@ -4767,6 +4785,9 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
|
||||
companyId?: string;
|
||||
limit?: number;
|
||||
staleClaimMs?: number;
|
||||
requestId?: string;
|
||||
onDeferredDeadline?: (at: number) => void;
|
||||
onError?: (error: unknown) => void;
|
||||
} = {},
|
||||
) {
|
||||
const now = new Date();
|
||||
@@ -4779,6 +4800,7 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
|
||||
input.companyId
|
||||
? eq(agentWakeupRequests.companyId, input.companyId)
|
||||
: undefined,
|
||||
input.requestId ? eq(agentWakeupRequests.id, input.requestId) : undefined,
|
||||
eq(agentWakeupRequests.requestedByActorType, "system"),
|
||||
eq(agentWakeupRequests.requestedByActorId, "native-status-committer"),
|
||||
inArray(agentWakeupRequests.status, ["queued", "claimed"]),
|
||||
@@ -4855,6 +4877,7 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
|
||||
candidate.claimedAt &&
|
||||
now.getTime() - candidate.claimedAt.getTime() < staleClaimMs
|
||||
) {
|
||||
input.onDeferredDeadline?.(candidate.claimedAt.getTime() + staleClaimMs);
|
||||
deferred += 1;
|
||||
continue;
|
||||
}
|
||||
@@ -5029,6 +5052,7 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
|
||||
if (wakeRun) dispatched += 1;
|
||||
else deferred += 1;
|
||||
} catch (error) {
|
||||
input.onError?.(error);
|
||||
await db
|
||||
.update(agentWakeupRequests)
|
||||
.set({
|
||||
@@ -5072,6 +5096,7 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
|
||||
withChatControlRecoveryGate,
|
||||
trackWakeup,
|
||||
resumeQueuedRuns,
|
||||
resumeDeferredExecutionReceipts,
|
||||
resumeQueuedRunsAfterDrain,
|
||||
parseHeartbeatPolicy,
|
||||
nextTimerHeartbeatAt,
|
||||
|
||||
@@ -1979,7 +1979,7 @@ export function createHeartbeatRecovery(db: Db, dependencies: HeartbeatRecoveryD
|
||||
|
||||
let settledProcessesDiscovered = false;
|
||||
|
||||
async function reapOrphanedRuns(opts?: { staleThresholdMs?: number; skipScheduledRetryCancellation?: boolean; skipCapturedInstructionCommit?: boolean; skipInstructionDirectoryCleanup?: boolean; skipStoppedInstructionCollection?: boolean; skipCancelledWorkspacePreservation?: boolean; skipObsoletePolicyReviews?: boolean; skipAutomaticCompletionReviews?: boolean; skipUnsafeWorkspaceExports?: boolean; skipNativePendingEffects?: boolean; skipNativeResponsePresentation?: boolean; skipNativeReviewResponse?: boolean; skipNativeStatusReassessment?: boolean; skipRetainedNativeSessionCleanup?: boolean; skipNativeWorkspaceFinalization?: boolean; skipNativeResultCommit?: boolean; skipNativeRunProjection?: boolean; skipNativeRecoveryNotices?: boolean; skipExternalConversationSettlement?: boolean; skipExternalAgentSettlement?: boolean; skipNativeEnvironmentSettlement?: boolean }) {
|
||||
async function reapOrphanedRuns(opts?: { staleThresholdMs?: number; skipNativeStatusWakeIntents?: boolean; skipScheduledRetryCancellation?: boolean; skipCapturedInstructionCommit?: boolean; skipInstructionDirectoryCleanup?: boolean; skipStoppedInstructionCollection?: boolean; skipCancelledWorkspacePreservation?: boolean; skipObsoletePolicyReviews?: boolean; skipAutomaticCompletionReviews?: boolean; skipUnsafeWorkspaceExports?: boolean; skipNativePendingEffects?: boolean; skipNativeResponsePresentation?: boolean; skipNativeReviewResponse?: boolean; skipNativeStatusReassessment?: boolean; skipRetainedNativeSessionCleanup?: boolean; skipNativeWorkspaceFinalization?: boolean; skipNativeResultCommit?: boolean; skipNativeRunProjection?: boolean; skipNativeRecoveryNotices?: boolean; skipExternalConversationSettlement?: boolean; skipExternalAgentSettlement?: boolean; skipNativeEnvironmentSettlement?: boolean }) {
|
||||
if (!opts?.skipCancelledWorkspacePreservation) await preserveCancelledNativeWorkspaces(db);
|
||||
const staleThresholdMs = opts?.staleThresholdMs ?? 0;
|
||||
const now = new Date();
|
||||
@@ -2023,7 +2023,7 @@ export function createHeartbeatRecovery(db: Db, dependencies: HeartbeatRecoveryD
|
||||
});
|
||||
if (!opts?.skipRetainedNativeSessionCleanup) scheduleRetainedNativeSessionCleanup();
|
||||
await reconcileRetainedWorkspaces(db, environmentRuntime, resumeRemoteStopComments);
|
||||
await dispatchPendingNativeStatusWakeups().catch((error) => {
|
||||
if (!opts?.skipNativeStatusWakeIntents) await dispatchPendingNativeStatusWakeups().catch((error) => {
|
||||
logger.warn(
|
||||
{ err: error },
|
||||
"failed to dispatch persisted native status wake intents before orphan reaping",
|
||||
|
||||
@@ -35,6 +35,7 @@ function callbacks() {
|
||||
getWorktreeExecutionCutoff: vi.fn(async (): Promise<Date | null> => null),
|
||||
applyRunDispatchPostCommitEffects: vi.fn<(effects: PostCommitEffect[]) => void>(),
|
||||
runDispatch: {
|
||||
nextScheduledRetryAt: vi.fn(async () => null),
|
||||
evaluateScheduledRetryGate: vi.fn<HeartbeatRetryDependencies["runDispatch"]["evaluateScheduledRetryGate"]>(async () => ({ allowed: true })),
|
||||
promoteDueScheduledRetries: vi.fn<HeartbeatRetryDependencies["runDispatch"]["promoteDueScheduledRetries"]>(async () => ({ promoted: 0, runIds: [], postCommitEffects: [] })),
|
||||
promoteScheduledRetry: vi.fn<HeartbeatRetryDependencies["runDispatch"]["promoteScheduledRetry"]>(async () => ({ outcome: "not_promoted" })),
|
||||
@@ -183,7 +184,9 @@ describe("heartbeat retry module callbacks", () => {
|
||||
});
|
||||
});
|
||||
|
||||
const support = await getEmbeddedPostgresTestSupport();
|
||||
import { resolveExternalTestDatabaseUrl } from "../../__tests__/helpers/external-test-database.js";
|
||||
const external = resolveExternalTestDatabaseUrl("PAPERCLIP_HEARTBEAT_RETRIES_TEST_DATABASE_URL");
|
||||
const support = external ? { supported: true, reason: undefined } : await getEmbeddedPostgresTestSupport();
|
||||
const describePostgres = support.supported ? describe : describe.skip;
|
||||
if (!support.supported) console.warn(`Skipping retry PostgreSQL tests: ${support.reason ?? "unsupported host"}`);
|
||||
|
||||
@@ -193,11 +196,11 @@ describePostgres("heartbeat retry module database wiring", () => {
|
||||
const now = new Date("2026-10-08T12:00:00Z");
|
||||
|
||||
beforeAll(async () => {
|
||||
database = await startEmbeddedPostgresTestDatabase("heartbeat-retries-");
|
||||
db = createDb(database.connectionString);
|
||||
if (!external) database = await startEmbeddedPostgresTestDatabase("heartbeat-retries-");
|
||||
db = createDb(external ?? database.connectionString, { maxConnections: 6 });
|
||||
}, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS);
|
||||
|
||||
afterAll(async () => { await database?.cleanup(); });
|
||||
afterAll(async () => { await db?.$client.end({ timeout: 1 }); await database?.cleanup(); });
|
||||
|
||||
afterEach(async () => {
|
||||
await db.delete(issues);
|
||||
|
||||
@@ -1,3 +1,6 @@
|
||||
import { notifyAttentionRetentionWork } from "../retention-work-notifications.js";
|
||||
import { notifyStatusCardIssueChange } from "../status-card-notifications.js";
|
||||
import { notifyRunDispatch } from "../run-dispatch-notifications.js";
|
||||
import { deriveTaskKeyWithHeartbeatFallback } from "./run-state.js";
|
||||
import { infrastructureRetryDelayMs } from "../system-recovery-state.js";
|
||||
import {
|
||||
@@ -77,7 +80,7 @@ type PlanApprovalResumeFailure = {
|
||||
export interface HeartbeatRetryDependencies {
|
||||
hasCaseHandoffForRun: (run: Pick<HeartbeatRun, "id" | "companyId">) => Promise<boolean>;
|
||||
runDispatch: Pick<ReturnType<typeof createRunDispatch>,
|
||||
"evaluateScheduledRetryGate" | "promoteDueScheduledRetries" | "promoteScheduledRetry">;
|
||||
"evaluateScheduledRetryGate" | "promoteDueScheduledRetries" | "promoteScheduledRetry" | "nextScheduledRetryAt">;
|
||||
getAgent: (agentId: string) => Promise<typeof agents.$inferSelect | null | undefined>;
|
||||
getRun: (
|
||||
runId: string,
|
||||
@@ -997,6 +1000,8 @@ export function createHeartbeatRetries(db: Db, dependencies: HeartbeatRetryDepen
|
||||
retryContextSnapshot.previousRunId = continuation.previousRunId;
|
||||
}
|
||||
|
||||
await notifyAttentionRetentionWork(tx, { companyId: run.companyId });
|
||||
await notifyRunDispatch(tx, { companyId: run.companyId });
|
||||
const wakeupRequest = await tx
|
||||
.insert(agentWakeupRequests)
|
||||
.values({
|
||||
@@ -1178,6 +1183,7 @@ export function createHeartbeatRetries(db: Db, dependencies: HeartbeatRetryDepen
|
||||
}
|
||||
|
||||
if (issueId) {
|
||||
await notifyStatusCardIssueChange(tx, run.companyId);
|
||||
await tx
|
||||
.update(issues)
|
||||
.set({
|
||||
@@ -1643,6 +1649,10 @@ export function createHeartbeatRetries(db: Db, dependencies: HeartbeatRetryDepen
|
||||
});
|
||||
}
|
||||
|
||||
async function nextScheduledRetryAt() {
|
||||
return runDispatch.nextScheduledRetryAt({ cutoff: await getWorktreeExecutionCutoff() });
|
||||
}
|
||||
|
||||
async function promoteDueScheduledRetries(now = new Date()) {
|
||||
const cutoff = await getWorktreeExecutionCutoff();
|
||||
const result = await runDispatch.promoteDueScheduledRetries({
|
||||
@@ -1750,6 +1760,7 @@ export function createHeartbeatRetries(db: Db, dependencies: HeartbeatRetryDepen
|
||||
};
|
||||
|
||||
const updated = await db.transaction(async (tx) => {
|
||||
await notifyRunDispatch(tx, { companyId: scheduled.run.companyId });
|
||||
const row = await tx
|
||||
.update(heartbeatRuns)
|
||||
.set({
|
||||
@@ -1901,6 +1912,7 @@ export function createHeartbeatRetries(db: Db, dependencies: HeartbeatRetryDepen
|
||||
finalizeAiConnectionBusyDeferral,
|
||||
finalizeWorkspaceBusyDeferral,
|
||||
promoteDueScheduledRetries,
|
||||
nextScheduledRetryAt,
|
||||
retryScheduledRetryNow,
|
||||
|
||||
scheduleBoundedRetry: async (
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
import { writeDeferredExecutionWait } from "../deferred-execution-wait.js";
|
||||
import type { DeferredReceiptRecoveryOptions } from "../deferred-execution-notifications.js";
|
||||
import { observeStoppedWorkspaceRecovery } from "../workspace-recovery.js";
|
||||
import { notifyScheduledRetryCancellation } from "../scheduled-retry-cancellation-notifications.js";
|
||||
import { notifyChatInboundWakeNotices } from "../chat-inbound-wake-notice-notifications.js";
|
||||
@@ -616,12 +618,7 @@ export function createHeartbeatRunControl(db: Db, dependencies: HeartbeatRunCont
|
||||
onBlocked: (reason, message) => { wait = { reason, message }; },
|
||||
});
|
||||
if (!admitted) {
|
||||
await db.update(agentWakeupRequests).set({
|
||||
payload: sql`jsonb_set(coalesce(${agentWakeupRequests.payload}, '{}'::jsonb), '{executionWait}',
|
||||
coalesce(${agentWakeupRequests.payload}->'executionWait', '{}'::jsonb) || ${JSON.stringify(wait)}::jsonb)`,
|
||||
updatedAt: new Date(),
|
||||
}).where(and(eq(agentWakeupRequests.id, wake.id), eq(agentWakeupRequests.companyId, run.companyId),
|
||||
eq(agentWakeupRequests.status, "deferred_issue_execution")));
|
||||
await writeDeferredExecutionWait(db, { companyId: run.companyId, requestId: wake.id }, wait);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
@@ -777,21 +774,25 @@ export function createHeartbeatRunControl(db: Db, dependencies: HeartbeatRunCont
|
||||
idempotencyKey: `task-retry-dispatch:${wake.id}` }, wake.id);
|
||||
}
|
||||
|
||||
async function resumeExecutionWaitComments() {
|
||||
async function resumeExecutionWaitComments(scope: DeferredReceiptRecoveryOptions = {}) {
|
||||
if ((await getSchedulingSuppression()).suppressed) return;
|
||||
await reconcileTaskChatRetryReceipts(db);
|
||||
await reconcileTaskChatRetryReceipts(db, scope);
|
||||
const retries = await db.select({ id: agentWakeupRequests.id, companyId: agentWakeupRequests.companyId })
|
||||
.from(agentWakeupRequests).innerJoin(issues, and(eq(issues.companyId, agentWakeupRequests.companyId),
|
||||
sql`${issues.id}::text = ${agentWakeupRequests.payload}->>'issueId'`,
|
||||
eq(issues.assigneeAgentId, agentWakeupRequests.agentId)))
|
||||
.innerJoin(companies, and(eq(companies.id, issues.companyId), eq(companies.status, "active")))
|
||||
.where(and(eq(agentWakeupRequests.reason, "task_retry"), eq(agentWakeupRequests.status, "deferred_issue_execution"),
|
||||
.where(and(scope.companyId ? eq(agentWakeupRequests.companyId, scope.companyId) : undefined,
|
||||
scope.requestId ? eq(agentWakeupRequests.id, scope.requestId) : undefined,
|
||||
eq(agentWakeupRequests.reason, "task_retry"), eq(agentWakeupRequests.status, "deferred_issue_execution"),
|
||||
sql`${agentWakeupRequests.payload}->'taskRetry'->>'chatActionId' is null`,
|
||||
notInArray(issues.status, ["done", "cancelled"])))
|
||||
.orderBy(asc(agentWakeupRequests.updatedAt)).limit(50);
|
||||
for (const retry of retries) {
|
||||
await db.update(agentWakeupRequests).set({ updatedAt: new Date() }).where(eq(agentWakeupRequests.id, retry.id));
|
||||
await resumeTaskRetry(retry.companyId, retry.id).catch(err => logger.warn({ err, requestId: retry.id }, "Task retry dispatch deferred"));
|
||||
await resumeTaskRetry(retry.companyId, retry.id).catch(err => {
|
||||
scope.onError?.(err); logger.warn({ err, requestId: retry.id }, "Task retry dispatch deferred");
|
||||
});
|
||||
}
|
||||
const waits = await db.select({ wake: agentWakeupRequests })
|
||||
.from(agentWakeupRequests)
|
||||
@@ -799,7 +800,9 @@ export function createHeartbeatRunControl(db: Db, dependencies: HeartbeatRunCont
|
||||
sql`${issues.id}::text = ${agentWakeupRequests.payload}->>'issueId'`,
|
||||
eq(issues.assigneeAgentId, agentWakeupRequests.agentId)))
|
||||
.innerJoin(companies, and(eq(companies.id, issues.companyId), eq(companies.status, "active")))
|
||||
.where(and(exists(db.select({ id: issueRecoveryActions.id }).from(issueRecoveryActions).where(and(
|
||||
.where(and(scope.companyId ? eq(agentWakeupRequests.companyId, scope.companyId) : undefined,
|
||||
scope.requestId ? eq(agentWakeupRequests.id, scope.requestId) : undefined,
|
||||
exists(db.select({ id: issueRecoveryActions.id }).from(issueRecoveryActions).where(and(
|
||||
eq(issueRecoveryActions.companyId, issues.companyId), eq(issueRecoveryActions.sourceIssueId, issues.id),
|
||||
executionBlockerPredicate(),
|
||||
))), eq(agentWakeupRequests.status, "deferred_issue_execution"),
|
||||
@@ -831,11 +834,13 @@ export function createHeartbeatRunControl(db: Db, dependencies: HeartbeatRunCont
|
||||
if (!run || run.companyId !== wake.companyId || run.agentId !== wake.agentId) continue;
|
||||
if (canContinueCancelledRun(run)) {
|
||||
await resumeSavedComments(wake.companyId, wake.id).catch(err => {
|
||||
scope.onError?.(err);
|
||||
logger.warn({ err, runId: run.id }, "failed to resume saved input after provider cancellation");
|
||||
});
|
||||
continue;
|
||||
}
|
||||
await resumeRemoteStopComments(run, wake.id).catch(err => {
|
||||
scope.onError?.(err);
|
||||
logger.warn({ err, runId: run.id }, "failed to resume saved execution-wait message");
|
||||
});
|
||||
}
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { notifyRunDispatch } from "../run-dispatch-notifications.js";
|
||||
import { preserveAcceptedRunUsageProjectionSql } from "../usage-receipts.js";
|
||||
import { notifyNativeEnvironmentSettlement } from "../native-environment-settlement-notifications.js";
|
||||
import { notifyExternalAgentSettlement } from "../../modules/external-agents/index.js";
|
||||
@@ -2448,6 +2449,7 @@ export function createHeartbeatLifecycle(db: Db, dependencies: HeartbeatLifecycl
|
||||
.returning()
|
||||
.then((rows) => rows[0]);
|
||||
|
||||
await notifyRunDispatch(tx, { companyId: run.companyId });
|
||||
const queuedRun = await tx
|
||||
.insert(heartbeatRuns)
|
||||
.values({
|
||||
|
||||
@@ -14,6 +14,7 @@ import {
|
||||
ConfigurationIncompleteFailure,
|
||||
createHeartbeatRunPreparation,
|
||||
resolveExecutionRunAdapterConfig,
|
||||
readHeartbeatWakeComments,
|
||||
} from "./run-preparation.js";
|
||||
|
||||
// Existing integration suites exercise the real database and dispatch path.
|
||||
@@ -38,6 +39,12 @@ function databaseWithResults(...results: unknown[][]) {
|
||||
return { db: { select } as unknown as Db, select, scopes };
|
||||
}
|
||||
|
||||
function deferred<T>() {
|
||||
let resolve!: (value: T) => void;
|
||||
const promise = new Promise<T>(complete => { resolve = complete; });
|
||||
return { promise, resolve };
|
||||
}
|
||||
|
||||
const emptyRoutineContext = { routineId: null, env: null, responsibleUserId: null };
|
||||
|
||||
describe("heartbeat run preparation module", () => {
|
||||
@@ -120,6 +127,42 @@ describe("heartbeat run preparation module", () => {
|
||||
expect(database.select).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it("scopes one selected-comment snapshot for both wake consumers", async () => {
|
||||
const database = databaseWithResults([{ id: "comment-one", body: "selected wake" }]);
|
||||
expect(await readHeartbeatWakeComments(database.db, "company-one", "issue-one", ["comment-one"]))
|
||||
.toEqual([{ id: "comment-one", body: "selected wake" }]);
|
||||
expect(database.scopes[0]).toEqual(expect.arrayContaining(["company-one", "issue-one", "comment-one"]));
|
||||
expect(await readHeartbeatWakeComments(database.db, "company-one", "issue-one", [])).toEqual([]);
|
||||
expect(database.select).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
it("settles independent binding checks before surfacing a failure or resolving values", async () => {
|
||||
const projectCheck = deferred<[]>();
|
||||
const firstFailure = new Error("agent binding check failed");
|
||||
const secretsSvc = {
|
||||
resolveAdapterConfigForRuntime: vi.fn(), resolveEnvBindings: vi.fn(),
|
||||
collectMissingRuntimeBindings: vi.fn(async (_company, _env, context) => {
|
||||
if (context.consumerType === "agent") throw firstFailure;
|
||||
return context.consumerType === "project" ? projectCheck.promise : [];
|
||||
}),
|
||||
collectMissingAdapterConfigRuntimeBindings: vi.fn(async () => []),
|
||||
};
|
||||
let settled = false;
|
||||
const opening = resolveExecutionRunAdapterConfig({
|
||||
companyId: "company-one", agentId: "agent-one", projectId: "project-one",
|
||||
environmentId: "environment-one", environmentEnv: { SHARED: "environment" },
|
||||
executionRunConfig: { env: { SHARED: "agent" } }, projectEnv: { SHARED: "project" }, secretsSvc,
|
||||
});
|
||||
void opening.then(() => { settled = true; }, () => { settled = true; });
|
||||
await vi.waitFor(() => expect(secretsSvc.collectMissingRuntimeBindings).toHaveBeenCalledTimes(3));
|
||||
expect(secretsSvc.collectMissingAdapterConfigRuntimeBindings).toHaveBeenCalledOnce();
|
||||
expect(settled).toBe(false);
|
||||
expect(secretsSvc.resolveAdapterConfigForRuntime).not.toHaveBeenCalled();
|
||||
expect(secretsSvc.resolveEnvBindings).not.toHaveBeenCalled();
|
||||
projectCheck.resolve([]);
|
||||
await expect(opening).rejects.toBe(firstFailure);
|
||||
});
|
||||
|
||||
it("keeps missing scoped credentials as a pre-dispatch failure", async () => {
|
||||
const secretsSvc = {
|
||||
resolveAdapterConfigForRuntime: vi.fn(),
|
||||
|
||||
@@ -394,76 +394,31 @@ export async function resolveExecutionRunAdapterConfig(input: {
|
||||
// no binding before resolving any secret value. Missing bindings short-circuit
|
||||
// to a configuration-incomplete blocker routed to a human owner instead of a
|
||||
// dispatched-then-failed run (which previously surfaced as opaque setup_failed).
|
||||
if (typeof input.secretsSvc.collectMissingRuntimeBindings === "function") {
|
||||
const missingBindings: MissingRuntimeBinding[] = [];
|
||||
if (environmentEnv && input.environmentId) {
|
||||
missingBindings.push(
|
||||
...(await input.secretsSvc.collectMissingRuntimeBindings(
|
||||
input.companyId,
|
||||
environmentEnv,
|
||||
{
|
||||
consumerType: "environment",
|
||||
consumerId: input.environmentId,
|
||||
responsibleUserId: input.responsibleUserId ?? null,
|
||||
},
|
||||
)),
|
||||
);
|
||||
}
|
||||
const collectMissing = input.secretsSvc.collectMissingRuntimeBindings;
|
||||
if (typeof collectMissing === "function") {
|
||||
const checks: Array<() => Promise<MissingRuntimeBinding[]>> = [];
|
||||
const checkEnv = (env: unknown, consumerType: "environment" | "agent" | "project" | "routine", consumerId: string) => {
|
||||
checks.push(() => collectMissing(input.companyId, env, {
|
||||
consumerType, consumerId, responsibleUserId: input.responsibleUserId ?? null,
|
||||
}));
|
||||
};
|
||||
if (environmentEnv && input.environmentId) checkEnv(environmentEnv, "environment", input.environmentId);
|
||||
if (input.agentId) {
|
||||
missingBindings.push(
|
||||
...(await input.secretsSvc.collectMissingRuntimeBindings(
|
||||
input.companyId,
|
||||
parseObject(executionRunConfig.env),
|
||||
{
|
||||
consumerType: "agent",
|
||||
consumerId: input.agentId,
|
||||
responsibleUserId: input.responsibleUserId ?? null,
|
||||
},
|
||||
)),
|
||||
);
|
||||
if (
|
||||
typeof input.secretsSvc.collectMissingAdapterConfigRuntimeBindings ===
|
||||
"function"
|
||||
) {
|
||||
missingBindings.push(
|
||||
...(await input.secretsSvc.collectMissingAdapterConfigRuntimeBindings(
|
||||
input.companyId,
|
||||
executionRunConfig,
|
||||
input.adapterType ?? null,
|
||||
{
|
||||
consumerType: "agent",
|
||||
consumerId: input.agentId,
|
||||
responsibleUserId: input.responsibleUserId ?? null,
|
||||
},
|
||||
)),
|
||||
);
|
||||
checkEnv(parseObject(executionRunConfig.env), "agent", input.agentId);
|
||||
const collectAdapterMissing = input.secretsSvc.collectMissingAdapterConfigRuntimeBindings;
|
||||
if (typeof collectAdapterMissing === "function") {
|
||||
checks.push(() => collectAdapterMissing(input.companyId, executionRunConfig, input.adapterType ?? null, {
|
||||
consumerType: "agent", consumerId: input.agentId!, responsibleUserId: input.responsibleUserId ?? null,
|
||||
}));
|
||||
}
|
||||
}
|
||||
if (projectEnv && input.projectId) {
|
||||
missingBindings.push(
|
||||
...(await input.secretsSvc.collectMissingRuntimeBindings(
|
||||
input.companyId,
|
||||
projectEnv,
|
||||
{
|
||||
consumerType: "project",
|
||||
consumerId: input.projectId,
|
||||
responsibleUserId: input.responsibleUserId ?? null,
|
||||
},
|
||||
)),
|
||||
);
|
||||
}
|
||||
if (routineEnv && input.routineId) {
|
||||
missingBindings.push(
|
||||
...(await input.secretsSvc.collectMissingRuntimeBindings(
|
||||
input.companyId,
|
||||
routineEnv,
|
||||
{
|
||||
consumerType: "routine",
|
||||
consumerId: input.routineId,
|
||||
responsibleUserId: input.responsibleUserId ?? null,
|
||||
},
|
||||
)),
|
||||
);
|
||||
if (projectEnv && input.projectId) checkEnv(projectEnv, "project", input.projectId);
|
||||
if (routineEnv && input.routineId) checkEnv(routineEnv, "routine", input.routineId);
|
||||
const results = await Promise.allSettled(checks.map(check => Promise.resolve().then(check)));
|
||||
const missingBindings: MissingRuntimeBinding[] = [];
|
||||
for (const result of results) {
|
||||
if (result.status === "rejected") throw result.reason;
|
||||
missingBindings.push(...result.value);
|
||||
}
|
||||
if (requiredScopedEnvBinding) {
|
||||
const requiredEnvKeys = new Set(requiredScopedEnvBinding.keys);
|
||||
@@ -2244,12 +2199,45 @@ export async function attestReviewedExternalChatRun(input: {
|
||||
throw new Error("reviewed_chat_execution_binding_not_ready");
|
||||
}
|
||||
|
||||
export async function readHeartbeatWakeComments(db: Db, companyId: string, issueId: string | null, commentIds: string[]) {
|
||||
if (commentIds.length === 0) return [];
|
||||
return db
|
||||
.select({
|
||||
id: issueComments.id,
|
||||
companyId: issueComments.companyId,
|
||||
issueId: issueComments.issueId,
|
||||
body: issueComments.body,
|
||||
origin: issueComments.origin,
|
||||
authorType: issueComments.authorType,
|
||||
authorAgentId: issueComments.authorAgentId,
|
||||
authorUserId: issueComments.authorUserId,
|
||||
presentation: issueComments.presentation,
|
||||
metadata: issueComments.metadata,
|
||||
deletedAt: issueComments.deletedAt,
|
||||
deletedByType: issueComments.deletedByType,
|
||||
deletedByAgentId: issueComments.deletedByAgentId,
|
||||
deletedByUserId: issueComments.deletedByUserId,
|
||||
deletedByRunId: issueComments.deletedByRunId,
|
||||
sourceTrust: issueComments.sourceTrust,
|
||||
createdAt: issueComments.createdAt,
|
||||
})
|
||||
.from(issueComments)
|
||||
.where(
|
||||
and(
|
||||
eq(issueComments.companyId, companyId),
|
||||
issueId ? eq(issueComments.issueId, issueId) : undefined,
|
||||
inArray(issueComments.id, commentIds),
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
export async function buildPaperclipWakePayload(input: {
|
||||
db: Db;
|
||||
companyId: string;
|
||||
agentId?: string | null;
|
||||
runId?: string | null;
|
||||
contextSnapshot: Record<string, unknown>;
|
||||
wakeCommentRows?: Awaited<ReturnType<typeof readHeartbeatWakeComments>>;
|
||||
continuationSummary?: {
|
||||
key: string;
|
||||
title: string | null;
|
||||
@@ -2314,36 +2302,9 @@ export async function buildPaperclipWakePayload(input: {
|
||||
)
|
||||
return null;
|
||||
|
||||
const commentRows =
|
||||
commentIds.length === 0
|
||||
? []
|
||||
: await input.db
|
||||
.select({
|
||||
id: issueComments.id,
|
||||
issueId: issueComments.issueId,
|
||||
body: issueComments.body,
|
||||
origin: issueComments.origin,
|
||||
authorType: issueComments.authorType,
|
||||
authorAgentId: issueComments.authorAgentId,
|
||||
authorUserId: issueComments.authorUserId,
|
||||
presentation: issueComments.presentation,
|
||||
metadata: issueComments.metadata,
|
||||
deletedAt: issueComments.deletedAt,
|
||||
deletedByType: issueComments.deletedByType,
|
||||
deletedByAgentId: issueComments.deletedByAgentId,
|
||||
deletedByUserId: issueComments.deletedByUserId,
|
||||
deletedByRunId: issueComments.deletedByRunId,
|
||||
sourceTrust: issueComments.sourceTrust,
|
||||
createdAt: issueComments.createdAt,
|
||||
})
|
||||
.from(issueComments)
|
||||
.where(
|
||||
and(
|
||||
eq(issueComments.companyId, input.companyId),
|
||||
issueId ? eq(issueComments.issueId, issueId) : undefined,
|
||||
inArray(issueComments.id, commentIds),
|
||||
),
|
||||
);
|
||||
const commentRows = input.wakeCommentRows
|
||||
? input.wakeCommentRows.filter(row => row.companyId === input.companyId && (!issueId || row.issueId === issueId) && commentIds.includes(row.id))
|
||||
: await readHeartbeatWakeComments(input.db, input.companyId, issueId, commentIds);
|
||||
|
||||
const commentsById = new Map(
|
||||
commentRows.map((comment) => [comment.id, comment]),
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { notifyStatusCardIssueChange } from "../status-card-notifications.js";
|
||||
import { DELIVERY_QUEUES, notifyDeliveryWork } from "../delivery-work-notifications.js";
|
||||
import { notifyIssueReviewState } from "../issue-review-state-notifications.js";
|
||||
import { insertIssueComment } from "../issue-comment-writes.js";
|
||||
@@ -595,6 +596,7 @@ export function createHeartbeatScheduling(
|
||||
try {
|
||||
await db.transaction(async (tx) => {
|
||||
await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors);
|
||||
await notifyStatusCardIssueChange(tx, claimed.companyId);
|
||||
const released = await tx.update(issues)
|
||||
.set({ monitorWakeRequestedAt: null, updatedAt: input.now })
|
||||
.where(issueMonitorClaimCondition(claimed)).returning({ id: issues.id });
|
||||
@@ -628,6 +630,7 @@ export function createHeartbeatScheduling(
|
||||
// failed, and never clobber a monitor that was replaced in the meantime.
|
||||
await db.transaction(async tx => {
|
||||
await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors);
|
||||
await notifyStatusCardIssueChange(tx, claimed.companyId);
|
||||
await tx.update(issues).set({ monitorWakeRequestedAt: null, updatedAt: input.now })
|
||||
.where(issueMonitorClaimCondition(claimed));
|
||||
});
|
||||
@@ -827,6 +830,7 @@ export function createHeartbeatScheduling(
|
||||
}
|
||||
|
||||
const consumed = await db.transaction(async tx => {
|
||||
await notifyStatusCardIssueChange(tx, claimed.companyId);
|
||||
await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors);
|
||||
return tx.update(issues)
|
||||
.set({
|
||||
@@ -977,6 +981,7 @@ export function createHeartbeatScheduling(
|
||||
|
||||
const staleClaimThreshold = new Date(now.getTime() - 5 * 60 * 1000);
|
||||
const claimed = await db.transaction(async (tx) => {
|
||||
await notifyStatusCardIssueChange(tx, issue.companyId);
|
||||
await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors);
|
||||
const [updated] = await tx
|
||||
.update(issues)
|
||||
@@ -1050,6 +1055,7 @@ export function createHeartbeatScheduling(
|
||||
for (const due of dueMonitors) {
|
||||
if (!canContinue()) break;
|
||||
const claimed = await db.transaction(async (tx) => {
|
||||
await notifyStatusCardIssueChange(tx, due.companyId);
|
||||
await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors);
|
||||
const [updated] = await tx
|
||||
.update(issues)
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import { createHash } from "node:crypto";
|
||||
import type { Db, agents, agentTaskSessions, heartbeatRuns, issueComments } from "@paperclipai/db";
|
||||
import type { Db, agents, agentTaskSessions, heartbeatRuns } from "@paperclipai/db";
|
||||
import { FAST_RESPONSE_AGENT_GUIDANCE, fastResponseHistoryBody } from "@paperclipai/shared";
|
||||
import type { PaperclipTurnContext } from "@paperclipai/adapter-utils/server-utils";
|
||||
import { parseObject } from "../../adapters/utils.js";
|
||||
@@ -15,8 +15,9 @@ import { getTaskPlanContext } from "../task-plan-context.js";
|
||||
import { caseExecutionMarkdown, getCaseExecutionContext } from "../case-context.js";
|
||||
import type { TrustPresetResolution } from "../trust-preset-resolver.js";
|
||||
import { liveVoiceExecutionGuidance } from "../voice/voice-execution-guidance.js";
|
||||
import { buildPaperclipWakePayload, ConfigurationIncompleteFailure, PAPERCLIP_WAKE_PAYLOAD_KEY, type createHeartbeatRunPreparation } from "./run-preparation.js";
|
||||
import { buildPaperclipWakePayload, ConfigurationIncompleteFailure, readHeartbeatWakeComments, PAPERCLIP_WAKE_PAYLOAD_KEY, type createHeartbeatRunPreparation } from "./run-preparation.js";
|
||||
import { buildPaperclipTaskMarkdown } from "./task-markdown.js";
|
||||
import { deriveCommentId, extractWakeCommentIds } from "../../modules/run-dispatch/index.js";
|
||||
import { workspacePreferenceService } from "../workspace-preferences.js";
|
||||
|
||||
type RunPreparation = ReturnType<typeof createHeartbeatRunPreparation>;
|
||||
@@ -30,8 +31,6 @@ export interface HeartbeatTaskContextInput {
|
||||
"id" | "identifier" | "title" | "status" | "priority" | "workMode" |
|
||||
"conversationAgentId" | "description" | "projectId"> | null;
|
||||
issueContext: IssueContext | null;
|
||||
wakeCommentContext: Pick<typeof issueComments.$inferSelect,
|
||||
"id" | "body" | "sourceTrust" | "presentation" | "metadata"> | null;
|
||||
context: Record<string, unknown>;
|
||||
taskSession: Pick<typeof agentTaskSessions.$inferSelect, "lastRunId"> | null;
|
||||
};
|
||||
@@ -53,35 +52,9 @@ function readNonEmptyString(value: unknown): string | null {
|
||||
/** Rebuilds task prompt fields on the run's context; acquires no execution resources. */
|
||||
export async function prepareHeartbeatTaskContext(db: Db, input: HeartbeatTaskContextInput) {
|
||||
const { run, agent } = input;
|
||||
const { issueRef, issueContext, wakeCommentContext, context, taskSession } = input.task;
|
||||
const { issueRef, issueContext, context, taskSession } = input.task;
|
||||
const { trustPreset, experimentalInstanceSettings } = input.policy;
|
||||
const { issuesSvc, getPinnedSkillTestContext } = input.services;
|
||||
const continuationSummary = issueRef && !isConversation(issueContext)
|
||||
? await getIssueContinuationSummaryDocument(db, issueRef.id)
|
||||
: null;
|
||||
const exposeLowTrustRaw = trustPreset.kind === "low_trust_review";
|
||||
const safeContinuationSummary =
|
||||
continuationSummary && !exposeLowTrustRaw
|
||||
? redactQuarantinedBodyForHigherTrust(continuationSummary)
|
||||
: continuationSummary;
|
||||
const safeWakeCommentContext =
|
||||
wakeCommentContext && !exposeLowTrustRaw
|
||||
? sanitizeQuarantinedCommentForHigherTrust(wakeCommentContext)
|
||||
: wakeCommentContext;
|
||||
const issueAncestors = issueRef
|
||||
? await issuesSvc.getAncestors(issueRef.id)
|
||||
: [];
|
||||
if (continuationSummary) {
|
||||
context.paperclipContinuationSummary = {
|
||||
key: safeContinuationSummary!.key,
|
||||
title: safeContinuationSummary!.title,
|
||||
body: safeContinuationSummary!.body,
|
||||
sourceTrust: safeContinuationSummary!.sourceTrust ?? null,
|
||||
updatedAt: safeContinuationSummary!.updatedAt.toISOString(),
|
||||
};
|
||||
} else {
|
||||
delete context.paperclipContinuationSummary;
|
||||
}
|
||||
const admittedScope = parseObject(context.paperclipAgentJwtScope);
|
||||
if (!input.policy.recovering || (admittedScope.kind !== "standard" && admittedScope.kind !== "skill_test")) {
|
||||
const savedIssue = parseObject(context.paperclipIssue);
|
||||
@@ -93,10 +66,58 @@ export async function prepareHeartbeatTaskContext(db: Db, input: HeartbeatTaskCo
|
||||
: { kind: "standard" };
|
||||
}
|
||||
const isSkillTest = parseObject(context.paperclipAgentJwtScope).kind === "skill_test";
|
||||
const pinnedSkillTestContext =
|
||||
isSkillTest && issueRef
|
||||
? await getPinnedSkillTestContext(agent.companyId, issueRef.id)
|
||||
: null;
|
||||
const wakeCommentId = deriveCommentId(context, null);
|
||||
const commentIds = [...new Set([...extractWakeCommentIds(context), ...(wakeCommentId ? [wakeCommentId] : [])])];
|
||||
const [summary, ancestors, pinnedSkillTestRead, wakeComments] = await Promise.allSettled([
|
||||
issueRef && !isConversation(issueContext) ? getIssueContinuationSummaryDocument(db, issueRef.id) : Promise.resolve(null),
|
||||
issueRef ? issuesSvc.getAncestors(issueRef.id) : Promise.resolve([]),
|
||||
isSkillTest && issueRef ? getPinnedSkillTestContext(agent.companyId, issueRef.id) : Promise.resolve(null),
|
||||
readHeartbeatWakeComments(db, agent.companyId, readNonEmptyString(context.issueId), commentIds),
|
||||
]);
|
||||
if (summary.status === "rejected") throw summary.reason;
|
||||
if (ancestors.status === "rejected") throw ancestors.reason;
|
||||
if (pinnedSkillTestRead.status === "rejected") throw pinnedSkillTestRead.reason;
|
||||
if (wakeComments.status === "rejected") throw wakeComments.reason;
|
||||
const wakeCommentRows = wakeComments.value;
|
||||
const selectedWakeComment = issueRef && wakeCommentId ? wakeCommentRows.find(comment => comment.id === wakeCommentId && comment.issueId === issueRef.id) ?? null : null;
|
||||
const wakeCommentContext = selectedWakeComment ? {
|
||||
id: selectedWakeComment.id,
|
||||
body: selectedWakeComment.deletedAt ? "" : selectedWakeComment.body,
|
||||
authorType: selectedWakeComment.authorType,
|
||||
authorAgentId: selectedWakeComment.authorAgentId,
|
||||
authorUserId: selectedWakeComment.authorUserId,
|
||||
presentation: selectedWakeComment.deletedAt ? null : selectedWakeComment.presentation,
|
||||
metadata: selectedWakeComment.deletedAt ? null : selectedWakeComment.metadata,
|
||||
deletedAt: selectedWakeComment.deletedAt,
|
||||
deletedByType: selectedWakeComment.deletedByType,
|
||||
deletedByAgentId: selectedWakeComment.deletedByAgentId,
|
||||
deletedByUserId: selectedWakeComment.deletedByUserId,
|
||||
deletedByRunId: selectedWakeComment.deletedByRunId,
|
||||
sourceTrust: selectedWakeComment.sourceTrust,
|
||||
} : null;
|
||||
const continuationSummary = summary.value;
|
||||
const issueAncestors = ancestors.value;
|
||||
const pinnedSkillTestContext = pinnedSkillTestRead.value;
|
||||
const exposeLowTrustRaw = trustPreset.kind === "low_trust_review";
|
||||
const safeContinuationSummary =
|
||||
continuationSummary && !exposeLowTrustRaw
|
||||
? redactQuarantinedBodyForHigherTrust(continuationSummary)
|
||||
: continuationSummary;
|
||||
const safeWakeCommentContext =
|
||||
wakeCommentContext && !exposeLowTrustRaw
|
||||
? sanitizeQuarantinedCommentForHigherTrust(wakeCommentContext)
|
||||
: wakeCommentContext;
|
||||
if (continuationSummary) {
|
||||
context.paperclipContinuationSummary = {
|
||||
key: safeContinuationSummary!.key,
|
||||
title: safeContinuationSummary!.title,
|
||||
body: safeContinuationSummary!.body,
|
||||
sourceTrust: safeContinuationSummary!.sourceTrust ?? null,
|
||||
updatedAt: safeContinuationSummary!.updatedAt.toISOString(),
|
||||
};
|
||||
} else {
|
||||
delete context.paperclipContinuationSummary;
|
||||
}
|
||||
if (isSkillTest && !pinnedSkillTestContext) {
|
||||
throw new ConfigurationIncompleteFailure("The skill test's saved version is unavailable.", {
|
||||
configurationIncomplete: { reason: "skill_unavailable", actionUrl: "/skills" },
|
||||
@@ -132,6 +153,7 @@ export async function prepareHeartbeatTaskContext(db: Db, input: HeartbeatTaskCo
|
||||
agentId: agent.id,
|
||||
runId: run.id,
|
||||
contextSnapshot: context,
|
||||
wakeCommentRows,
|
||||
continuationSummary,
|
||||
issueSummary: issueRef
|
||||
? {
|
||||
|
||||
@@ -1,3 +1,6 @@
|
||||
import { notifyStatusCardEvaluation } from "./status-card-notifications.js";
|
||||
import { notifyDeferredExecutionEligibility } from "./deferred-execution-notifications.js";
|
||||
import { notifyRunDispatch } from "./run-dispatch-notifications.js";
|
||||
import { notifySkillVersionRetentionReconstruction } from "./skill-version-retention-notifications.js";
|
||||
import { notifyNativeEnvironmentSettlement } from "./native-environment-settlement-notifications.js";
|
||||
import { notifyExternalAgentSettlement } from "../modules/external-agents/index.js";
|
||||
@@ -625,11 +628,14 @@ export function instanceSettingsService(db: Db, options: InstanceSettingsService
|
||||
}
|
||||
const now = new Date();
|
||||
const [updated] = await db.transaction(async tx => {
|
||||
if (patch.enableStatusCards !== undefined) await notifyStatusCardEvaluation(tx);
|
||||
await notifyDeliveryWork(tx, DELIVERY_QUEUES.mcpEvents, "*");
|
||||
await notifyDeliveryWork(tx, DELIVERY_QUEUES.dotEvents, "*");
|
||||
await notifyDeliveryWork(tx, DELIVERY_QUEUES.grokDispatch);
|
||||
if (patch.enableWorktreeRunExecution !== undefined) await notifyScheduledRetryCancellation(tx);
|
||||
if (patch.enableWorktreeRunExecution !== undefined) {
|
||||
await notifyRunDispatch(tx);
|
||||
await notifyDeferredExecutionEligibility(tx);
|
||||
await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers);
|
||||
await notifyExternalAgentSettlement(tx);
|
||||
await notifyInstructionCommit(tx);
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
import { notifyStatusCardIssueChange } from "./status-card-notifications.js";
|
||||
import { notifyIssueRecovery } from "./issue-recovery-notifications.js";
|
||||
import type { PgInsertValue } from "drizzle-orm/pg-core";
|
||||
import { issueComments, type Db } from "@paperclipai/db";
|
||||
import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js";
|
||||
@@ -6,6 +8,9 @@ type WriteDb = Db | Parameters<Parameters<Db["transaction"]>[0]>[0];
|
||||
|
||||
export function insertIssueComment(db: WriteDb, input: PgInsertValue<typeof issueComments> & { companyId: string }) {
|
||||
return db.transaction(async tx => {
|
||||
await notifyStatusCardIssueChange(tx, input.companyId);
|
||||
await notifyIssueRecovery(tx, { companyId: input.companyId,
|
||||
issueId: typeof input.issueId === "string" ? input.issueId : undefined });
|
||||
await notifyDeliveryWork(tx, DELIVERY_QUEUES.dotEvents, `company:${input.companyId}`);
|
||||
return tx.insert(issueComments).values(input).returning();
|
||||
});
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { notifyStatusCardIssueChange } from "./status-card-notifications.js";
|
||||
import { and, eq, sql } from "drizzle-orm";
|
||||
import type { PgColumn } from "drizzle-orm/pg-core";
|
||||
import type { Db } from "@paperclipai/db";
|
||||
@@ -127,6 +128,7 @@ export async function rekeyCompanyIssueIdentifiers(
|
||||
// overload, which returns NULL for a non-pattern argument.
|
||||
const tailStart = sql`${fromPrefix.length + 1}::int`;
|
||||
|
||||
await notifyStatusCardIssueChange(tx, companyId);
|
||||
const rekeyedIssues = await tx
|
||||
.update(issues)
|
||||
.set({ identifier: sql`${toPrefix} || substring(${issues.identifier} from ${tailStart})` })
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
import { notifyAttentionRetentionWork } from "./retention-work-notifications.js";
|
||||
import { notifyIssueRecovery } from "./issue-recovery-notifications.js";
|
||||
import { notifyNativeRecoveryNotices } from "./native-recovery-notice-notifications.js";
|
||||
import { notifyUnsafeWorkspaceExportRecovery } from "./unsafe-workspace-export-notifications.js";
|
||||
import { writeExecutionControl } from "./execution-control-notifications.js";
|
||||
@@ -322,6 +324,8 @@ export function issueRecoveryActionService(db: DbOrTransaction) {
|
||||
): Promise<IssueRecoveryAction> {
|
||||
try {
|
||||
const created = await db.transaction(async (tx) => {
|
||||
await notifyAttentionRetentionWork(tx, { companyId: input.companyId });
|
||||
await notifyIssueRecovery(tx, { companyId: input.companyId });
|
||||
await notifyNativeRecoveryNotices(tx, { companyId: input.companyId, issueId: input.sourceIssueId });
|
||||
await notifyDeliveryWork(tx, DELIVERY_QUEUES.executionDisposition);
|
||||
await notifyDeliveryWork(tx, DELIVERY_QUEUES.executionContinuation);
|
||||
@@ -413,6 +417,7 @@ export function issueRecoveryActionService(db: DbOrTransaction) {
|
||||
Math.min(nextAttemptCount, effectiveMaxAttempts),
|
||||
);
|
||||
const [exhausted] = await writeExecutionControl(db, [DELIVERY_QUEUES.executionDisposition, DELIVERY_QUEUES.executionContinuation], async tx => {
|
||||
await notifyIssueRecovery(tx, { companyId: input.companyId });
|
||||
await notifyNativeRecoveryNotices(tx, { companyId: input.companyId, issueId: input.sourceIssueId });
|
||||
if ((!input.preserveExistingOwner && existing.cause !== input.cause)
|
||||
|| (input.evidence?.runId !== undefined && input.evidence.runId !== existing.evidence?.runId)) {
|
||||
@@ -474,6 +479,7 @@ export function issueRecoveryActionService(db: DbOrTransaction) {
|
||||
return toReadModel(exhausted);
|
||||
}
|
||||
const [updated] = await writeExecutionControl(db, [DELIVERY_QUEUES.executionDisposition, DELIVERY_QUEUES.executionContinuation], async tx => {
|
||||
await notifyIssueRecovery(tx, { companyId: input.companyId });
|
||||
await notifyNativeRecoveryNotices(tx, { companyId: input.companyId, issueId: input.sourceIssueId });
|
||||
if ((!input.preserveExistingOwner && existing.cause !== input.cause)
|
||||
|| (input.evidence?.runId !== undefined && input.evidence.runId !== existing.evidence?.runId)) {
|
||||
@@ -548,6 +554,7 @@ export function issueRecoveryActionService(db: DbOrTransaction) {
|
||||
|
||||
try {
|
||||
const [created] = await writeExecutionControl(db, [DELIVERY_QUEUES.executionDisposition, DELIVERY_QUEUES.executionContinuation], async tx => {
|
||||
await notifyIssueRecovery(tx, { companyId: input.companyId });
|
||||
await notifyNativeRecoveryNotices(tx, { companyId: input.companyId, issueId: input.sourceIssueId });
|
||||
await notifyUnsafeWorkspaceExportRecovery(tx, { companyId: input.companyId, issueId: input.sourceIssueId });
|
||||
return tx
|
||||
@@ -595,6 +602,7 @@ export function issueRecoveryActionService(db: DbOrTransaction) {
|
||||
const pending = await tx.select({ id: issueRecoveryActions.id }).from(issueRecoveryActions)
|
||||
.where(and(...predicates)).limit(1);
|
||||
if (!pending.length) return [];
|
||||
await notifyIssueRecovery(tx, { companyId: input.companyId });
|
||||
await notifyNativeRecoveryNotices(tx, { companyId: input.companyId, issueId: input.sourceIssueId });
|
||||
await notifyUnsafeWorkspaceExportRecovery(tx, { companyId: input.companyId, issueId: input.sourceIssueId });
|
||||
return tx
|
||||
|
||||
@@ -0,0 +1,16 @@
|
||||
import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js";
|
||||
|
||||
/** An issue hint is reserved for local facts such as comments. Lifecycle,
|
||||
* graph, run and eligibility changes retain company scope because their
|
||||
* consequences can include ancestors and shared execution references. */
|
||||
export type IssueRecoveryScope = { companyId: string; issueId?: string };
|
||||
|
||||
export async function notifyIssueRecovery(transaction: object, scope?: IssueRecoveryScope) {
|
||||
const companyHint = scope ? JSON.stringify({ companyId: scope.companyId }) : "*";
|
||||
const issueHint = scope ? JSON.stringify({ companyId: scope.companyId, issueId: scope.issueId }) : "*";
|
||||
for (const queue of [DELIVERY_QUEUES.orphanBlockers, DELIVERY_QUEUES.activeRecoveryActions, DELIVERY_QUEUES.strandedAssignments, DELIVERY_QUEUES.resolvedDependencies,
|
||||
DELIVERY_QUEUES.taskWatchdogs, DELIVERY_QUEUES.staleIssueLocks]) {
|
||||
await notifyDeliveryWork(transaction, queue,
|
||||
queue === DELIVERY_QUEUES.strandedAssignments || queue === DELIVERY_QUEUES.staleIssueLocks ? issueHint : companyHint);
|
||||
}
|
||||
}
|
||||
@@ -1,4 +1,6 @@
|
||||
import { notifyRoutineExecutionStatus } from "./routine-execution-status-notifications.js";
|
||||
import { notifyStatusCardIssueChange } from "./status-card-notifications.js";
|
||||
import { notifyRunDispatch } from "./run-dispatch-notifications.js";
|
||||
import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js";
|
||||
import { notifyExternalConversationSettlement } from "./external-conversation-settlement-notifications.js";
|
||||
import { notifyNativeReviewResponse } from "./native-review-response-notifications.js";
|
||||
@@ -6,6 +8,8 @@ import { notifyNativeStatusReassessment } from "./native-status-reassessment-not
|
||||
|
||||
export async function notifyIssueReviewState(tx: object, scope: { companyId: string; issueId?: string; runId?: string; agentId?: string }) {
|
||||
await notifyRoutineExecutionStatus(tx, scope.issueId, scope.companyId);
|
||||
await notifyStatusCardIssueChange(tx, scope.companyId);
|
||||
await notifyRunDispatch(tx, scope);
|
||||
await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors);
|
||||
await notifyExternalConversationSettlement(tx, scope);
|
||||
await notifyNativeReviewResponse(tx, scope);
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { getTableName } from "drizzle-orm";
|
||||
import { PgDialect } from "drizzle-orm/pg-core";
|
||||
import { getTableName, type SQL } from "drizzle-orm";
|
||||
|
||||
const mockCreateChild = vi.fn();
|
||||
|
||||
@@ -48,8 +49,18 @@ function createFakeDb(args: {
|
||||
const toolActionRequestUpdates: Array<Record<string, unknown>> = [];
|
||||
const inserts: Array<{ table: string; values: Record<string, unknown> }> = [];
|
||||
let selectCallCount = 0;
|
||||
let transactionDepth = 0;
|
||||
|
||||
const db: any = {
|
||||
execute: vi.fn(async (query: SQL) => {
|
||||
const statement = new PgDialect().sqlToQuery(query);
|
||||
expect(transactionDepth).toBeGreaterThan(0);
|
||||
expect(statement.params).toContain(interactionRow.companyId);
|
||||
expect(statement.params).toContain(interactionRow.issueId);
|
||||
if (statement.sql.includes("origin_kind")) return [{ id: interactionRow.issueId, originKind: null, originId: null }];
|
||||
if (statement.sql.includes("with recursive affected")) return [{ issueId: interactionRow.issueId }];
|
||||
throw new Error(`Unexpected interaction fixture query: ${statement.sql}`);
|
||||
}),
|
||||
select: vi.fn(() => {
|
||||
selectCallCount += 1;
|
||||
const rows = selectCallCount === 1 ? [interactionRow] : (args.parentRows ?? []);
|
||||
@@ -84,7 +95,10 @@ function createFakeDb(args: {
|
||||
inserts.push({ table: getTableName(table as never), values });
|
||||
},
|
||||
})),
|
||||
transaction: async (callback: (tx: typeof db) => Promise<void>) => callback(db),
|
||||
transaction: async (callback: (tx: typeof db) => Promise<void>) => {
|
||||
transactionDepth++;
|
||||
try { return await callback(db); } finally { transactionDepth--; }
|
||||
},
|
||||
};
|
||||
|
||||
return {
|
||||
@@ -177,6 +191,7 @@ describe("issueThreadInteractionService", () => {
|
||||
};
|
||||
|
||||
let selectCallCount = 0;
|
||||
let transactionDepth = 0;
|
||||
const db: any = {
|
||||
select: vi.fn(() => {
|
||||
selectCallCount += 1;
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { notifyStatusCardIssueChange } from "./status-card-notifications.js";
|
||||
import { notifyNativeStatusReassessment } from "./native-status-reassessment-notifications.js";
|
||||
import { notifyIssueReviewState } from "./issue-review-state-notifications.js";
|
||||
import { notifyChatConfirmationResolution } from "./chat-confirmation-notifications.js";
|
||||
@@ -376,7 +377,7 @@ type ResolvedInteractionResult = {
|
||||
};
|
||||
|
||||
type IssueThreadInteractionRow = typeof issueThreadInteractions.$inferSelect;
|
||||
type IssueTouchDb = Pick<Db, "update">;
|
||||
type IssueTouchDb = Pick<Db, "update" | "transaction">;
|
||||
|
||||
function isNativeCompletionReview(
|
||||
row: Pick<IssueThreadInteractionRow, "kind" | "payload">,
|
||||
@@ -801,11 +802,11 @@ function hydrateInteraction(
|
||||
}
|
||||
}
|
||||
|
||||
async function touchIssue(db: IssueTouchDb, issueId: string) {
|
||||
await db
|
||||
.update(issues)
|
||||
.set({ updatedAt: new Date() })
|
||||
.where(eq(issues.id, issueId));
|
||||
async function touchIssue(db: IssueTouchDb, issueId: string, companyId: string) {
|
||||
await db.transaction(async tx => {
|
||||
await notifyStatusCardIssueChange(tx, companyId);
|
||||
await tx.update(issues).set({ updatedAt: new Date() }).where(eq(issues.id, issueId));
|
||||
});
|
||||
}
|
||||
|
||||
function isTerminalIssueStatus(status: string) {
|
||||
@@ -2013,7 +2014,7 @@ async function expireStaleRequestConfirmationTarget(
|
||||
"Interaction has already been resolved",
|
||||
);
|
||||
}
|
||||
await touchIssue(tx, args.row.issueId);
|
||||
await touchIssue(tx, args.row.issueId, args.row.companyId);
|
||||
const expired = hydrateInteraction(updated);
|
||||
await enqueueTerminalIssueInteractionChatPublications(tx, expired);
|
||||
return expired;
|
||||
@@ -2468,7 +2469,7 @@ export function issueThreadInteractionService(
|
||||
};
|
||||
}
|
||||
} else {
|
||||
await touchIssue(tx, args.issue.id);
|
||||
await touchIssue(tx, args.issue.id, args.issue.companyId);
|
||||
}
|
||||
|
||||
// Snapshot the continuation target only after every lifecycle mutation
|
||||
@@ -2682,7 +2683,7 @@ export function issueThreadInteractionService(
|
||||
tx,
|
||||
);
|
||||
} else {
|
||||
await touchIssue(tx, args.issue.id);
|
||||
await touchIssue(tx, args.issue.id, args.issue.companyId);
|
||||
}
|
||||
await enqueueTerminalIssueInteractionChatPublications(
|
||||
tx as unknown as Db,
|
||||
@@ -2864,7 +2865,7 @@ export function issueThreadInteractionService(
|
||||
await enqueueTerminalIssueInteractionChatPublications(tx as unknown as Db, hydrateInteraction(previous));
|
||||
}
|
||||
}
|
||||
await touchIssue(tx, issue.id);
|
||||
await touchIssue(tx, issue.id, issue.companyId);
|
||||
inserted = true;
|
||||
await enqueueIssueInteractionChatPublications(tx as unknown as Db, hydrateInteraction(row));
|
||||
await notifyDeliveryWork(tx, DELIVERY_QUEUES.connection);
|
||||
@@ -2908,7 +2909,7 @@ export function issueThreadInteractionService(
|
||||
)
|
||||
.returning();
|
||||
if (!updated) throw interactionAlreadyResolvedError();
|
||||
await touchIssue(db, issue.id);
|
||||
await touchIssue(db, issue.id, issue.companyId);
|
||||
return hydrateInteraction(updated) as ConnectionIntentInteraction;
|
||||
},
|
||||
resolveConnectionIntent: async (
|
||||
@@ -2972,7 +2973,7 @@ export function issueThreadInteractionService(
|
||||
return row;
|
||||
});
|
||||
if (!updated) throw interactionAlreadyResolvedError();
|
||||
await touchIssue(db, issue.id);
|
||||
await touchIssue(db, issue.id, issue.companyId);
|
||||
const interaction = hydrateInteraction(
|
||||
updated,
|
||||
) as ConnectionIntentInteraction;
|
||||
@@ -3351,7 +3352,7 @@ export function issueThreadInteractionService(
|
||||
})
|
||||
.where(eq(issueThreadInteractions.id, current.id))
|
||||
.returning();
|
||||
await touchIssue(tx, issue.id);
|
||||
await touchIssue(tx, issue.id, issue.companyId);
|
||||
return row;
|
||||
});
|
||||
return hydrateInteraction(updated);
|
||||
@@ -3414,10 +3415,8 @@ export function issueThreadInteractionService(
|
||||
if (updated) cancelled.push(updated);
|
||||
}
|
||||
|
||||
for (const issueId of new Set(
|
||||
cancelled.map((interaction) => interaction.issueId),
|
||||
)) {
|
||||
await touchIssue(db, issueId);
|
||||
for (const [issueId, companyId] of new Map(cancelled.map(interaction => [interaction.issueId, interaction.companyId]))) {
|
||||
await touchIssue(db, issueId, companyId);
|
||||
}
|
||||
await emitResolvedInteractionsTelemetry(db, cancelled);
|
||||
return cancelled;
|
||||
@@ -3506,10 +3505,8 @@ export function issueThreadInteractionService(
|
||||
}
|
||||
|
||||
if (expired.length > 0) {
|
||||
for (const issueId of new Set(
|
||||
expired.map((interaction) => interaction.issueId),
|
||||
)) {
|
||||
await touchIssue(db, issueId);
|
||||
for (const [issueId, companyId] of new Map(expired.map(interaction => [interaction.issueId, interaction.companyId]))) {
|
||||
await touchIssue(db, issueId, companyId);
|
||||
}
|
||||
await emitResolvedInteractionsTelemetry(db, expired);
|
||||
}
|
||||
@@ -4052,7 +4049,7 @@ export function issueThreadInteractionService(
|
||||
return interaction;
|
||||
}
|
||||
|
||||
await touchIssue(db, issue.id);
|
||||
await touchIssue(db, issue.id, issue.companyId);
|
||||
if (superseded.length > 0) {
|
||||
await emitResolvedInteractionsTelemetry(
|
||||
db,
|
||||
@@ -4357,7 +4354,7 @@ export function issueThreadInteractionService(
|
||||
.where(eq(issueThreadInteractions.id, interactionId))
|
||||
.returning();
|
||||
|
||||
await touchIssue(tx, issue.id);
|
||||
await touchIssue(tx, issue.id, issue.companyId);
|
||||
await mutationOptions.afterResolveInTransaction?.(tx, hydrateInteraction(updated));
|
||||
await enqueueTerminalIssueInteractionChatPublications(tx as unknown as Db, hydrateInteraction(updated));
|
||||
current.status = updated.status;
|
||||
@@ -4549,7 +4546,7 @@ export function issueThreadInteractionService(
|
||||
throw interactionAlreadyResolvedError();
|
||||
}
|
||||
|
||||
await touchIssue(tx, issue.id);
|
||||
await touchIssue(tx, issue.id, issue.companyId);
|
||||
await mutationOptions.afterResolveInTransaction?.(tx, hydrateInteraction(updated));
|
||||
if (complete) await enqueueTerminalIssueInteractionChatPublications(tx as unknown as Db, hydrateInteraction(updated));
|
||||
return {
|
||||
@@ -4622,7 +4619,7 @@ export function issueThreadInteractionService(
|
||||
throw interactionAlreadyResolvedError();
|
||||
}
|
||||
|
||||
await touchIssue(db, issue.id);
|
||||
await touchIssue(db, issue.id, issue.companyId);
|
||||
const rejected = hydrateInteraction(updated);
|
||||
await emitInteractionResolvedTelemetry(db, rejected);
|
||||
return rejected;
|
||||
@@ -4720,7 +4717,7 @@ export function issueThreadInteractionService(
|
||||
}
|
||||
|
||||
if (expired.length > 0) {
|
||||
await touchIssue(db, issue.id);
|
||||
await touchIssue(db, issue.id, issue.companyId);
|
||||
await emitResolvedInteractionsTelemetry(db, expired);
|
||||
}
|
||||
return expired;
|
||||
@@ -4952,7 +4949,7 @@ export function issueThreadInteractionService(
|
||||
}
|
||||
|
||||
if (expired.length > 0) {
|
||||
await touchIssue(db, issue.id);
|
||||
await touchIssue(db, issue.id, issue.companyId);
|
||||
await emitResolvedInteractionsTelemetry(db, expired);
|
||||
}
|
||||
return expired;
|
||||
@@ -5049,7 +5046,7 @@ export function issueThreadInteractionService(
|
||||
}
|
||||
|
||||
if (expired.length > 0) {
|
||||
await touchIssue(db, issue.id);
|
||||
await touchIssue(db, issue.id, issue.companyId);
|
||||
await emitResolvedInteractionsTelemetry(db, expired);
|
||||
}
|
||||
return expired;
|
||||
@@ -5189,7 +5186,7 @@ export function issueThreadInteractionService(
|
||||
if (updated) expired.push(hydrateInteraction(updated));
|
||||
}
|
||||
if (expired.length > 0) {
|
||||
await touchIssue(db, issue.id);
|
||||
await touchIssue(db, issue.id, issue.companyId);
|
||||
await emitResolvedInteractionsTelemetry(db, expired);
|
||||
}
|
||||
return expired;
|
||||
@@ -5294,7 +5291,7 @@ export function issueThreadInteractionService(
|
||||
return row;
|
||||
});
|
||||
|
||||
await touchIssue(db, issue.id);
|
||||
await touchIssue(db, issue.id, issue.companyId);
|
||||
const withdrawn = hydrateInteraction(updated);
|
||||
await emitInteractionResolvedTelemetry(db, withdrawn);
|
||||
return withdrawn;
|
||||
@@ -5417,7 +5414,7 @@ export function issueThreadInteractionService(
|
||||
return row;
|
||||
});
|
||||
|
||||
await touchIssue(db, issue.id);
|
||||
await touchIssue(db, issue.id, issue.companyId);
|
||||
const answered = hydrateInteraction(updated);
|
||||
await emitInteractionResolvedTelemetry(db, answered);
|
||||
return answered;
|
||||
@@ -5553,7 +5550,7 @@ export function issueThreadInteractionService(
|
||||
return row;
|
||||
});
|
||||
|
||||
await touchIssue(db, issue.id);
|
||||
await touchIssue(db, issue.id, issue.companyId);
|
||||
const skipped = hydrateInteraction(updated);
|
||||
await emitInteractionResolvedTelemetry(db, skipped);
|
||||
return skipped;
|
||||
@@ -5630,7 +5627,7 @@ export function issueThreadInteractionService(
|
||||
return row;
|
||||
});
|
||||
|
||||
await touchIssue(db, issue.id);
|
||||
await touchIssue(db, issue.id, issue.companyId);
|
||||
const cancelled = hydrateInteraction(updated);
|
||||
await emitInteractionResolvedTelemetry(db, cancelled);
|
||||
return cancelled;
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
import { notifyAttentionRetentionWork } from "./retention-work-notifications.js";
|
||||
import { notifyStatusCardIssueChange } from "./status-card-notifications.js";
|
||||
import { and, eq } from "drizzle-orm";
|
||||
import { heartbeatRuns, issues, type Db } from "@paperclipai/db";
|
||||
import { setIssueTitleSchema, type SetIssueTitle } from "@paperclipai/shared";
|
||||
@@ -76,6 +78,8 @@ export async function setIssueTitle(
|
||||
|| (issue.title === title && !issue.titleNeedsGeneration)) {
|
||||
return { result: await saveReceipt({ id: issue.id, title: issue.title, titleNeedsGeneration: issue.titleNeedsGeneration, changed: false }), publication: null };
|
||||
}
|
||||
await notifyAttentionRetentionWork(tx, { companyId });
|
||||
await notifyStatusCardIssueChange(tx, companyId);
|
||||
await tx.update(issues).set({ title, titleNeedsGeneration: false, updatedAt: new Date() }).where(predicate);
|
||||
await issueReferenceService(tx).syncIssue(issueId, tx);
|
||||
const { publication } = await persistActivity(tx, {
|
||||
|
||||
@@ -1,4 +1,7 @@
|
||||
import { notifyRoutineExecutionStatus } from "./routine-execution-status-notifications.js";
|
||||
import { notifyStatusCardIssueChange } from "./status-card-notifications.js";
|
||||
import { notifyDeferredExecutionEligibility } from "./deferred-execution-notifications.js";
|
||||
import { notifyIssueRecovery } from "./issue-recovery-notifications.js";
|
||||
import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js";
|
||||
import { notifyNativeStatusReassessment } from "./native-status-reassessment-notifications.js";
|
||||
import { notifyUnsafeWorkspaceExportRecovery } from "./unsafe-workspace-export-notifications.js";
|
||||
@@ -759,6 +762,8 @@ export function issueTreeControlService(db: Db) {
|
||||
const releaseReason = input.reason ?? "Subtree resume applied.";
|
||||
|
||||
const { hold: resumeHold } = await db.transaction(async (tx) => {
|
||||
await notifyIssueRecovery(tx, { companyId });
|
||||
await notifyDeferredExecutionEligibility(tx, { companyId });
|
||||
const [createdHold] = await tx
|
||||
.insert(issueTreeHolds)
|
||||
.values({
|
||||
@@ -837,6 +842,8 @@ export function issueTreeControlService(db: Db) {
|
||||
}
|
||||
|
||||
const { hold, members } = await db.transaction(async (tx) => {
|
||||
await notifyIssueRecovery(tx, { companyId });
|
||||
await notifyDeferredExecutionEligibility(tx, { companyId });
|
||||
const [createdHold] = await tx
|
||||
.insert(issueTreeHolds)
|
||||
.values({
|
||||
@@ -1001,6 +1008,7 @@ export function issueTreeControlService(db: Db) {
|
||||
const now = new Date();
|
||||
const releasedCancelHoldIds = activeCancelHolds.map((hold) => hold.id);
|
||||
const updatedIssues = await db.transaction(async (tx) => {
|
||||
await notifyDeferredExecutionEligibility(tx, { companyId });
|
||||
await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors);
|
||||
await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers);
|
||||
const restored: TreeStatusUpdateResult["updatedIssues"] = [];
|
||||
@@ -1013,6 +1021,7 @@ export function issueTreeControlService(db: Db) {
|
||||
await notifyChatResponseAvailability(tx, { companyId, issueId });
|
||||
if (status === "blocked" || status === "in_review") await notifyUnsafeWorkspaceExportRecovery(tx, { companyId, issueId });
|
||||
}
|
||||
await notifyStatusCardIssueChange(tx, companyId);
|
||||
const rows = await tx
|
||||
.update(issues)
|
||||
.set({
|
||||
@@ -1179,7 +1188,10 @@ export function issueTreeControlService(db: Db) {
|
||||
throw conflict("Issue tree hold is already released");
|
||||
}
|
||||
|
||||
const [updated] = await db
|
||||
const [updated] = await db.transaction(async tx => {
|
||||
await notifyIssueRecovery(tx, { companyId });
|
||||
await notifyDeferredExecutionEligibility(tx, { companyId });
|
||||
return tx
|
||||
.update(issueTreeHolds)
|
||||
.set({
|
||||
status: "released",
|
||||
@@ -1197,6 +1209,7 @@ export function issueTreeControlService(db: Db) {
|
||||
})
|
||||
.where(and(eq(issueTreeHolds.id, holdId), eq(issueTreeHolds.companyId, companyId)))
|
||||
.returning();
|
||||
});
|
||||
|
||||
const members = await db
|
||||
.select()
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import { nonIdleSlackIssueCondition } from "./slack-conversation-state.js";
|
||||
import { and, isNull, type SQL } from "drizzle-orm";
|
||||
import { and, isNull, isNotNull, or, type SQL } from "drizzle-orm";
|
||||
import { issues } from "@paperclipai/db";
|
||||
|
||||
export function visibleIssueCondition(): SQL {
|
||||
@@ -14,3 +14,9 @@ export function visibleIssueSql(alias = "issues") {
|
||||
export function executionIssueCondition(): SQL {
|
||||
return and(visibleIssueCondition(), isNull(issues.conversationAgentId), nonIdleSlackIssueCondition())!;
|
||||
}
|
||||
|
||||
export function attentionIssueCondition(includeConversations = false): SQL {
|
||||
return includeConversations
|
||||
? or(executionIssueCondition(), and(visibleIssueCondition(), isNotNull(issues.conversationAgentId)))!
|
||||
: executionIssueCondition();
|
||||
}
|
||||
+259
-170
@@ -1,4 +1,9 @@
|
||||
import { notifyRoutineExecutionStatus } from "./routine-execution-status-notifications.js";
|
||||
import { notifyIssueAttentionRetentionWork } from "./retention-work-notifications.js";
|
||||
import { notifyStatusCardIssueChange } from "./status-card-notifications.js";
|
||||
import { notifyDeferredExecutionEligibility } from "./deferred-execution-notifications.js";
|
||||
import { notifyIssueRecovery } from "./issue-recovery-notifications.js";
|
||||
import { notifyRunDispatch } from "./run-dispatch-notifications.js";
|
||||
import { notifyExternalConversationSettlement } from "./external-conversation-settlement-notifications.js";
|
||||
import { notifyNativeRecoveryNotices } from "./native-recovery-notice-notifications.js";
|
||||
import { notifyNativeRunProjection } from "./native-run-projection-notifications.js";
|
||||
@@ -2538,33 +2543,37 @@ async function listPendingFinalizeBlockerIssueIds(
|
||||
),
|
||||
);
|
||||
|
||||
if (lockSources) {
|
||||
const rows: Array<{ issueId: string | null; executionWorkspaceId: string | null;
|
||||
phase: string; status: string; startedAt: Date }> = [];
|
||||
for (const chunk of chunkList(executionWorkspaceIds, ISSUE_LIST_RELATED_QUERY_CHUNK_SIZE)) {
|
||||
if (lockSources) {
|
||||
|
||||
|
||||
await dbOrTx.select({ id: executionWorkspaces.id }).from(executionWorkspaces)
|
||||
.where(and(eq(executionWorkspaces.companyId, companyId), inArray(executionWorkspaces.id, executionWorkspaceIds)))
|
||||
.for("update", { noWait: true });
|
||||
}
|
||||
const query = dbOrTx
|
||||
.select({
|
||||
issueId: workspaceOperations.issueId,
|
||||
executionWorkspaceId: workspaceOperations.executionWorkspaceId,
|
||||
phase: workspaceOperations.phase,
|
||||
status: workspaceOperations.status,
|
||||
startedAt: workspaceOperations.startedAt,
|
||||
})
|
||||
.from(workspaceOperations)
|
||||
.where(
|
||||
and(
|
||||
eq(workspaceOperations.companyId, companyId),
|
||||
inArray(
|
||||
workspaceOperations.executionWorkspaceId,
|
||||
executionWorkspaceIds,
|
||||
await dbOrTx.select({ id: executionWorkspaces.id }).from(executionWorkspaces)
|
||||
.where(and(eq(executionWorkspaces.companyId, companyId), inArray(executionWorkspaces.id, chunk)))
|
||||
.for("update", { noWait: true });
|
||||
}
|
||||
const query = dbOrTx
|
||||
.select({
|
||||
issueId: workspaceOperations.issueId,
|
||||
executionWorkspaceId: workspaceOperations.executionWorkspaceId,
|
||||
phase: workspaceOperations.phase,
|
||||
status: workspaceOperations.status,
|
||||
startedAt: workspaceOperations.startedAt,
|
||||
})
|
||||
.from(workspaceOperations)
|
||||
.where(
|
||||
and(
|
||||
eq(workspaceOperations.companyId, companyId),
|
||||
inArray(
|
||||
workspaceOperations.executionWorkspaceId,
|
||||
chunk,
|
||||
),
|
||||
),
|
||||
),
|
||||
);
|
||||
);
|
||||
|
||||
const rows = await (lockSources ? query.for("share", { noWait: true }) : query);
|
||||
rows.push(...await (lockSources ? query.for("share", { noWait: true }) : query));
|
||||
}
|
||||
|
||||
const latestAttributedByBlockerWorkspace = new Map<
|
||||
string,
|
||||
@@ -3432,6 +3441,7 @@ type IssueBlockerAttentionEdge = {
|
||||
blockerIssueId: string;
|
||||
};
|
||||
type IssueBlockerAttentionQueryRow = IssueBlockerAttentionNode & {
|
||||
executionWorkspaceId?: string | null;
|
||||
issueId: string | null;
|
||||
blockerIssueId: string;
|
||||
};
|
||||
@@ -3820,11 +3830,15 @@ async function terminalExplicitBlockersByRoot(
|
||||
return terminalByRoot;
|
||||
}
|
||||
|
||||
async function listIssueBlockerAttentionMap(
|
||||
async function listIssueBlockerAttentionMap(dbOrTx: any, companyId: string, issueRows: IssueBlockerAttentionInputNode[]) {
|
||||
return (await readIssueBlockerAttentionBatch(dbOrTx, companyId, issueRows)).attention;
|
||||
}
|
||||
|
||||
async function readIssueBlockerAttentionBatch(
|
||||
dbOrTx: any,
|
||||
companyId: string,
|
||||
issueRows: IssueBlockerAttentionInputNode[],
|
||||
): Promise<Map<string, IssueBlockerAttention>> {
|
||||
) {
|
||||
const roots = issueRows.filter(
|
||||
(row) => row.companyId === companyId && row.status === "blocked",
|
||||
);
|
||||
@@ -3834,36 +3848,42 @@ async function listIssueBlockerAttentionMap(
|
||||
attentionMap.set(row.id, createIssueBlockerAttention());
|
||||
}
|
||||
}
|
||||
if (roots.length === 0) return attentionMap;
|
||||
const deferredRootIds = new Set<string>(roots.slice(BLOCKER_ATTENTION_MAX_NODES).map(root => root.id));
|
||||
let edgeCount = 0, fetchedRows = 0;
|
||||
const result = () => ({ attention: attentionMap, deferredRootIds, nodeCount: nodesById.size, edgeCount, fetchedRows });
|
||||
if (roots.length === 0) return { attention: attentionMap, deferredRootIds, nodeCount: 0, edgeCount, fetchedRows };
|
||||
|
||||
const nodesById = new Map<string, IssueBlockerAttentionNode>();
|
||||
const edgesByIssueId = new Map<string, IssueBlockerAttentionEdge[]>();
|
||||
for (const root of roots) nodesById.set(root.id, { ...root });
|
||||
// Returned rows are already bounded by the residual fetch budget. Keep their
|
||||
// positive live-work evidence even when they cannot enter the traversal graph.
|
||||
const observedNodes = new Map<string, IssueBlockerAttentionNode>();
|
||||
const observedChildrenByParent = new Map<string, Set<string>>();
|
||||
const admittedRoots = roots.slice(0, BLOCKER_ATTENTION_MAX_NODES);
|
||||
for (const root of admittedRoots) nodesById.set(root.id, { ...root });
|
||||
|
||||
let frontier = roots.map((root) => root.id);
|
||||
let truncated = false;
|
||||
type Traversal = { rootId: string; nodes: Set<string>; expanded: Set<string>; frontier: string[]; truncated: boolean };
|
||||
const traversals = new Map<string, Traversal>(admittedRoots.map(root => [root.id, {
|
||||
rootId: root.id, nodes: new Set([root.id]), expanded: new Set<string>(), frontier: [root.id], truncated: false,
|
||||
}]));
|
||||
const expanded = new Set<string>();
|
||||
const queried = new Set<string>();
|
||||
const pendingFinalizeBlockerIssueIds = new Set<string>();
|
||||
for (
|
||||
let depth = 0;
|
||||
frontier.length > 0 && depth < BLOCKER_ATTENTION_MAX_DEPTH;
|
||||
depth += 1
|
||||
) {
|
||||
const nextFrontier = new Set<string>();
|
||||
for (let depth = 0; depth < BLOCKER_ATTENTION_MAX_DEPTH; depth += 1) {
|
||||
const active = [...traversals.values()].filter(traversal => !traversal.truncated && traversal.frontier.length > 0);
|
||||
if (active.length === 0) break;
|
||||
const unread = new Set<string>();
|
||||
for (const traversal of active) for (const id of traversal.frontier) if (!expanded.has(id)) unread.add(id);
|
||||
const frontier = [...unread];
|
||||
|
||||
for (const chunk of chunkList(
|
||||
[...new Set(frontier)],
|
||||
frontier,
|
||||
ISSUE_LIST_RELATED_QUERY_CHUNK_SIZE,
|
||||
)) {
|
||||
const readinessByIssueId = await listIssueDependencyReadinessMap(
|
||||
dbOrTx,
|
||||
companyId,
|
||||
chunk,
|
||||
);
|
||||
for (const readiness of readinessByIssueId.values()) {
|
||||
for (const blockerIssueId of readiness.pendingFinalizeBlockerIssueIds) {
|
||||
pendingFinalizeBlockerIssueIds.add(blockerIssueId);
|
||||
}
|
||||
}
|
||||
const remaining = Math.min(BLOCKER_ATTENTION_MAX_NODES - nodesById.size,
|
||||
BLOCKER_ATTENTION_MAX_NODES * 2 - edgeCount);
|
||||
if (remaining <= 0) break;
|
||||
const rowLimit = remaining + 1;
|
||||
const explicitBlockerRowsPromise: Promise<
|
||||
IssueBlockerAttentionQueryRow[]
|
||||
> = dbOrTx
|
||||
@@ -3877,6 +3897,7 @@ async function listIssueBlockerAttentionMap(
|
||||
title: issues.title,
|
||||
status: issues.status,
|
||||
executionRunId: issues.executionRunId,
|
||||
executionWorkspaceId: issues.executionWorkspaceId,
|
||||
assigneeAgentId: issues.assigneeAgentId,
|
||||
assigneeUserId: issues.assigneeUserId,
|
||||
})
|
||||
@@ -3889,7 +3910,7 @@ async function listIssueBlockerAttentionMap(
|
||||
inArray(issueRelations.relatedIssueId, chunk),
|
||||
eq(issues.companyId, companyId),
|
||||
),
|
||||
);
|
||||
).limit(rowLimit);
|
||||
const childRowsPromise: Promise<IssueBlockerAttentionQueryRow[]> = dbOrTx
|
||||
.select({
|
||||
issueId: issues.parentId,
|
||||
@@ -3914,68 +3935,83 @@ async function listIssueBlockerAttentionMap(
|
||||
BLOCKER_ATTENTION_CHILD_TERMINAL_STATUSES,
|
||||
),
|
||||
),
|
||||
);
|
||||
).limit(rowLimit);
|
||||
const [explicitBlockerRows, childRows] = await Promise.all([
|
||||
explicitBlockerRowsPromise,
|
||||
childRowsPromise,
|
||||
]);
|
||||
|
||||
for (const id of chunk) queried.add(id);
|
||||
fetchedRows += explicitBlockerRows.length + childRows.length;
|
||||
const pendingFinalize = await listPendingFinalizeBlockerIssueIds(dbOrTx, companyId,
|
||||
explicitBlockerRows.flatMap(row => row.status === "done" && row.executionWorkspaceId
|
||||
? [{ blockerIssueId: row.blockerIssueId, executionWorkspaceId: row.executionWorkspaceId }] : []));
|
||||
for (const id of pendingFinalize) pendingFinalizeBlockerIssueIds.add(id);
|
||||
let complete = explicitBlockerRows.length < rowLimit && childRows.length < rowLimit;
|
||||
const unresolvedExplicitBlockerRows = explicitBlockerRows.filter(
|
||||
(row) =>
|
||||
row.status !== "done" ||
|
||||
pendingFinalizeBlockerIssueIds.has(row.blockerIssueId),
|
||||
);
|
||||
appendBlockerAttentionEdges(edgesByIssueId, [
|
||||
...unresolvedExplicitBlockerRows
|
||||
.filter(
|
||||
(row): row is IssueBlockerAttentionQueryRow & { issueId: string } =>
|
||||
row.issueId !== null,
|
||||
)
|
||||
.map((row) => ({
|
||||
issueId: row.issueId,
|
||||
blockerIssueId: row.blockerIssueId,
|
||||
})),
|
||||
...childRows
|
||||
.filter(
|
||||
(row): row is IssueBlockerAttentionQueryRow & { issueId: string } =>
|
||||
row.issueId !== null,
|
||||
)
|
||||
.map((row) => ({
|
||||
issueId: row.issueId,
|
||||
blockerIssueId: row.blockerIssueId,
|
||||
})),
|
||||
]);
|
||||
|
||||
for (const row of [...unresolvedExplicitBlockerRows, ...childRows]) {
|
||||
if (!row.issueId || nodesById.has(row.blockerIssueId)) continue;
|
||||
nodesById.set(row.blockerIssueId, {
|
||||
id: row.blockerIssueId,
|
||||
companyId: row.companyId,
|
||||
parentId: row.parentId,
|
||||
identifier: row.identifier,
|
||||
title: row.title,
|
||||
status: row.status,
|
||||
executionRunId: row.executionRunId,
|
||||
assigneeAgentId: row.assigneeAgentId,
|
||||
assigneeUserId: row.assigneeUserId,
|
||||
});
|
||||
nextFrontier.add(row.blockerIssueId);
|
||||
if (!row.issueId) continue;
|
||||
observedNodes.set(row.blockerIssueId, row);
|
||||
const observed = observedChildrenByParent.get(row.issueId) ?? new Set<string>();
|
||||
observed.add(row.blockerIssueId);
|
||||
observedChildrenByParent.set(row.issueId, observed);
|
||||
if (!nodesById.has(row.blockerIssueId)) {
|
||||
if (nodesById.size >= BLOCKER_ATTENTION_MAX_NODES) { complete = false; continue; }
|
||||
nodesById.set(row.blockerIssueId, {
|
||||
id: row.blockerIssueId, companyId: row.companyId, parentId: row.parentId, identifier: row.identifier,
|
||||
title: row.title, status: row.status, executionRunId: row.executionRunId,
|
||||
assigneeAgentId: row.assigneeAgentId, assigneeUserId: row.assigneeUserId,
|
||||
});
|
||||
}
|
||||
const edges = edgesByIssueId.get(row.issueId) ?? [];
|
||||
if (edges.some(edge => edge.blockerIssueId === row.blockerIssueId)) continue;
|
||||
if (edgeCount >= BLOCKER_ATTENTION_MAX_NODES * 2) { complete = false; continue; }
|
||||
edges.push({ issueId: row.issueId, blockerIssueId: row.blockerIssueId });
|
||||
edgesByIssueId.set(row.issueId, edges);
|
||||
edgeCount++;
|
||||
}
|
||||
if (complete) for (const id of chunk) expanded.add(id);
|
||||
|
||||
}
|
||||
|
||||
if (nodesById.size > BLOCKER_ATTENTION_MAX_NODES) {
|
||||
truncated = true;
|
||||
break;
|
||||
for (const traversal of active) {
|
||||
if (traversal.frontier.some(id => !expanded.has(id))) {
|
||||
for (const id of traversal.frontier) if (queried.has(id)) {
|
||||
traversal.expanded.add(id);
|
||||
for (const edge of edgesByIssueId.get(id) ?? []) if (nodesById.has(edge.blockerIssueId)) traversal.nodes.add(edge.blockerIssueId);
|
||||
}
|
||||
traversal.truncated = true;
|
||||
traversal.frontier = [];
|
||||
deferredRootIds.add(traversal.rootId);
|
||||
continue;
|
||||
}
|
||||
const nextFrontier = new Set<string>();
|
||||
for (const id of traversal.frontier) {
|
||||
traversal.expanded.add(id);
|
||||
for (const edge of edgesByIssueId.get(id) ?? []) {
|
||||
if (traversal.nodes.has(edge.blockerIssueId) || !nodesById.has(edge.blockerIssueId)) continue;
|
||||
traversal.nodes.add(edge.blockerIssueId);
|
||||
nextFrontier.add(edge.blockerIssueId);
|
||||
}
|
||||
}
|
||||
traversal.truncated = traversal.nodes.size > BLOCKER_ATTENTION_MAX_NODES;
|
||||
traversal.frontier = traversal.truncated ? [] : [...nextFrontier];
|
||||
}
|
||||
frontier = [...nextFrontier];
|
||||
}
|
||||
if (frontier.length > 0) truncated = true;
|
||||
for (const traversal of traversals.values()) {
|
||||
if (traversal.frontier.length > 0) traversal.truncated = true;
|
||||
}
|
||||
|
||||
const nodeIds = [...nodesById.keys()];
|
||||
const evidenceNodes = new Map([...nodesById, ...observedNodes]);
|
||||
const nodeIds = [...evidenceNodes.keys()];
|
||||
const activeIssueIds = new Set<string>();
|
||||
const agentIds = new Set<string>();
|
||||
const issueIdByExecutionRunId = new Map<string, string>();
|
||||
for (const node of nodesById.values()) {
|
||||
for (const node of evidenceNodes.values()) {
|
||||
if (node.assigneeAgentId) agentIds.add(node.assigneeAgentId);
|
||||
if (node.executionRunId)
|
||||
issueIdByExecutionRunId.set(node.executionRunId, node.id);
|
||||
@@ -4033,7 +4069,7 @@ async function listIssueBlockerAttentionMap(
|
||||
}
|
||||
}
|
||||
|
||||
const explicitWaitCandidateIds = [...nodesById.values()]
|
||||
const explicitWaitCandidateIds = [...evidenceNodes.values()]
|
||||
.filter((node) => node.status !== "done")
|
||||
.map((node) => node.id);
|
||||
const explicitWaitingIssueIds = new Set<string>();
|
||||
@@ -4108,23 +4144,26 @@ async function listIssueBlockerAttentionMap(
|
||||
ownerType: string;
|
||||
ownerAgentId: string | null;
|
||||
ownerUserId: string | null;
|
||||
}> = await dbOrTx
|
||||
.select({
|
||||
id: issueRecoveryActions.id,
|
||||
sourceIssueId: issueRecoveryActions.sourceIssueId,
|
||||
status: issueRecoveryActions.status,
|
||||
ownerType: issueRecoveryActions.ownerType,
|
||||
ownerAgentId: issueRecoveryActions.ownerAgentId,
|
||||
ownerUserId: issueRecoveryActions.ownerUserId,
|
||||
})
|
||||
.from(issueRecoveryActions)
|
||||
.where(
|
||||
and(
|
||||
eq(issueRecoveryActions.companyId, companyId),
|
||||
inArray(issueRecoveryActions.status, ["active", "escalated"]),
|
||||
inArray(issueRecoveryActions.sourceIssueId, explicitWaitCandidateIds),
|
||||
),
|
||||
);
|
||||
}> = [];
|
||||
for (const chunk of chunkList(explicitWaitCandidateIds, ISSUE_LIST_RELATED_QUERY_CHUNK_SIZE)) {
|
||||
recoveryActionRows.push(...await dbOrTx
|
||||
.select({
|
||||
id: issueRecoveryActions.id,
|
||||
sourceIssueId: issueRecoveryActions.sourceIssueId,
|
||||
status: issueRecoveryActions.status,
|
||||
ownerType: issueRecoveryActions.ownerType,
|
||||
ownerAgentId: issueRecoveryActions.ownerAgentId,
|
||||
ownerUserId: issueRecoveryActions.ownerUserId,
|
||||
})
|
||||
.from(issueRecoveryActions)
|
||||
.where(
|
||||
and(
|
||||
eq(issueRecoveryActions.companyId, companyId),
|
||||
inArray(issueRecoveryActions.status, ["active", "escalated"]),
|
||||
inArray(issueRecoveryActions.sourceIssueId, chunk),
|
||||
),
|
||||
));
|
||||
}
|
||||
const recoveryActionIds = recoveryActionRows.map((row) => row.id);
|
||||
const liveRecoveryActionIds = new Set<string>();
|
||||
for (const chunk of chunkList(
|
||||
@@ -4187,22 +4226,11 @@ async function listIssueBlockerAttentionMap(
|
||||
}
|
||||
}
|
||||
|
||||
const agentRows: IssueBlockerAttentionAgentRow[] =
|
||||
agentIds.size > 0
|
||||
? await dbOrTx
|
||||
.select({
|
||||
id: agents.id,
|
||||
companyId: agents.companyId,
|
||||
status: agents.status,
|
||||
})
|
||||
.from(agents)
|
||||
.where(
|
||||
and(
|
||||
eq(agents.companyId, companyId),
|
||||
inArray(agents.id, [...agentIds]),
|
||||
),
|
||||
)
|
||||
: [];
|
||||
const agentRows: IssueBlockerAttentionAgentRow[] = [];
|
||||
for (const chunk of chunkList([...agentIds], ISSUE_LIST_RELATED_QUERY_CHUNK_SIZE)) {
|
||||
agentRows.push(...await dbOrTx.select({ id: agents.id, companyId: agents.companyId, status: agents.status })
|
||||
.from(agents).where(and(eq(agents.companyId, companyId), inArray(agents.id, chunk))));
|
||||
}
|
||||
const agentsById = new Map(agentRows.map((agent) => [agent.id, agent]));
|
||||
|
||||
type PathClassification = {
|
||||
@@ -4215,6 +4243,7 @@ async function listIssueBlockerAttentionMap(
|
||||
const classifyPath = (
|
||||
nodeId: string,
|
||||
seen: Set<string>,
|
||||
truncated: boolean,
|
||||
): PathClassification => {
|
||||
const sample = blockerSampleIdentifier(nodesById.get(nodeId));
|
||||
if (truncated || seen.has(nodeId)) {
|
||||
@@ -4319,7 +4348,7 @@ async function listIssueBlockerAttentionMap(
|
||||
const nextSeen = new Set(seen);
|
||||
nextSeen.add(nodeId);
|
||||
const classified = downstream.map((edge) =>
|
||||
classifyPath(edge.blockerIssueId, nextSeen),
|
||||
classifyPath(edge.blockerIssueId, nextSeen, truncated),
|
||||
);
|
||||
const stalledChild = classified.find(
|
||||
(result) => result.stalled || result.sampleStalledBlockerIdentifier,
|
||||
@@ -4385,18 +4414,22 @@ async function listIssueBlockerAttentionMap(
|
||||
};
|
||||
};
|
||||
|
||||
const pathHasLiveWork = (nodeId: string, seen: Set<string>): boolean => {
|
||||
if (seen.has(nodeId)) return false;
|
||||
const nodeHasLiveWork = (node: IssueBlockerAttentionNode) =>
|
||||
node.status === "in_progress" || activeIssueIds.has(node.id) ||
|
||||
explicitWaitingIssueIds.has(node.id) || Boolean(node.assigneeUserId);
|
||||
const observedChildHasLiveWork = (parentId: string) =>
|
||||
[...observedChildrenByParent.get(parentId) ?? []].some(id => {
|
||||
const node = observedNodes.get(id);
|
||||
return node?.companyId === companyId && nodeHasLiveWork(node);
|
||||
});
|
||||
const pathHasLiveWork = (nodeId: string, seen: Set<string>, traversal: Traversal): boolean => {
|
||||
if (seen.has(nodeId) || !traversal.nodes.has(nodeId)) return false;
|
||||
const node = nodesById.get(nodeId);
|
||||
if (!node || node.companyId !== companyId) return false;
|
||||
if (
|
||||
node.status === "in_progress" ||
|
||||
activeIssueIds.has(node.id) ||
|
||||
explicitWaitingIssueIds.has(node.id) ||
|
||||
Boolean(node.assigneeUserId)
|
||||
)
|
||||
return true;
|
||||
if (nodeHasLiveWork(node)) return true;
|
||||
|
||||
if (!traversal.expanded.has(nodeId)) return false;
|
||||
if (observedChildHasLiveWork(nodeId)) return true;
|
||||
const nextSeen = new Set(seen);
|
||||
nextSeen.add(nodeId);
|
||||
return (edgesByIssueId.get(node.id) ?? []).some((edge) => {
|
||||
@@ -4406,7 +4439,7 @@ async function listIssueBlockerAttentionMap(
|
||||
!pendingFinalizeBlockerIssueIds.has(edge.blockerIssueId)
|
||||
)
|
||||
return false;
|
||||
return pathHasLiveWork(edge.blockerIssueId, nextSeen);
|
||||
return pathHasLiveWork(edge.blockerIssueId, nextSeen, traversal);
|
||||
});
|
||||
};
|
||||
|
||||
@@ -4419,6 +4452,7 @@ async function listIssueBlockerAttentionMap(
|
||||
};
|
||||
|
||||
for (const root of roots) {
|
||||
const traversal = traversals.get(root.id)!;
|
||||
const topLevelEdges = (edgesByIssueId.get(root.id) ?? []).filter((edge) => {
|
||||
const blocker = nodesById.get(edge.blockerIssueId);
|
||||
return (
|
||||
@@ -4440,7 +4474,7 @@ async function listIssueBlockerAttentionMap(
|
||||
|
||||
const classified = topLevelEdges.map((edge) => ({
|
||||
edge,
|
||||
result: classifyPath(edge.blockerIssueId, new Set([root.id])),
|
||||
result: classifyPath(edge.blockerIssueId, new Set([root.id]), traversal.truncated),
|
||||
}));
|
||||
const coveredBlockerCount = classified.filter(
|
||||
(entry) => entry.result.covered,
|
||||
@@ -4522,8 +4556,8 @@ async function listIssueBlockerAttentionMap(
|
||||
stalledEntry?.result.sampleStalledBlockerIdentifier ??
|
||||
sampleStalledFromChain ??
|
||||
null,
|
||||
blockingTreeLive: topLevelEdges.some((edge) =>
|
||||
pathHasLiveWork(edge.blockerIssueId, new Set([root.id])),
|
||||
blockingTreeLive: observedChildHasLiveWork(root.id) || topLevelEdges.some((edge) =>
|
||||
pathHasLiveWork(edge.blockerIssueId, new Set([root.id]), traversal),
|
||||
),
|
||||
directBlockerIssueId: sampleEntry?.edge.blockerIssueId ?? null,
|
||||
terminalBlockerIssueId,
|
||||
@@ -4538,7 +4572,7 @@ async function listIssueBlockerAttentionMap(
|
||||
);
|
||||
}
|
||||
|
||||
return attentionMap;
|
||||
return result();
|
||||
}
|
||||
|
||||
type IssueReviewAttentionInput = Pick<IssueRow, "id" | "companyId" | "status">;
|
||||
@@ -7598,7 +7632,7 @@ export function issueService(db: Db) {
|
||||
companyId: string,
|
||||
blockedByIssueIds: string[],
|
||||
actor: { agentId?: string | null; userId?: string | null } = {},
|
||||
dbOrTx: any = db,
|
||||
dbOrTx: any,
|
||||
) {
|
||||
const deduped = [...new Set(blockedByIssueIds)];
|
||||
if (deduped.some((candidate) => candidate === issueId)) {
|
||||
@@ -7627,6 +7661,9 @@ export function issueService(db: Db) {
|
||||
await assertNoBlockingCycles(companyId, issueId, deduped, dbOrTx);
|
||||
}
|
||||
|
||||
await notifyIssueAttentionRetentionWork(dbOrTx, { companyId, issueId });
|
||||
await notifyIssueRecovery(dbOrTx, { companyId });
|
||||
await notifyDeferredExecutionEligibility(dbOrTx, { companyId });
|
||||
await dbOrTx
|
||||
.delete(issueRelations)
|
||||
.where(
|
||||
@@ -7730,6 +7767,7 @@ export function issueService(db: Db) {
|
||||
}
|
||||
|
||||
const now = new Date();
|
||||
await notifyStatusCardIssueChange(tx, lockedIssue.companyId);
|
||||
const adopted = await tx
|
||||
.update(issues)
|
||||
.set({
|
||||
@@ -7783,13 +7821,14 @@ export function issueService(db: Db) {
|
||||
sql`select ${heartbeatRuns.id} from ${heartbeatRuns} where ${heartbeatRuns.id} = ${input.actorRunId} for update`,
|
||||
);
|
||||
const actorRun = await tx
|
||||
.select({ status: heartbeatRuns.status })
|
||||
.select({ status: heartbeatRuns.status, companyId: heartbeatRuns.companyId })
|
||||
.from(heartbeatRuns)
|
||||
.where(eq(heartbeatRuns.id, input.actorRunId))
|
||||
.then((rows) => rows[0] ?? null);
|
||||
if (!actorRun || TERMINAL_HEARTBEAT_RUN_STATUSES.has(actorRun.status))
|
||||
return null;
|
||||
|
||||
await notifyStatusCardIssueChange(tx, actorRun.companyId);
|
||||
const now = new Date();
|
||||
const adopted = await tx
|
||||
.update(issues)
|
||||
@@ -9372,6 +9411,9 @@ export function issueService(db: Db) {
|
||||
return listIssueDependencyReadinessMap(dbOrTx, companyId, issueIds);
|
||||
},
|
||||
|
||||
listBlockerAttentionBatch: async (companyId: string, issueRows: IssueBlockerAttentionInputNode[], dbOrTx: any = db) =>
|
||||
readIssueBlockerAttentionBatch(dbOrTx, companyId, issueRows),
|
||||
|
||||
listBlockerAttention: async (
|
||||
companyId: string,
|
||||
issueRows: IssueBlockerAttentionInputNode[],
|
||||
@@ -9675,27 +9717,32 @@ export function issueService(db: Db) {
|
||||
});
|
||||
|
||||
if (blockParentUntilDone) {
|
||||
const existingBlockers = await db
|
||||
.select({ blockerIssueId: issueRelations.issueId })
|
||||
.from(issueRelations)
|
||||
.where(
|
||||
and(
|
||||
eq(issueRelations.companyId, parent.companyId),
|
||||
eq(issueRelations.relatedIssueId, parent.id),
|
||||
eq(issueRelations.type, "blocks"),
|
||||
),
|
||||
await db.transaction(async tx => {
|
||||
await tx.select({ id: issues.id }).from(issues).where(and(
|
||||
eq(issues.companyId, parent.companyId), eq(issues.id, parent.id))).for("update");
|
||||
const existingBlockers = await tx
|
||||
.select({ blockerIssueId: issueRelations.issueId })
|
||||
.from(issueRelations)
|
||||
.where(
|
||||
and(
|
||||
eq(issueRelations.companyId, parent.companyId),
|
||||
eq(issueRelations.relatedIssueId, parent.id),
|
||||
eq(issueRelations.type, "blocks"),
|
||||
),
|
||||
);
|
||||
await syncBlockedByIssueIds(
|
||||
parent.id,
|
||||
parent.companyId,
|
||||
[
|
||||
...new Set([
|
||||
...existingBlockers.map((row) => row.blockerIssueId),
|
||||
child.id,
|
||||
]),
|
||||
],
|
||||
{ agentId: actorAgentId ?? null, userId: actorUserId ?? null },
|
||||
tx,
|
||||
);
|
||||
await syncBlockedByIssueIds(
|
||||
parent.id,
|
||||
parent.companyId,
|
||||
[
|
||||
...new Set([
|
||||
...existingBlockers.map((row) => row.blockerIssueId),
|
||||
child.id,
|
||||
]),
|
||||
],
|
||||
{ agentId: actorAgentId ?? null, userId: actorUserId ?? null },
|
||||
);
|
||||
});
|
||||
[child] = await withIssueRelationSummaries(
|
||||
parent.companyId,
|
||||
[child],
|
||||
@@ -10606,7 +10653,12 @@ export function issueService(db: Db) {
|
||||
: values.parentId ? `issue:${values.parentId}` : undefined);
|
||||
}
|
||||
if (values.monitorNextCheckAt) await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors);
|
||||
await notifyRunDispatch(tx, { companyId: values.companyId });
|
||||
await notifyDeferredExecutionEligibility(tx, { companyId: values.companyId });
|
||||
await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers);
|
||||
values.id ??= randomUUID();
|
||||
await notifyIssueAttentionRetentionWork(tx, { companyId, issueId: values.id, parentId: values.parentId, originKind: values.originKind, originId: values.originId });
|
||||
await notifyStatusCardIssueChange(tx, companyId);
|
||||
if (values.originKind === "routine_execution") await notifyRoutineExecutionStatus(tx, values.id, companyId);
|
||||
const [issue] = await tx.insert(issues).values(values).returning();
|
||||
await attachOwnedDraftImages(tx, issue, issue.description, { userId: issueData.createdByUserId, agentId: issueData.createdByAgentId });
|
||||
@@ -10892,6 +10944,7 @@ export function issueService(db: Db) {
|
||||
}
|
||||
}
|
||||
|
||||
await notifyStatusCardIssueChange(tx, companyId);
|
||||
await insertRowsInChunks(tx, issues, issueRows);
|
||||
await insertRowsInChunks(tx, issueLabels, labelRows);
|
||||
});
|
||||
@@ -10908,6 +10961,7 @@ export function issueService(db: Db) {
|
||||
const censorUsernameInLogs = (await instanceSettings.getGeneral())
|
||||
.censorUsernameInLogs;
|
||||
await db.transaction(async (tx) => {
|
||||
for (const companyId of new Set(rows.map(row => row.companyId))) await notifyStatusCardIssueChange(tx, companyId);
|
||||
const commentRows = rows.map((row) => {
|
||||
const createdAt = row.createdAt ? new Date(row.createdAt) : null;
|
||||
return {
|
||||
@@ -11488,13 +11542,19 @@ export function issueService(db: Db) {
|
||||
(blockedByIssueIds !== undefined || issueData.unblockDescriptor !== undefined)))) {
|
||||
patch.statusVersion = sql`${issues.statusVersion} + 1` as unknown as number;
|
||||
}
|
||||
await notifyStatusCardIssueChange(tx, receiptExisting.companyId);
|
||||
if ((patch.status !== undefined && patch.status !== receiptExisting.status)
|
||||
|| patch.completedAt !== undefined || patch.cancelledAt !== undefined
|
||||
|| issueData.parentId !== undefined || issueData.executionWorkspaceId !== undefined) {
|
||||
await notifyTerminalWorkspaceWork(tx, `issue:${id}`);
|
||||
}
|
||||
await notifyIssueAttentionRetentionWork(tx, { companyId: receiptExisting.companyId, issueId: id, parentId: patch.parentId, originKind: patch.originKind, originId: patch.originId });
|
||||
if (patch.status !== undefined || patch.assigneeAgentId !== undefined || patch.assigneeUserId !== undefined
|
||||
|| patch.hiddenAt !== undefined || patch.conversationAgentId !== undefined) {
|
||||
|| patch.hiddenAt !== undefined || patch.conversationAgentId !== undefined || patch.parentId !== undefined
|
||||
|| patch.executionState !== undefined || patch.executionRunId !== undefined || patch.checkoutRunId !== undefined
|
||||
|| blockedByIssueIds !== undefined) {
|
||||
await notifyRunDispatch(tx, { companyId: receiptExisting.companyId });
|
||||
await notifyDeferredExecutionEligibility(tx, { companyId: receiptExisting.companyId });
|
||||
await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers);
|
||||
await notifyRoutineExecutionStatus(tx, id);
|
||||
}
|
||||
@@ -11925,7 +11985,9 @@ export function issueService(db: Db) {
|
||||
);
|
||||
if (settings?.environmentId !== environmentId) continue;
|
||||
|
||||
await db
|
||||
await db.transaction(async tx => {
|
||||
await notifyStatusCardIssueChange(tx, companyId);
|
||||
await tx
|
||||
.update(issues)
|
||||
.set({
|
||||
executionWorkspaceSettings: {
|
||||
@@ -11935,6 +11997,7 @@ export function issueService(db: Db) {
|
||||
updatedAt: new Date(),
|
||||
})
|
||||
.where(eq(issues.id, row.id));
|
||||
});
|
||||
cleared += 1;
|
||||
}
|
||||
|
||||
@@ -11958,11 +12021,15 @@ export function issueService(db: Db) {
|
||||
await requestRestorePointDeletion(tx, { sourceIssueId: id });
|
||||
const [owner] = await tx.select({ companyId: issues.companyId }).from(issues).where(eq(issues.id, id));
|
||||
if (owner) {
|
||||
await notifyIssueAttentionRetentionWork(tx, { companyId: owner.companyId, issueId: id });
|
||||
await notifyExternalConversationSettlement(tx, { companyId: owner.companyId });
|
||||
await notifyApprovalWork(tx, owner.companyId);
|
||||
await notifyStatusCardIssueChange(tx, owner.companyId);
|
||||
}
|
||||
await notifyDecisionTargetIssueRemoval(tx, id);
|
||||
await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors);
|
||||
await notifyRunDispatch(tx, owner ? { companyId: owner.companyId } : undefined);
|
||||
if (owner) await notifyDeferredExecutionEligibility(tx, owner);
|
||||
await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers);
|
||||
await notifyRoutineExecutionStatus(tx, id);
|
||||
removedIssue = await tx
|
||||
@@ -12091,7 +12158,11 @@ export function issueService(db: Db) {
|
||||
)
|
||||
: isNull(issues.executionRunId);
|
||||
const updated = await db.transaction(async tx => {
|
||||
await notifyIssueAttentionRetentionWork(tx, { companyId: issueCompany.companyId, issueId: id });
|
||||
await notifyStatusCardIssueChange(tx, issueCompany.companyId);
|
||||
await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors);
|
||||
await notifyRunDispatch(tx, { companyId: issueCompany.companyId });
|
||||
await notifyDeferredExecutionEligibility(tx, { companyId: issueCompany.companyId });
|
||||
await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers);
|
||||
await notifyRoutineExecutionStatus(tx, id);
|
||||
return tx
|
||||
@@ -12144,7 +12215,9 @@ export function issueService(db: Db) {
|
||||
current.executionRunId === checkoutRunId) &&
|
||||
checkoutRunId
|
||||
) {
|
||||
const adopted = await db
|
||||
const adopted = await db.transaction(async tx => {
|
||||
await notifyStatusCardIssueChange(tx, issueCompany.companyId);
|
||||
return tx
|
||||
.update(issues)
|
||||
.set({
|
||||
checkoutRunId,
|
||||
@@ -12165,6 +12238,7 @@ export function issueService(db: Db) {
|
||||
)
|
||||
.returning()
|
||||
.then((rows) => rows[0] ?? null);
|
||||
});
|
||||
if (adopted) return adopted;
|
||||
}
|
||||
|
||||
@@ -12221,7 +12295,11 @@ export function issueService(db: Db) {
|
||||
}
|
||||
const staleExecutionRunId = current.executionRunId;
|
||||
const adopted = await db.transaction(async tx => {
|
||||
await notifyIssueAttentionRetentionWork(tx, { companyId: issueCompany.companyId, issueId: id });
|
||||
await notifyStatusCardIssueChange(tx, issueCompany.companyId);
|
||||
await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors);
|
||||
await notifyRunDispatch(tx, { companyId: issueCompany.companyId });
|
||||
await notifyDeferredExecutionEligibility(tx, { companyId: issueCompany.companyId });
|
||||
await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers);
|
||||
await notifyRoutineExecutionStatus(tx, id);
|
||||
return tx
|
||||
@@ -12477,7 +12555,10 @@ export function issueService(db: Db) {
|
||||
existing.status === "in_progress" ? "todo" : existing.status;
|
||||
await notifyTerminalWorkspaceWork(tx, `issue:${id}`);
|
||||
await notifyNativeStatusReassessment(tx, { companyId: existing.companyId, issueId: id });
|
||||
await notifyIssueAttentionRetentionWork(tx, { companyId: existing.companyId, issueId: id });
|
||||
await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors);
|
||||
await notifyRunDispatch(tx, { companyId: existing.companyId });
|
||||
await notifyDeferredExecutionEligibility(tx, { companyId: existing.companyId });
|
||||
await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers);
|
||||
await notifyRoutineExecutionStatus(tx, id);
|
||||
const updated = await tx
|
||||
@@ -12530,6 +12611,8 @@ export function issueService(db: Db) {
|
||||
patch.assigneeAgentId = null;
|
||||
}
|
||||
|
||||
await notifyIssueAttentionRetentionWork(tx, { companyId: existing.companyId, issueId: id });
|
||||
await notifyStatusCardIssueChange(tx, existing.companyId);
|
||||
await notifyTerminalWorkspaceWork(tx, `issue:${id}`);
|
||||
if (existing.checkoutRunId || existing.executionRunId || options.clearAssignee) {
|
||||
await notifyExternalConversationSettlement(tx, { companyId: existing.companyId, issueId: id });
|
||||
@@ -12539,6 +12622,8 @@ export function issueService(db: Db) {
|
||||
await notifyUnsafeWorkspaceExportRecovery(tx, { companyId: existing.companyId, issueId: id });
|
||||
}
|
||||
await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors);
|
||||
await notifyRunDispatch(tx, { companyId: existing.companyId });
|
||||
await notifyDeferredExecutionEligibility(tx, { companyId: existing.companyId });
|
||||
await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers);
|
||||
await notifyRoutineExecutionStatus(tx, id);
|
||||
const updated = await tx
|
||||
@@ -12588,12 +12673,12 @@ export function issueService(db: Db) {
|
||||
return created;
|
||||
},
|
||||
|
||||
deleteLabel: async (id: string) =>
|
||||
db
|
||||
.delete(labels)
|
||||
.where(eq(labels.id, id))
|
||||
.returning()
|
||||
.then((rows) => rows[0] ?? null),
|
||||
deleteLabel: async (id: string) => db.transaction(async tx => {
|
||||
const [label] = await tx.select({ companyId: labels.companyId }).from(labels).where(eq(labels.id, id));
|
||||
if (!label) return null;
|
||||
await notifyStatusCardIssueChange(tx, label.companyId);
|
||||
return tx.delete(labels).where(eq(labels.id, id)).returning().then(rows => rows[0] ?? null);
|
||||
}),
|
||||
|
||||
listComments: async (
|
||||
issueId: string,
|
||||
@@ -12737,6 +12822,7 @@ export function issueService(db: Db) {
|
||||
.from(issueComments).where(eq(issueComments.id, commentId));
|
||||
if (!source) return null;
|
||||
await notifyExternalConversationSettlement(tx, source);
|
||||
await notifyStatusCardIssueChange(tx, source.companyId);
|
||||
const [comment] = await tx
|
||||
.delete(issueComments)
|
||||
.where(eq(issueComments.id, commentId))
|
||||
@@ -12776,6 +12862,7 @@ export function issueService(db: Db) {
|
||||
.for("update");
|
||||
if (!source) return null;
|
||||
await notifyExternalConversationSettlement(tx, source);
|
||||
await notifyStatusCardIssueChange(tx, source.companyId);
|
||||
await notifyChatInboundWakeNotices(tx, source);
|
||||
const now = new Date();
|
||||
const [comment] = await tx
|
||||
@@ -12843,7 +12930,7 @@ export function issueService(db: Db) {
|
||||
},
|
||||
dbOrTx: any = db,
|
||||
): Promise<IssueComment> {
|
||||
if (dbOrTx === db && (actor.runId || actor.userId)) {
|
||||
if (dbOrTx === db) {
|
||||
const append = () =>
|
||||
measureRunTransaction(db, "response", async (tx) => {
|
||||
// Serialize run-authored comments on the issue so a provider retry
|
||||
@@ -13086,6 +13173,7 @@ export function issueService(db: Db) {
|
||||
currentUserRedactionOptions.enabled,
|
||||
);
|
||||
}
|
||||
await notifyStatusCardIssueChange(dbOrTx, issue.companyId);
|
||||
comment =
|
||||
shouldUpgradeExternalAuthorization ||
|
||||
shouldUpgradeAttachmentAuthorization
|
||||
@@ -13800,6 +13888,7 @@ export function issueService(db: Db) {
|
||||
.then((rows) => rows[0] ?? null);
|
||||
if (!existing) return null;
|
||||
|
||||
await notifyIssueRecovery(tx, { companyId: existing.companyId });
|
||||
await tx.delete(issueAttachments).where(eq(issueAttachments.id, id));
|
||||
await tx.delete(assets).where(eq(assets.id, existing.assetId));
|
||||
return existing;
|
||||
|
||||
@@ -77,14 +77,18 @@ async function renewOnLivenessPool(
|
||||
});
|
||||
}
|
||||
|
||||
export async function hasLiveLegacyController(db: Db, run: Run): Promise<boolean> {
|
||||
if (run.runtimeMode === "native" || !run.controllerBootId) return false;
|
||||
const [owner] = await db.select({ id: heartbeatRuns.id }).from(heartbeatRuns).where(and(
|
||||
export async function liveLegacyControllerExpiresAt(db: Db, run: Run): Promise<Date | null> {
|
||||
if (run.runtimeMode === "native" || !run.controllerBootId) return null;
|
||||
const [owner] = await db.select({ expiresAt: heartbeatRuns.controllerLeaseExpiresAt }).from(heartbeatRuns).where(and(
|
||||
eq(heartbeatRuns.id, run.id), eq(heartbeatRuns.companyId, run.companyId),
|
||||
eq(heartbeatRuns.status, "running"),
|
||||
gt(heartbeatRuns.controllerLeaseExpiresAt, sql`clock_timestamp()`),
|
||||
));
|
||||
return Boolean(owner);
|
||||
return owner?.expiresAt ?? null;
|
||||
}
|
||||
|
||||
export async function hasLiveLegacyController(db: Db, run: Run): Promise<boolean> {
|
||||
return (await liveLegacyControllerExpiresAt(db, run)) !== null;
|
||||
}
|
||||
|
||||
export type RevokedLegacyControllerState = Pick<Run, "executionStage" | "processPid" | "processGroupId">;
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { notifyAttentionRetentionWork } from "./retention-work-notifications.js";
|
||||
import { notifyNativeEnvironmentSettlement } from "./native-environment-settlement-notifications.js";
|
||||
import { notifyExternalAgentSettlement } from "../modules/external-agents/index.js";
|
||||
import { notifyExternalConversationSettlement } from "./external-conversation-settlement-notifications.js";
|
||||
@@ -14,6 +15,7 @@ import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notificatio
|
||||
export type NativeFinalizationWorkScope = { companyId: string; runId?: string; issueId?: string };
|
||||
|
||||
export async function notifyNativeFinalizationWork(transaction: object, scope: NativeFinalizationWorkScope) {
|
||||
await notifyAttentionRetentionWork(transaction, { companyId: scope.companyId });
|
||||
if (scope.issueId || scope.runId) {
|
||||
await notifyNativeEnvironmentSettlement(transaction, scope);
|
||||
await notifyExternalAgentSettlement(transaction, scope);
|
||||
|
||||
@@ -73,7 +73,7 @@ describe("native same-conversation chat attachment reuse", () => {
|
||||
beforeAll(async () => {
|
||||
const external = resolveExternalTestDatabaseUrl("PAPERCLIP_CHAT_REUSE_TEST_DATABASE_URL");
|
||||
if (!external) temporary = await startEmbeddedPostgresTestDatabase("native-chat-reuse-");
|
||||
db = createDb(external ?? temporary!.connectionString);
|
||||
db = createDb(external ?? temporary!.connectionString, { maxConnections: 6 });
|
||||
const storageRoot = await mkdtemp(
|
||||
path.join(tmpdir(), "paperclip-chat-reuse-"),
|
||||
);
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { notifyIssueRecovery } from "../issue-recovery-notifications.js";
|
||||
import { createHash } from "node:crypto";
|
||||
|
||||
import { and, desc, eq, inArray, isNull, sql } from "drizzle-orm";
|
||||
@@ -1582,7 +1583,9 @@ export async function prepareReusedChatAttachment(input: {
|
||||
if (!workProduct) {
|
||||
throw new Error("paperclip_runner_chat_attachment_origin_not_persisted");
|
||||
}
|
||||
await input.db
|
||||
await input.db.transaction(async tx => {
|
||||
await notifyIssueRecovery(tx, { companyId: input.binding.companyId });
|
||||
await tx
|
||||
.update(issueWorkProducts)
|
||||
.set({
|
||||
title: input.title,
|
||||
@@ -1602,6 +1605,7 @@ export async function prepareReusedChatAttachment(input: {
|
||||
eq(issueWorkProducts.createdByRunId, input.binding.runId),
|
||||
),
|
||||
);
|
||||
});
|
||||
const comment = await issueService(input.db).addComment(
|
||||
input.binding.issueId,
|
||||
`Prepared ${input.title} for this response.`,
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { notifyStatusCardIssueChange } from "../status-card-notifications.js";
|
||||
import { notifyExternalAgentSettlement } from "../../modules/external-agents/index.js";
|
||||
import { pendingNativeRunProjectionCondition } from "./native-run-projection.js";
|
||||
import { nativeStatusReassessmentSourceIsSuperseded, reassessNativeStatusFromSource } from "./native-status-reassessment.js";
|
||||
@@ -334,6 +335,7 @@ export async function claimNativeSessionResumptions(input: {
|
||||
nativePhaseUpdatedAt: now,
|
||||
updatedAt: now,
|
||||
}).where(eq(heartbeatRuns.id, row.run.id));
|
||||
await notifyStatusCardIssueChange(tx, run.companyId);
|
||||
await tx.update(issues).set({
|
||||
executionRunId: run.id,
|
||||
executionAgentNameKey: agent.name.trim().toLowerCase() || null,
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
import { notifyAttentionRetentionWork } from "../retention-work-notifications.js";
|
||||
import { notifyRunDispatch } from "../run-dispatch-notifications.js";
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { and, eq } from "drizzle-orm";
|
||||
import { agentWakeupRequests, heartbeatRuns, type Db } from "@paperclipai/db";
|
||||
@@ -49,6 +51,8 @@ export async function materializeNativeProviderCapacityRetry(input: {
|
||||
scheduledRetryAt: dueAt.toISOString(),
|
||||
executionRetryAccounting: accountingForScheduledRetry(run, NATIVE_PROVIDER_OVERLOADED_CODE, attempt),
|
||||
};
|
||||
await notifyAttentionRetentionWork(input.tx, { companyId: input.companyId });
|
||||
await notifyRunDispatch(input.tx, { companyId: run.companyId });
|
||||
const [wake] = await input.tx.insert(agentWakeupRequests).values({
|
||||
companyId: run.companyId, agentId: run.agentId, source: "automation", triggerDetail: "system",
|
||||
reason: NATIVE_PROVIDER_OVERLOADED_CODE, status: "queued", runId: retryId,
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
import { notifyAttentionRetentionWork } from "../retention-work-notifications.js";
|
||||
import { notifyStatusCardIssueChange } from "../status-card-notifications.js";
|
||||
import {
|
||||
agents,
|
||||
heartbeatRuns,
|
||||
@@ -240,6 +242,8 @@ export async function claimNativeReviewExecutionLock(db: Db, input: {
|
||||
if (!run || !["queued", "running"].includes(run.status) || !admitted || !expected
|
||||
|| admitted.nativeReviewInteractionId !== expected.nativeReviewInteractionId
|
||||
|| admitted.nativeReviewDecisionId !== expected.nativeReviewDecisionId) return false;
|
||||
await notifyAttentionRetentionWork(tx, { companyId: input.companyId });
|
||||
await notifyStatusCardIssueChange(tx, input.companyId);
|
||||
const claimed = await tx.update(issues).set({
|
||||
executionRunId: input.runId,
|
||||
executionAgentNameKey: input.agentNameKey,
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { resolveExternalTestDatabaseUrl } from "../../__tests__/helpers/external-test-database.js";
|
||||
import type { PrpStructuredRunResult } from "../../vendor/paperclip-runner/index.js";
|
||||
import { nativeCompletionFeedback } from "./native-completion-feedback.js";
|
||||
import { createHash, randomUUID } from "node:crypto";
|
||||
@@ -61,10 +62,9 @@ describe("native runner file handoff", () => {
|
||||
const runId = "00000000-0000-4000-8000-000000009104";
|
||||
|
||||
beforeAll(async () => {
|
||||
temporary = await startEmbeddedPostgresTestDatabase(
|
||||
"native-runner-file-handoff-",
|
||||
);
|
||||
db = createDb(temporary.connectionString);
|
||||
const external = resolveExternalTestDatabaseUrl("PAPERCLIP_NATIVE_HANDOFF_TEST_DATABASE_URL");
|
||||
if (!external) temporary = await startEmbeddedPostgresTestDatabase("native-runner-file-handoff-");
|
||||
db = createDb(external ?? temporary!.connectionString, { maxConnections: 6 });
|
||||
temporaryRoot = await mkdtemp(
|
||||
path.join(tmpdir(), "paperclip-native-file-handoff-"),
|
||||
);
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { notifyIssueRecovery } from "../issue-recovery-notifications.js";
|
||||
import { createHash, randomUUID } from "node:crypto";
|
||||
import { execFile } from "node:child_process";
|
||||
import { constants } from "node:fs";
|
||||
@@ -1265,7 +1266,9 @@ export async function prepareNativeRunnerFileHandoff(input: {
|
||||
) {
|
||||
throw new Error("paperclip_runner_file_handoff_origin_not_persisted");
|
||||
}
|
||||
await input.db
|
||||
await input.db.transaction(async tx => {
|
||||
await notifyIssueRecovery(tx, { companyId: input.binding.companyId });
|
||||
await tx
|
||||
.update(issueWorkProducts)
|
||||
.set({ title: verified.title, updatedAt: new Date() })
|
||||
.where(
|
||||
@@ -1276,6 +1279,7 @@ export async function prepareNativeRunnerFileHandoff(input: {
|
||||
eq(issueWorkProducts.createdByRunId, input.binding.runId),
|
||||
),
|
||||
);
|
||||
});
|
||||
const comment = await issueService(input.db).addComment(
|
||||
input.binding.issueId,
|
||||
`Prepared ${verified.title} for this response.`,
|
||||
|
||||
@@ -4302,6 +4302,17 @@ describe("retained native cleanup activation", () => {
|
||||
releaseCommit = release;
|
||||
});
|
||||
const db = {
|
||||
execute: async (query: SQL) => {
|
||||
// Retention hints read the fixture's run/issue relationship inside the
|
||||
// same cleanup transaction; this fixture has one issue and no ancestors.
|
||||
const statement = new PgDialect().sqlToQuery(query);
|
||||
expect(transactionOpen).toBe(true);
|
||||
expect(statement.params).toContain(run.companyId);
|
||||
if (statement.sql.includes("from heartbeat_runs")) return [{ id: run.nativeIssueId }];
|
||||
if (statement.sql.includes("origin_kind")) return [{ id: run.nativeIssueId, originKind: null, originId: null }];
|
||||
if (statement.sql.includes("with recursive affected")) return [{ issueId: run.nativeIssueId }];
|
||||
throw new Error(`Unexpected cleanup fixture query: ${statement.sql}`);
|
||||
},
|
||||
select: () => ({
|
||||
from: (table: unknown) => {
|
||||
const rows =
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { notifyStatusCardIssueChange } from "../status-card-notifications.js";
|
||||
import { notifyNativeFinalizationWork } from "../native-finalization-notifications.js";
|
||||
import { alias } from "drizzle-orm/pg-core";
|
||||
import { notifyUnsafeWorkspaceExportRecovery, type UnsafeWorkspaceExportScope } from "../unsafe-workspace-export-notifications.js";
|
||||
@@ -107,6 +108,7 @@ async function recoverCandidate(db: Db, candidate: { runId: string; companyId: s
|
||||
nativePhase: "result_accepted", nativePhaseUpdatedAt: now,
|
||||
resultJson: { ...run.resultJson, finalizationPhase: "result_accepted", failureCode: null, originalFailureCode: null, nextAttemptAt: null }, updatedAt: now,
|
||||
}).where(eq(heartbeatRuns.id, run.id)).returning();
|
||||
await notifyStatusCardIssueChange(tx, run.companyId);
|
||||
await tx.update(issues).set({ executionRunId: run.id, updatedAt: now }).where(eq(issues.id, issue.id));
|
||||
await appendHeartbeatRunEvent(tx as unknown as Db, { companyId: run.companyId, agentId: run.agentId, runId: run.id,
|
||||
eventType: "workspace_export_omitted", stream: "system", level: "info",
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { notifyStatusCardIssueChange } from "../status-card-notifications.js";
|
||||
import { notifyNativeFinalizationWork } from "../native-finalization-notifications.js";
|
||||
import { notifyEnvironmentCleanup } from "../environment-cleanup-notifications.js";
|
||||
import { randomUUID } from "node:crypto";
|
||||
@@ -141,6 +142,7 @@ export async function retryNativeWorkspaceExport(input: {
|
||||
await tx.update(heartbeatRuns).set({ status: "running", finishedAt: null, error: null, errorCode: null,
|
||||
nativePhase: "result_accepted", nativePhaseUpdatedAt: now,
|
||||
resultJson: { ...current.run.resultJson, workspaceExportRetry: retry, finalizationPhase: "result_accepted", failureCode: null, originalFailureCode: null, nextAttemptAt: null }, updatedAt: now }).where(eq(heartbeatRuns.id, current.run.id));
|
||||
await notifyStatusCardIssueChange(tx, current.run.companyId);
|
||||
await tx.update(issues).set({ executionRunId: current.run.id, updatedAt: now }).where(eq(issues.id, current.issue.id));
|
||||
// Reclaim the same lease only for control-plane copyback. The normal
|
||||
// finalization settlement will stop/retain it again on success or failure.
|
||||
|
||||
@@ -131,7 +131,7 @@ describe("native workspace finalization recovery", () => {
|
||||
const external = resolveExternalTestDatabaseUrl("PAPERCLIP_NATIVE_WORKSPACE_FINALIZER_TEST_DATABASE_URL");
|
||||
if (!external) temporary = await startEmbeddedPostgresTestDatabase("paperclip-native-workspace-finalizer-");
|
||||
connectionString = external ?? temporary!.connectionString;
|
||||
db = createDb(connectionString);
|
||||
db = createDb(connectionString, { maxConnections: 6 });
|
||||
|
||||
await db.insert(companies).values([
|
||||
{ id: companyId, name: "Native workspace owner", issuePrefix: "NWO" },
|
||||
@@ -452,7 +452,7 @@ describe("native workspace finalization recovery", () => {
|
||||
const seeded = await seedRun({ executionWorkspaceId: randomUUID(), title: "Live copyback" });
|
||||
const live = await withNativeWorkspaceFinalizationOwnership({ db, companyId, runId: seeded.runId }, async (ownership) => {
|
||||
await ownership.assertHeld();
|
||||
expect(await resumeNativeWorkspaceFinalization({ db: createDb(connectionString), runId: seeded.runId })).toBeNull();
|
||||
expect(await resumeNativeWorkspaceFinalization({ db: createDb(connectionString, { maxConnections: 6 }), runId: seeded.runId })).toBeNull();
|
||||
expect(await db.select().from(workspaceOperations).where(eq(workspaceOperations.heartbeatRunId, seeded.runId))).toEqual([]);
|
||||
return "copied";
|
||||
});
|
||||
@@ -476,7 +476,7 @@ describe("native workspace finalization recovery", () => {
|
||||
try {
|
||||
await entered;
|
||||
await db.execute(sql`select pg_terminate_backend(pid) from pg_locks where locktype = 'advisory' and database = (select oid from pg_database where datname = current_database()) and pid <> pg_backend_pid()`);
|
||||
const second = await withNativeWorkspaceFinalizationOwnership({ db: createDb(connectionString), companyId, runId: seeded.runId }, async () => "must not run");
|
||||
const second = await withNativeWorkspaceFinalizationOwnership({ db: createDb(connectionString, { maxConnections: 6 }), companyId, runId: seeded.runId }, async () => "must not run");
|
||||
expect(second).toEqual({ acquired: false, reason: "busy" });
|
||||
const [held] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, seeded.runId));
|
||||
expect(held.runnerProfileJson?.nativeWorkspaceFinalizationOwner).toBeTruthy();
|
||||
@@ -489,7 +489,7 @@ describe("native workspace finalization recovery", () => {
|
||||
|
||||
it("recovers its own joined copyback receipt after the application pool disconnects during cleanup", async () => {
|
||||
const seeded = await seedRun({ executionWorkspaceId: randomUUID(), title: "Reconnect joined copyback" });
|
||||
const interruptedDb = createDb(connectionString);
|
||||
const interruptedDb = createDb(connectionString, { maxConnections: 6 });
|
||||
let joined = false;
|
||||
await expect(withNativeWorkspaceFinalizationOwnership({ db: interruptedDb, companyId, runId: seeded.runId }, async () => {
|
||||
await fs.writeFile(path.join(workspaceRoot, "joined-copyback.txt"), "durable work");
|
||||
@@ -507,7 +507,7 @@ describe("native workspace finalization recovery", () => {
|
||||
.toEqual({ acquired: false, reason: "owner_unverified" });
|
||||
}
|
||||
await db.update(heartbeatRuns).set({ runnerProfileJson: beforeReconnect.runnerProfileJson }).where(eq(heartbeatRuns.id, seeded.runId));
|
||||
expect(await withNativeWorkspaceFinalizationOwnership({ db: createDb(connectionString), companyId, runId: seeded.runId }, async () => "resumed"))
|
||||
expect(await withNativeWorkspaceFinalizationOwnership({ db: createDb(connectionString, { maxConnections: 6 }), companyId, runId: seeded.runId }, async () => "resumed"))
|
||||
.toEqual({ acquired: true, value: "resumed" });
|
||||
});
|
||||
|
||||
|
||||
@@ -1,4 +1,6 @@
|
||||
import { notifyRoutineExecutionStatus } from "../routine-execution-status-notifications.js";
|
||||
import { notifyAttentionRetentionWork } from "../retention-work-notifications.js";
|
||||
import { notifyStatusCardIssueChange } from "../status-card-notifications.js";
|
||||
import { DELIVERY_QUEUES, notifyDeliveryWork } from "../delivery-work-notifications.js";
|
||||
import { notifyUnsafeWorkspaceExportRecovery } from "../unsafe-workspace-export-notifications.js";
|
||||
import { and, eq, gt, inArray, ne, notInArray, or, sql } from "drizzle-orm";
|
||||
@@ -180,6 +182,7 @@ export async function reconstructNativeWorkspace(input: {
|
||||
...(source.interveningExecutions.length ? { interveningExecutions: source.interveningExecutions } : {}) } };
|
||||
await markExecutionReconciliation(db, source.action, decision, input.actorId);
|
||||
const now = new Date();
|
||||
await notifyAttentionRetentionWork(tx, { companyId: input.companyId });
|
||||
await notifyUnsafeWorkspaceExportRecovery(tx, { companyId: input.companyId, issueId: input.issueId });
|
||||
await tx.update(issueRecoveryActions).set({ status: "resolved", outcome: "handed_back", resolvedAt: now, updatedAt: now,
|
||||
resolutionNote: "workspace_reconstruction_authorized", nextAction: "Continue from saved sources. The original unexported edits are lost; verify the selected attachments before rebuilding. Do not repeat completed external actions or reset the original work limits.",
|
||||
@@ -189,6 +192,7 @@ export async function reconstructNativeWorkspace(input: {
|
||||
await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors);
|
||||
await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers);
|
||||
await notifyRoutineExecutionStatus(tx, input.issueId, input.companyId);
|
||||
await notifyStatusCardIssueChange(tx, input.companyId);
|
||||
await tx.update(issues).set({ status: "todo", unblockDescriptor: null, blockedTransitionAt: null,
|
||||
blockedOwnerNotifiedAt: null, updatedAt: now }).where(eq(issues.id, issue.id));
|
||||
const publication = await persistActivity(db, { companyId: input.companyId, actorType: "user", actorId: input.actorId,
|
||||
|
||||
@@ -1,4 +1,6 @@
|
||||
import { notifyStatusCardIssueChange } from "../status-card-notifications.js";
|
||||
import { nativeRunKeyScope, nativeSkillTestToolAllowed, assertNativeSkillTestTool } from "./native-skill-test-scope.js";
|
||||
import { notifyAttentionRetentionWork } from "../retention-work-notifications.js";
|
||||
import { requiredQualityReviewSchema } from "@paperclipai/shared";
|
||||
import { priorRecoveryApiReceipt } from "../recovery-effect-receipts.js";
|
||||
import { DELIVERY_QUEUES, notifyDeliveryWork } from "../delivery-work-notifications.js";
|
||||
@@ -2067,8 +2069,10 @@ export class PaperclipRunnerToolAuthority {
|
||||
replayMonitor = summarizeIssueMonitor(target, normalizeIssueExecutionPolicy(target.executionPolicy));
|
||||
return { taskId: target.id, status: target.status, monitor: replayMonitor };
|
||||
}
|
||||
await notifyAttentionRetentionWork(tx, { companyId: this.binding.companyId });
|
||||
const patch = prepareIssueMonitorUpdate(target, input.monitor, actor);
|
||||
await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors);
|
||||
await notifyStatusCardIssueChange(tx, this.binding.companyId);
|
||||
const [updated] = await tx.update(issues).set({ ...patch, updatedAt: new Date() })
|
||||
.where(and(eq(issues.id, target.id), eq(issues.companyId, this.binding.companyId))).returning();
|
||||
if (!updated) throw notFound("Task not found");
|
||||
|
||||
@@ -497,7 +497,7 @@ else if(m.id!==undefined) send({id:m.id,result:{}});
|
||||
const [agent] = await server.db.select().from(agents).where(eq(agents.id, fixture.agentId));
|
||||
const context = structuredClone(run.contextSnapshot ?? {});
|
||||
await prepareHeartbeatTaskContext(server.db, { run, agent,
|
||||
task: { issueRef: issue, issueContext: null, wakeCommentContext: null, context, taskSession: null },
|
||||
task: { issueRef: issue, issueContext: null, context, taskSession: null },
|
||||
policy: { recovering: true, trustPreset: { kind: "standard" }, experimentalInstanceSettings: {} },
|
||||
services: { issuesSvc: { getAncestors: async () => [] },
|
||||
getPinnedSkillTestContext: createHeartbeatRunPreparation(server.db).getPinnedSkillTestContext },
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { notifyNativeStatusWakeWork } from "../deferred-execution-notifications.js";
|
||||
import { notifyExternalAgentSettlement } from "../../modules/external-agents/index.js";
|
||||
import { lockCurrentNativeWorkAssessmentEvidence } from "./work-assessments.js";
|
||||
import { hasNativeWorkspaceFinalizationOwner } from "./native-workspace-finalization-recovery-state.js";
|
||||
@@ -478,6 +479,7 @@ async function enqueueWake(input: {
|
||||
.limit(1)
|
||||
.then((rows) => rows[0] ?? null);
|
||||
if (existing) return existing.id;
|
||||
await notifyNativeStatusWakeWork(input.tx, { companyId: input.companyId });
|
||||
const inserted = await input.tx
|
||||
.insert(agentWakeupRequests)
|
||||
.values({
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { notifyAttentionRetentionWork } from "../retention-work-notifications.js";
|
||||
import { and, eq, isNull, ne, sql } from "drizzle-orm";
|
||||
import { completionContracts, heartbeatRuns, issues, issueThreadInteractions, statusDecisionEffects, statusDecisions, workAssessments, type Db } from "@paperclipai/db";
|
||||
import { hasPublishedGitHubReview } from "../chat-channels/github/review-wait.js";
|
||||
@@ -70,6 +71,7 @@ export async function supersedeExternalReviewRequestsInTransaction(
|
||||
publications: ActivityPublication[],
|
||||
) {
|
||||
for (const { interaction } of candidates) {
|
||||
await notifyAttentionRetentionWork(tx, { companyId: binding.companyId });
|
||||
const reason = "A later published review no longer requests human input.";
|
||||
const [retired] = await tx.update(issueThreadInteractions).set({ status: "cancelled",
|
||||
result: { version: 1, outcome: "withdrawn", reason }, resolvedAt: new Date(), updatedAt: new Date(),
|
||||
|
||||
@@ -1,8 +1,14 @@
|
||||
import { notifyIssueAttentionRetentionWork } from "./retention-work-notifications.js";
|
||||
import { notifyStatusCardIssueChange } from "./status-card-notifications.js";
|
||||
import { notifyIssueRecovery } from "./issue-recovery-notifications.js";
|
||||
import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js";
|
||||
|
||||
export type NativeStatusReassessmentScope = { companyId: string; issueId?: string; runId?: string; agentId?: string };
|
||||
|
||||
export function notifyNativeStatusReassessment(tx: object, scope: NativeStatusReassessmentScope) {
|
||||
export async function notifyNativeStatusReassessment(tx: object, scope: NativeStatusReassessmentScope) {
|
||||
await notifyIssueAttentionRetentionWork(tx, { companyId: scope.companyId, issueId: scope.issueId, runId: scope.runId });
|
||||
await notifyStatusCardIssueChange(tx, scope.companyId);
|
||||
await notifyIssueRecovery(tx, scope ? { companyId: scope.companyId } : undefined);
|
||||
return notifyDeliveryWork(tx, DELIVERY_QUEUES.nativeStatusReassessment,
|
||||
JSON.stringify({ companyId: scope.companyId, issueId: scope.issueId, runId: scope.runId, agentId: scope.agentId }));
|
||||
}
|
||||
@@ -1,3 +1,4 @@
|
||||
import { notifyAttentionRetentionWork } from "./retention-work-notifications.js";
|
||||
import { createAgentLifecycle } from "./agent-lifecycle.js";
|
||||
import { and, eq, isNull, ne } from "drizzle-orm";
|
||||
import type { Db } from "@paperclipai/db";
|
||||
@@ -340,6 +341,7 @@ export function pluginManagedAgentService(
|
||||
// and its canonical history. Transfer only its proven stable-key marker to
|
||||
// the replacement installation, atomically with the new scoped bindings.
|
||||
await upsertBinding(companyId, declaration, agentId, {}, adapterType, tx);
|
||||
await notifyAttentionRetentionWork(tx, { companyId, sourceKind: "agent_error_alert" });
|
||||
await tx.update(agents).set({
|
||||
metadata: managedMetadata(options.pluginId, options.pluginKey, declaration, agent.metadata),
|
||||
updatedAt: new Date(),
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { notifyStatusCardIssueChange } from "./status-card-notifications.js";
|
||||
import { effectiveProjectWorkspacePolicy, isWorkspaceProvider } from "./project-workspace-provider.js";
|
||||
import { releaseProviderRecoveryTransfers, stageProviderFreshWorkspace } from "./provider-fresh-workspace.js";
|
||||
import { randomUUID, createHash } from "node:crypto";
|
||||
@@ -418,6 +419,7 @@ export async function preparePluginWorkspace(
|
||||
},
|
||||
})
|
||||
.returning();
|
||||
await notifyStatusCardIssueChange(tx, task.companyId);
|
||||
await tx
|
||||
.update(issues)
|
||||
.set({
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { notifyStatusCardIssueChange } from "./status-card-notifications.js";
|
||||
import { notifyChatInboundWakeNotices } from "./chat-inbound-wake-notice-notifications.js";
|
||||
import { listProjectWorkspaceProviders, resolveProjectWorkspaceProvider } from "./project-workspace-provider.js";
|
||||
import { retireRemovedPluginWorkspaceDependents } from "./plugin-workspaces.js";
|
||||
@@ -1013,6 +1014,7 @@ export function projectService(db: Db, options: { captureWorkspaceUpdates?: bool
|
||||
|
||||
const row = await db.transaction(async (tx) => {
|
||||
if (updates.visibility !== undefined) {
|
||||
await notifyStatusCardIssueChange(tx, existingProject.companyId);
|
||||
await notifyChatInboundWakeNotices(tx, { companyId: existingProject.companyId });
|
||||
await tx.execute(sql`select pg_advisory_xact_lock(hashtextextended(${`issue-privacy-tree:${existingProject.companyId}`}, 0))`);
|
||||
}
|
||||
@@ -1086,6 +1088,7 @@ export function projectService(db: Db, options: { captureWorkspaceUpdates?: bool
|
||||
await tx.select({ id: projectWorkspaces.id }).from(projectWorkspaces).where(eq(projectWorkspaces.projectId, id)).for("update");
|
||||
await recordProjectLifecycleEvent(tx as unknown as Db, project.companyId, id, "delete");
|
||||
await notifyDeliveryWork(tx, DELIVERY_QUEUES.skillSourceSync);
|
||||
await notifyStatusCardIssueChange(tx, project.companyId);
|
||||
const [row] = await tx.delete(projects).where(eq(projects.id, id)).returning();
|
||||
return row ? { ...row, urlKey: deriveProjectUrlKey(row.name, row.id) } : null;
|
||||
}),
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { notifyStatusCardIssueChange } from "./status-card-notifications.js";
|
||||
import { assertTaskWorkspaceExecutionAccess } from "./task-workspace-access.js";
|
||||
import { isDeepStrictEqual } from "node:util";
|
||||
import { createHash, randomUUID } from "node:crypto";
|
||||
@@ -152,6 +153,7 @@ export async function reserveFreshPluginTaskWorkspace(input: {
|
||||
providerType: "adapter_managed", cwd: `/workspaces/${input.fresh.attemptId}`,
|
||||
metadata: { runnerTaskFreshWorkspace: input.fresh, runnerTaskWorkspaceOwner: input.owner },
|
||||
});
|
||||
await notifyStatusCardIssueChange(tx, input.companyId);
|
||||
await tx.update(issues).set({ executionWorkspaceId: input.fresh.attemptId, executionWorkspacePreference: "reuse_existing",
|
||||
workspaceBindingRevision: task.workspaceBindingRevision + 1,
|
||||
workspacePendingSelection: task.workspacePendingSelection?.expectedBindingRevision === task.workspaceBindingRevision
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { notifyRunDispatch } from "../run-dispatch-notifications.js";
|
||||
import { notifyNativeEnvironmentSettlement } from "../native-environment-settlement-notifications.js";
|
||||
import { notifyExternalAgentSettlement } from "../../modules/external-agents/index.js";
|
||||
import { notifyNativeStatusReassessment } from "../native-status-reassessment-notifications.js";
|
||||
@@ -26,7 +27,7 @@ import {
|
||||
decideLegacyContinuation, legacyDispositionEpisode, legacyDispositionFingerprint,
|
||||
LEGACY_DISPOSITION_REPAIR_INSTRUCTION, type LegacyDispositionEpisode,
|
||||
} from "./legacy-continuation.js";
|
||||
import { CONTROLLER_LEASE_LOST_FAILURE_CODE, hasLiveLegacyController } from "../legacy-controller-lease.js";
|
||||
import { CONTROLLER_LEASE_LOST_FAILURE_CODE, hasLiveLegacyController, liveLegacyControllerExpiresAt } from "../legacy-controller-lease.js";
|
||||
import { isCompletedOnboardingHandoffWake } from "../chat-completion-delivery.js";
|
||||
import { instanceSettingsService } from "../instance-settings.js";
|
||||
import { isWaitingConversation, settleConversationTurnForRun, deliverConversationComments } from "../agent-conversations.js";
|
||||
@@ -244,6 +245,7 @@ type ResolvedDependencyWakeBackstopOptions = {
|
||||
runId?: string | null;
|
||||
companyId?: string | null;
|
||||
blockerIssueId?: string | null;
|
||||
issueId?: string;
|
||||
source?: ResolvedDependencyWakeBackstopSource;
|
||||
};
|
||||
|
||||
@@ -2201,7 +2203,7 @@ export function recoveryService(
|
||||
return Boolean(budgetBlock);
|
||||
}
|
||||
|
||||
async function reconcileUnassignedBlockingIssues() {
|
||||
async function reconcileUnassignedBlockingIssues(opts?: { companyId?: string }) {
|
||||
const candidates = await db
|
||||
.select({
|
||||
id: issues.id,
|
||||
@@ -2215,6 +2217,8 @@ export function recoveryService(
|
||||
.where(
|
||||
and(
|
||||
eq(issueRelations.type, "blocks"),
|
||||
eq(issueRelations.companyId, issues.companyId),
|
||||
opts?.companyId ? eq(issues.companyId, opts.companyId) : undefined,
|
||||
inArray(issues.status, ["todo", "blocked"]),
|
||||
isNull(issues.assigneeAgentId),
|
||||
isNull(issues.assigneeUserId),
|
||||
@@ -2716,6 +2720,7 @@ export function recoveryService(
|
||||
const now = new Date();
|
||||
const retryAt = readProviderQuotaRetryAt(input.latestRun, now);
|
||||
return database.transaction(async (tx) => {
|
||||
await notifyRunDispatch(tx, { companyId: input.issue.companyId });
|
||||
const wakeup = await tx
|
||||
.insert(agentWakeupRequests)
|
||||
.values({
|
||||
@@ -3316,6 +3321,7 @@ export function recoveryService(
|
||||
created = Boolean(enqueuedRun);
|
||||
} else {
|
||||
scheduledRun = await db.transaction(async (tx) => {
|
||||
await notifyRunDispatch(tx, { companyId: input.issue.companyId });
|
||||
const wakeup = await tx
|
||||
.insert(agentWakeupRequests)
|
||||
.values({
|
||||
@@ -3497,7 +3503,7 @@ export function recoveryService(
|
||||
return Boolean(run || wake);
|
||||
}
|
||||
|
||||
async function reconcileActiveRecoveryActions() {
|
||||
async function reconcileActiveRecoveryActions(opts?: { companyId?: string }) {
|
||||
const rows = await db
|
||||
.select({
|
||||
action: issueRecoveryActions,
|
||||
@@ -3511,7 +3517,8 @@ export function recoveryService(
|
||||
eq(issues.companyId, issueRecoveryActions.companyId),
|
||||
),
|
||||
)
|
||||
.where(inArray(issueRecoveryActions.status, ["active", "escalated"]));
|
||||
.where(and(inArray(issueRecoveryActions.status, ["active", "escalated"]),
|
||||
opts?.companyId ? eq(issueRecoveryActions.companyId, opts.companyId) : undefined));
|
||||
|
||||
const result = {
|
||||
requeued: 0,
|
||||
@@ -4523,14 +4530,20 @@ export function recoveryService(
|
||||
}
|
||||
|
||||
async function reconcileStrandedAssignedIssues(opts?: {
|
||||
companyId?: string;
|
||||
issueId?: string;
|
||||
onDeferredDeadline?: (at: number) => void;
|
||||
issueCreatedAtGte?: Date | null;
|
||||
skipExternalConversationSettlement?: boolean;
|
||||
skipRelatedRecovery?: boolean;
|
||||
}) {
|
||||
const candidates = await db
|
||||
.select(recoveryIssueFields)
|
||||
.from(issues)
|
||||
.where(
|
||||
and(
|
||||
opts?.companyId ? eq(issues.companyId, opts.companyId) : undefined,
|
||||
opts?.issueId ? eq(issues.id, opts.issueId) : undefined,
|
||||
isNull(issues.assigneeUserId),
|
||||
inArray(issues.status, ["todo", "in_progress", "in_review"]),
|
||||
or(
|
||||
@@ -5711,6 +5724,7 @@ export function recoveryService(
|
||||
classification.baseBackoffMs *
|
||||
Math.pow(2, Math.max(0, consecutive - 1));
|
||||
if (elapsed < requiredDelay) {
|
||||
opts?.onDeferredDeadline?.(latestFinishedAt.getTime() + requiredDelay);
|
||||
result.skipped += 1;
|
||||
continue;
|
||||
}
|
||||
@@ -5770,16 +5784,18 @@ export function recoveryService(
|
||||
}
|
||||
}
|
||||
|
||||
const orphanBlockerRecovery = await reconcileUnassignedBlockingIssues();
|
||||
result.orphanBlockersAssigned = orphanBlockerRecovery.assigned;
|
||||
result.skipped += orphanBlockerRecovery.skipped;
|
||||
result.issueIds.push(...orphanBlockerRecovery.issueIds);
|
||||
if (!opts?.skipRelatedRecovery) {
|
||||
const orphanBlockerRecovery = await reconcileUnassignedBlockingIssues(opts);
|
||||
result.orphanBlockersAssigned = orphanBlockerRecovery.assigned;
|
||||
result.skipped += orphanBlockerRecovery.skipped;
|
||||
result.issueIds.push(...orphanBlockerRecovery.issueIds);
|
||||
|
||||
const activeRecovery = await reconcileActiveRecoveryActions();
|
||||
result.continuationRequeued += activeRecovery.requeued;
|
||||
result.escalated += activeRecovery.escalated;
|
||||
result.skipped += activeRecovery.skipped;
|
||||
result.issueIds.push(...activeRecovery.issueIds);
|
||||
const activeRecovery = await reconcileActiveRecoveryActions(opts);
|
||||
result.continuationRequeued += activeRecovery.requeued;
|
||||
result.escalated += activeRecovery.escalated;
|
||||
result.skipped += activeRecovery.skipped;
|
||||
result.issueIds.push(...activeRecovery.issueIds);
|
||||
}
|
||||
result.issueIds = [...new Set(result.issueIds)];
|
||||
|
||||
return result;
|
||||
@@ -5812,7 +5828,7 @@ export function recoveryService(
|
||||
source === "workspace.finalize"
|
||||
? "workspace_finalize_reconciliation"
|
||||
: "issue_graph_liveness_reconciliation";
|
||||
const useCursor = !opts?.blockerIssueId;
|
||||
const useCursor = !opts?.blockerIssueId && !opts?.issueId;
|
||||
|
||||
const queryCandidates = (afterIssueId: string | null) => {
|
||||
const filters = [
|
||||
@@ -5822,6 +5838,7 @@ export function recoveryService(
|
||||
sql`${issues.assigneeAgentId} is not null`,
|
||||
];
|
||||
if (opts?.companyId) filters.push(eq(issues.companyId, opts.companyId));
|
||||
if (opts?.issueId) filters.push(eq(issues.id, opts.issueId));
|
||||
if (afterIssueId) filters.push(gt(issues.id, afterIssueId));
|
||||
|
||||
if (opts?.blockerIssueId) {
|
||||
@@ -6112,6 +6129,7 @@ export function recoveryService(
|
||||
// fallback below. Without it, a terminal issue named in the run context
|
||||
// snapshot would still terminalize the shared run and defeat the guard.
|
||||
runReferencedByActiveIssue?: boolean;
|
||||
onDeferredDeadline?: (at: number) => void;
|
||||
},
|
||||
): Promise<{ terminalized: boolean; status: string }> {
|
||||
// Act only on a run in "running" status. A "queued" run has no process yet,
|
||||
@@ -6127,8 +6145,11 @@ export function recoveryService(
|
||||
// Another controller may own a sandbox run whose PID has no meaning on
|
||||
// this host. Its live lease owns both execution and finalization, even if
|
||||
// the issue is already terminal or this process has no in-memory handle.
|
||||
if (await hasLiveLegacyController(db, run))
|
||||
const controllerDeadline = await liveLegacyControllerExpiresAt(db, run);
|
||||
if (controllerDeadline) {
|
||||
options?.onDeferredDeadline?.(controllerDeadline.getTime());
|
||||
return { terminalized: false, status: run.status };
|
||||
}
|
||||
|
||||
const pid = run.processPid ?? null;
|
||||
const processGroupId = run.processGroupId ?? null;
|
||||
@@ -6349,7 +6370,7 @@ export function recoveryService(
|
||||
// still claims to be live but can no longer reach a terminal status on its
|
||||
// own, so a stuck "running" run can no longer block the sweep. Idempotent and
|
||||
// safe: clears at most one row's worth of lock columns per candidate.
|
||||
async function sweepStaleIssueLocks() {
|
||||
async function sweepStaleIssueLocks(opts?: { companyId?: string; issueId?: string; onDeferredDeadline?: (at: number) => void }) {
|
||||
const result = {
|
||||
cleared: 0,
|
||||
issueIds: [] as string[],
|
||||
@@ -6366,7 +6387,9 @@ export function recoveryService(
|
||||
})
|
||||
.from(issues)
|
||||
.where(
|
||||
sql`(${issues.checkoutRunId} is not null or ${issues.executionRunId} is not null)`,
|
||||
and(sql`(${issues.checkoutRunId} is not null or ${issues.executionRunId} is not null)`,
|
||||
opts?.companyId ? eq(issues.companyId, opts.companyId) : undefined,
|
||||
opts?.issueId ? eq(issues.id, opts.issueId) : undefined),
|
||||
);
|
||||
|
||||
const referencedRunIds = [
|
||||
@@ -6381,7 +6404,8 @@ export function recoveryService(
|
||||
? await db
|
||||
.select()
|
||||
.from(heartbeatRuns)
|
||||
.where(inArray(heartbeatRuns.id, referencedRunIds))
|
||||
.where(and(inArray(heartbeatRuns.id, referencedRunIds),
|
||||
opts?.companyId ? eq(heartbeatRuns.companyId, opts.companyId) : undefined))
|
||||
: [];
|
||||
const runStatusById = new Map<string, string>();
|
||||
for (const row of runRows) runStatusById.set(row.id, row.status);
|
||||
@@ -6392,8 +6416,15 @@ export function recoveryService(
|
||||
// the same run id in a stale lock column. The terminal reference alone must
|
||||
// not terminalize a run that an active issue still owns, so exclude these
|
||||
// runs from the issue-terminal authority below.
|
||||
const ownershipReferences = opts?.issueId && referencedRunIds.length > 0
|
||||
? await db.select({ companyId: issues.companyId, status: issues.status,
|
||||
checkoutRunId: issues.checkoutRunId, executionRunId: issues.executionRunId }).from(issues).where(and(
|
||||
opts.companyId ? eq(issues.companyId, opts.companyId) : undefined,
|
||||
or(inArray(issues.checkoutRunId, referencedRunIds), inArray(issues.executionRunId, referencedRunIds)),
|
||||
))
|
||||
: candidates;
|
||||
const runIdsReferencedByActiveIssue = new Set<string>();
|
||||
for (const issue of candidates) {
|
||||
for (const issue of ownershipReferences) {
|
||||
if (issue.status === "done" || issue.status === "cancelled") continue;
|
||||
for (const runId of [issue.checkoutRunId, issue.executionRunId]) {
|
||||
if (runId) runIdsReferencedByActiveIssue.add(runId);
|
||||
@@ -6434,6 +6465,7 @@ export function recoveryService(
|
||||
referencingIssueTerminalStatus:
|
||||
issueTerminalStatusByRunId.get(row.id) ?? null,
|
||||
runReferencedByActiveIssue: runIdsReferencedByActiveIssue.has(row.id),
|
||||
onDeferredDeadline: opts?.onDeferredDeadline,
|
||||
});
|
||||
runStatusById.set(row.id, outcome.status);
|
||||
if (outcome.terminalized) result.terminalizedRunIds.push(row.id);
|
||||
@@ -6537,6 +6569,8 @@ export function recoveryService(
|
||||
recordWatchdogDecision,
|
||||
scanSilentActiveRuns,
|
||||
reconcileStrandedAssignedIssues,
|
||||
reconcileUnassignedBlockingIssues,
|
||||
reconcileActiveRecoveryActions,
|
||||
reconcileLegacyContinuation,
|
||||
legacyRepairDispatchBlock,
|
||||
sweepStaleIssueLocks,
|
||||
|
||||
@@ -0,0 +1,113 @@
|
||||
import { registerTaskWatchdogWork } from "../task-watchdog-work.js";
|
||||
import { and, asc, eq, gt, inArray, isNotNull, isNull, or, sql } from "drizzle-orm";
|
||||
import { issueRecoveryActions, issueRelations, issues, type Db } from "@paperclipai/db";
|
||||
import type { createDeliveryWorkCoordinator } from "../delivery-work-coordinator.js";
|
||||
import { DELIVERY_QUEUES, type DeliveryQueue } from "../delivery-work-notifications.js";
|
||||
import type { IssueRecoveryScope } from "../issue-recovery-notifications.js";
|
||||
import { registerScopedDeadlineWork } from "../scoped-deadline-work.js";
|
||||
|
||||
type Entry = { id: string; companyId: string; dueAt: number };
|
||||
type Scope = { companyId: string; issueId: string; onDeferredDeadline: (at: number) => void };
|
||||
const PAGE_SIZE = 100;
|
||||
|
||||
export function registerIssueRecoveryWork(
|
||||
db: Db,
|
||||
coordinator: ReturnType<typeof createDeliveryWorkCoordinator>,
|
||||
owner: {
|
||||
stranded(scope: Scope): Promise<unknown>;
|
||||
orphanBlockers(scope: { companyId: string }): Promise<unknown>;
|
||||
recoveryActions(scope: { companyId: string }): Promise<unknown>;
|
||||
dependencies(scope: Scope): Promise<{ enqueueFailed: number }>;
|
||||
locks(scope: Scope): Promise<unknown>;
|
||||
watchdogs(scope: { companyId: string; watchdogId: string; onDeferredDeadline: (at: number) => void }): Promise<{ incomplete: number }>;
|
||||
schedulingSuppressed(): Promise<boolean>;
|
||||
},
|
||||
canRun: () => boolean,
|
||||
) {
|
||||
function register(queue: DeliveryQueue, statuses: string[] | null, check: (scope: Scope) => Promise<unknown>) {
|
||||
const cursors = new Map<string, string>();
|
||||
let enabled = false;
|
||||
return registerScopedDeadlineWork<Entry, IssueRecoveryScope>(coordinator, {
|
||||
queue,
|
||||
reconstructionIntervalMs: 30_000,
|
||||
beforePass: async () => { enabled = !(await owner.schedulingSuppressed()); },
|
||||
decodeScope: hint => hint === "*" ? undefined : JSON.parse(hint) as IssueRecoveryScope,
|
||||
matches: (entry, scope) => !scope || (entry.companyId === scope.companyId && (!scope.issueId || entry.id === scope.issueId)),
|
||||
load: async scope => {
|
||||
const key = scope ? JSON.stringify([scope.companyId, scope.issueId]) : "*";
|
||||
if (!enabled) { cursors.delete(key); return []; }
|
||||
const cursor = cursors.get(key);
|
||||
const rows = await db.select({ id: issues.id, companyId: issues.companyId }).from(issues).where(and(
|
||||
scope ? eq(issues.companyId, scope.companyId) : undefined,
|
||||
scope?.issueId ? eq(issues.id, scope.issueId) : undefined,
|
||||
cursor ? gt(issues.id, cursor) : undefined,
|
||||
statuses ? inArray(issues.status, statuses) : or(isNotNull(issues.checkoutRunId), isNotNull(issues.executionRunId)),
|
||||
statuses ? isNull(issues.hiddenAt) : undefined,
|
||||
statuses ? isNull(issues.assigneeUserId) : undefined,
|
||||
statuses ? or(isNotNull(issues.assigneeAgentId), eq(issues.status, "in_review")) : undefined,
|
||||
)).orderBy(asc(issues.id)).limit(PAGE_SIZE);
|
||||
const complete = rows.length < PAGE_SIZE;
|
||||
if (complete) cursors.delete(key); else cursors.set(key, rows.at(-1)!.id);
|
||||
return { rows: rows.map(row => ({ ...row, dueAt: Date.now() })), complete };
|
||||
},
|
||||
check: async entry => {
|
||||
if (!enabled) return null;
|
||||
let next: number | null = null;
|
||||
await check({ companyId: entry.companyId, issueId: entry.id,
|
||||
onDeferredDeadline: at => { next = next === null ? at : Math.min(next, at); } });
|
||||
return next === null ? null : { ...entry, dueAt: next };
|
||||
},
|
||||
}, canRun);
|
||||
}
|
||||
function registerCompany(queue: DeliveryQueue, kind: "blockers" | "actions", check: (scope: { companyId: string }) => Promise<unknown>) {
|
||||
const cursors = new Map<string, string>();
|
||||
let enabled = false;
|
||||
return registerScopedDeadlineWork<Entry, IssueRecoveryScope>(coordinator, {
|
||||
queue,
|
||||
reconstructionIntervalMs: 30_000,
|
||||
beforePass: async () => { enabled = !(await owner.schedulingSuppressed()); },
|
||||
decodeScope: hint => hint === "*" ? undefined : JSON.parse(hint) as IssueRecoveryScope,
|
||||
matches: (entry, scope) => !scope || entry.companyId === scope.companyId,
|
||||
load: async scope => {
|
||||
const key = scope?.companyId ?? "*";
|
||||
if (!enabled) { cursors.delete(key); return []; }
|
||||
const cursor = cursors.get(key);
|
||||
const rows = kind === "actions"
|
||||
? await db.selectDistinct({ id: issueRecoveryActions.companyId, companyId: issueRecoveryActions.companyId })
|
||||
.from(issueRecoveryActions).where(and(inArray(issueRecoveryActions.status, ["active", "escalated"]),
|
||||
scope ? eq(issueRecoveryActions.companyId, scope.companyId) : undefined,
|
||||
cursor ? gt(issueRecoveryActions.companyId, cursor) : undefined))
|
||||
.orderBy(asc(issueRecoveryActions.companyId)).limit(PAGE_SIZE)
|
||||
: await db.selectDistinct({ id: issues.companyId, companyId: issues.companyId }).from(issues).where(and(
|
||||
inArray(issues.status, ["todo", "blocked"]), isNull(issues.assigneeAgentId), isNull(issues.assigneeUserId),
|
||||
isNotNull(issues.createdByAgentId),
|
||||
scope ? eq(issues.companyId, scope.companyId) : undefined,
|
||||
cursor ? gt(issues.companyId, cursor) : undefined,
|
||||
sql`exists (select 1 from ${issueRelations} r inner join ${issues} dependent
|
||||
on dependent.id = r.related_issue_id and dependent.company_id = r.company_id
|
||||
where r.company_id = ${issues.companyId} and r.issue_id = ${issues.id}
|
||||
and r.type = 'blocks' and dependent.status not in ('done', 'cancelled'))`,
|
||||
)).orderBy(asc(issues.companyId)).limit(PAGE_SIZE);
|
||||
const complete = rows.length < PAGE_SIZE;
|
||||
if (complete) cursors.delete(key); else cursors.set(key, rows.at(-1)!.id);
|
||||
return { rows: rows.map(row => ({ ...row, dueAt: Date.now() })), complete };
|
||||
},
|
||||
check: async entry => { if (enabled) await check({ companyId: entry.companyId }); return null; },
|
||||
}, canRun);
|
||||
}
|
||||
const assignments = register(DELIVERY_QUEUES.strandedAssignments, ["todo", "in_progress", "in_review"], owner.stranded);
|
||||
const orphanBlockers = registerCompany(DELIVERY_QUEUES.orphanBlockers, "blockers", owner.orphanBlockers);
|
||||
const recoveryActions = registerCompany(DELIVERY_QUEUES.activeRecoveryActions, "actions", owner.recoveryActions);
|
||||
const dependencies = register(DELIVERY_QUEUES.resolvedDependencies, ["blocked"], async scope => {
|
||||
const result = await owner.dependencies(scope);
|
||||
if (result.enqueueFailed > 0) throw new Error("Resolved dependency wake remains undelivered");
|
||||
});
|
||||
const locks = register(DELIVERY_QUEUES.staleIssueLocks, null, owner.locks);
|
||||
const watchdogs = registerTaskWatchdogWork(db, coordinator, {
|
||||
reconcile: owner.watchdogs, enabled: async () => !(await owner.schedulingSuppressed()),
|
||||
}, canRun);
|
||||
const workers = [assignments, orphanBlockers, recoveryActions, dependencies, locks, watchdogs];
|
||||
return { ready: Promise.all(workers.map(worker => worker.ready)),
|
||||
wake() { for (const worker of workers) worker.invalidate(); },
|
||||
async stop() { await Promise.all(workers.map(worker => worker.stop())); } };
|
||||
}
|
||||
@@ -1,14 +1,89 @@
|
||||
import { notifyApprovalWork } from "./approval-work-notifications.js";
|
||||
import { RECOVERY_ORIGIN_KINDS, parseIssueGraphLivenessIncidentKey } from "./recovery/origins.js";
|
||||
import { sql } from "drizzle-orm";
|
||||
import type { Db } from "@paperclipai/db";
|
||||
import type { AttentionSourceKind } from "@paperclipai/shared";
|
||||
import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js";
|
||||
|
||||
export async function notifyRetentionWork(transaction: object, kind: AttentionSourceKind) {
|
||||
export const ATTENTION_RETENTION_MAX_ISSUE_IDS = 100;
|
||||
export type AttentionRetentionScope = { companyId?: string; sourceKind?: AttentionSourceKind;
|
||||
issueId?: string; issueIds?: string[]; sourceId?: string };
|
||||
|
||||
export async function notifyAttentionRetentionWork(transaction: object, scope: AttentionRetentionScope = {}) {
|
||||
const issueIds = [...new Set([...(scope.issueIds ?? []), ...(scope.issueId ? [scope.issueId] : [])])].sort();
|
||||
await notifyDeliveryWork(transaction, DELIVERY_QUEUES.attentionRetention, JSON.stringify({
|
||||
companyId: scope.companyId, sourceKind: scope.sourceKind, sourceId: scope.sourceId,
|
||||
issueId: issueIds.length === 1 ? issueIds[0] : undefined,
|
||||
issueIds: issueIds.length > 1 && issueIds.length <= ATTENTION_RETENTION_MAX_ISSUE_IDS ? issueIds : undefined,
|
||||
}));
|
||||
}
|
||||
|
||||
export async function notifyIssueAttentionRetentionWork(transaction: object,
|
||||
scope: { companyId: string; issueId?: string; parentId?: string | null; runId?: string; originKind?: string | null; originId?: string | null }) {
|
||||
const database = transaction as Db;
|
||||
const seeds = [scope.issueId, scope.parentId].filter((id): id is string => Boolean(id));
|
||||
if (!seeds.length && scope.runId) {
|
||||
const rows = await database.execute(sql`select issue_id::text as id from heartbeat_runs
|
||||
where company_id = ${scope.companyId}::uuid and id = ${scope.runId}::uuid and issue_id is not null
|
||||
union select native_issue_id::text from heartbeat_runs
|
||||
where company_id = ${scope.companyId}::uuid and id = ${scope.runId}::uuid and native_issue_id is not null
|
||||
union select issue.id::text from heartbeat_runs run inner join issues issue
|
||||
on issue.company_id = run.company_id
|
||||
and issue.id::text = coalesce(run.context_snapshot->>'issueId', run.context_snapshot->>'taskId')
|
||||
where run.company_id = ${scope.companyId}::uuid and run.id = ${scope.runId}::uuid`);
|
||||
for (const row of rows) seeds.push(row.id as string);
|
||||
}
|
||||
if (!seeds.length) return notifyAttentionRetentionWork(transaction, { companyId: scope.companyId });
|
||||
const origins = await database.execute(sql`select id::text as id, origin_kind as "originKind", origin_id as "originId" from issues
|
||||
where company_id = ${scope.companyId}::uuid
|
||||
and id in (${sql.join([...new Set(seeds)].map(id => sql`${id}::uuid`), sql`, `)})`);
|
||||
const current = origins.find(source => source.id === scope.issueId);
|
||||
origins.push({ originKind: scope.originKind ?? current?.originKind, originId: scope.originId ?? current?.originId });
|
||||
const references: string[] = [];
|
||||
for (const source of origins) {
|
||||
if (source.originKind === RECOVERY_ORIGIN_KINDS.strandedIssueRecovery && typeof source.originId === "string") {
|
||||
references.push(source.originId);
|
||||
}
|
||||
const incident = source.originKind === RECOVERY_ORIGIN_KINDS.issueGraphLivenessEscalation
|
||||
? parseIssueGraphLivenessIncidentKey(source.originId as string | null) : null;
|
||||
if (incident?.companyId === scope.companyId) references.push(incident.issueId, incident.leafIssueId);
|
||||
}
|
||||
if (references.length) {
|
||||
const trusted = await database.execute(sql`select id::text as id from issues
|
||||
where company_id = ${scope.companyId}::uuid
|
||||
and id::text in (${sql.join([...new Set(references)].map(id => sql`${id}`), sql`, `)})`);
|
||||
for (const row of trusted) seeds.push(row.id as string);
|
||||
}
|
||||
// UNION deduplicates visited UUIDs (including cycles). Each relation edge is
|
||||
// unique by schema, so 101 neighbours guarantee overflow of a 100-ID scope.
|
||||
// Bound that adjacency before materialization and stream at most 101 unique
|
||||
// IDs to the caller; an inner UNION DISTINCT would consume the full fan-in.
|
||||
const affected = await database.execute(sql`with recursive affected(id) as (
|
||||
select unnest(array[${sql.join([...new Set(seeds)].map(id => sql`${id}::uuid`), sql`, `)}])
|
||||
union
|
||||
select parent.id from affected current cross join lateral (
|
||||
(select related_issue_id as id from issue_relations where company_id = ${scope.companyId}::uuid
|
||||
and issue_id = current.id and type = 'blocks' limit ${ATTENTION_RETENTION_MAX_ISSUE_IDS + 1})
|
||||
union all select parent_id from issues where company_id = ${scope.companyId}::uuid
|
||||
and id = current.id and parent_id is not null
|
||||
) parent
|
||||
) select id::text as "issueId" from affected limit ${ATTENTION_RETENTION_MAX_ISSUE_IDS + 1}`);
|
||||
// The sentinel means the affected set is incomplete. Reconstruct the whole
|
||||
// company through the existing paged authority instead of dropping ancestors.
|
||||
await notifyAttentionRetentionWork(transaction, { companyId: scope.companyId,
|
||||
issueIds: affected.length <= ATTENTION_RETENTION_MAX_ISSUE_IDS ? affected.map(row => row.issueId as string) : undefined,
|
||||
});
|
||||
}
|
||||
|
||||
export async function notifyRetentionWork(transaction: object, kind: AttentionSourceKind, scope: { companyId?: string; sourceId?: string } = {}) {
|
||||
if (kind === "approval") await notifyDeliveryWork(transaction, DELIVERY_QUEUES.approvalRetention);
|
||||
if (kind === "join_request") await notifyDeliveryWork(transaction, DELIVERY_QUEUES.joinRequestRetention);
|
||||
if (kind !== "approval" && kind !== "join_request") await notifyAttentionRetentionWork(transaction, { ...scope, sourceKind: kind,
|
||||
sourceId: kind === "blocker_attention" ? undefined : scope.sourceId });
|
||||
}
|
||||
|
||||
export async function notifyCompanyRetentionWork(transaction: object, companyId: string) {
|
||||
await notifyAttentionRetentionWork(transaction, { companyId });
|
||||
await notifyApprovalWork(transaction, companyId);
|
||||
await notifyRetentionWork(transaction, "join_request");
|
||||
}
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { notifyStatusCardIssueChange } from "./status-card-notifications.js";
|
||||
import { notifyApprovalWork } from "./approval-work-notifications.js";
|
||||
import { notifyDecisionTargetIssueRemoval } from "./decision-targets.js";
|
||||
import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js";
|
||||
@@ -1949,6 +1950,7 @@ export function routineService(
|
||||
if (createdIssue) {
|
||||
await notifyApprovalWork(txDb, createdIssue.companyId);
|
||||
await notifyDecisionTargetIssueRemoval(txDb, createdIssue.id);
|
||||
await notifyStatusCardIssueChange(txDb, createdIssue.companyId);
|
||||
await txDb.delete(issues).where(eq(issues.id, createdIssue.id));
|
||||
}
|
||||
const failureReason = error instanceof Error ? error.message : String(error);
|
||||
|
||||
@@ -0,0 +1,11 @@
|
||||
import { notifyIssueRecovery } from "./issue-recovery-notifications.js";
|
||||
import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js";
|
||||
|
||||
export type RunDispatchScope = { companyId: string };
|
||||
|
||||
export async function notifyRunDispatch(transaction: object, scope?: RunDispatchScope) {
|
||||
await notifyIssueRecovery(transaction, scope ? { companyId: scope.companyId } : undefined);
|
||||
const hint = scope ? JSON.stringify({ companyId: scope.companyId }) : "*";
|
||||
await notifyDeliveryWork(transaction, DELIVERY_QUEUES.scheduledRetryDispatch, hint);
|
||||
await notifyDeliveryWork(transaction, DELIVERY_QUEUES.queuedRunDispatch, hint);
|
||||
}
|
||||
@@ -1,6 +1,7 @@
|
||||
import { createHmac, generateKeyPairSync, randomUUID, sign } from "node:crypto";
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { requestSandboxAbsenceProof, verifySandboxAbsenceReceipt } from "./sandbox-absence-proof.js";
|
||||
import { inspectCloudSandboxCleanupAbsence, requestSandboxAbsenceProof, verifySandboxAbsenceReceipt,
|
||||
verifySandboxCleanupAbsenceReceipt } from "./sandbox-absence-proof.js";
|
||||
|
||||
const { privateKey, publicKey } = generateKeyPairSync("ed25519");
|
||||
const binding = { companyId: randomUUID(), issueId: randomUUID(), runId: randomUUID(), leaseId: randomUUID(),
|
||||
@@ -9,10 +10,10 @@ const env = { PAPERCLIP_CLOUD_STACK_ID: "fixture-stack", PAPERCLIP_CLOUD_API_ORI
|
||||
PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN: "fleet-token", PAPERCLIP_PRIMARY_HOST: "fixture.paperclip.test", PAPERCLIP_AUTH_SECRET: "fixture-secret",
|
||||
PAPERCLIP_CLOUD_RUNTIME_IDENTITY_JWKS: JSON.stringify({ keys: [{ ...publicKey.export({ format: "jwk" }), kid: "fixture", alg: "EdDSA", use: "sig" }] }) };
|
||||
const now = new Date("2099-01-01T00:00:00Z"), seconds = now.getTime() / 1000;
|
||||
function receipt(overrides: Record<string, unknown> = {}) {
|
||||
const header = Buffer.from(JSON.stringify({ alg: "EdDSA", kid: "fixture", typ: "paperclip-sandbox-absence+jwt" })).toString("base64url");
|
||||
const payload = Buffer.from(JSON.stringify({ v: 1, iss: "paperclip-cloud", aud: "paperclip-sandbox-absence/v1", sub: "fixture-stack",
|
||||
...binding, absent: true, iat: seconds, exp: seconds + 60, ...overrides })).toString("base64url");
|
||||
function receipt(overrides: Record<string, unknown> = {}, purpose = "sandbox-absence", source: Record<string, unknown> = binding) {
|
||||
const header = Buffer.from(JSON.stringify({ alg: "EdDSA", kid: "fixture", typ: `paperclip-${purpose}+jwt` })).toString("base64url");
|
||||
const payload = Buffer.from(JSON.stringify({ v: 1, iss: "paperclip-cloud", aud: `paperclip-${purpose}/v1`, sub: "fixture-stack",
|
||||
...source, absent: true, iat: seconds, exp: seconds + 60, ...overrides })).toString("base64url");
|
||||
return `${header}.${payload}.${sign(null, Buffer.from(`${header}.${payload}`), privateKey).toString("base64url")}`;
|
||||
}
|
||||
describe("sandbox absence proof", () => {
|
||||
@@ -45,3 +46,44 @@ describe("sandbox absence proof", () => {
|
||||
expect(fetchImpl).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
describe("cleanup-only sandbox absence", () => {
|
||||
const cleanup = { purpose: "cleanup" as const, companyId: binding.companyId, environmentId: randomUUID(),
|
||||
leaseId: binding.leaseId, heartbeatRunId: null as string | null, provider: binding.provider, providerLeaseId: binding.providerLeaseId };
|
||||
const lease = { id: cleanup.leaseId, ...cleanup };
|
||||
const signed = (overrides: Record<string, unknown> = {}) => receipt(overrides, "sandbox-cleanup-absence", cleanup);
|
||||
it.each([null, binding.runId])("accepts exact absence with nullable run %s without issuing a run receipt", async heartbeatRunId => {
|
||||
const source = { ...cleanup, heartbeatRunId };
|
||||
const fetchImpl = vi.fn().mockResolvedValue(Response.json({ receipt: receipt({}, "sandbox-cleanup-absence", source) }));
|
||||
expect(await inspectCloudSandboxCleanupAbsence({ ...lease, heartbeatRunId }, { env, now, fetchImpl }))
|
||||
.toEqual({ providerLeaseId: cleanup.providerLeaseId, state: "destroyed" });
|
||||
const [url, init] = fetchImpl.mock.calls[0];
|
||||
expect(url).toBe("https://cloud.example.test/v1/tenant/sandbox-absence");
|
||||
expect(JSON.parse(init.body)).toEqual(source);
|
||||
expect(init.headers.authorization).not.toContain(env.PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN);
|
||||
expect(init.redirect).toBe("error");
|
||||
});
|
||||
it("requires every binding, current stack, signature and freshness, and rejects recovery receipts", () => {
|
||||
expect(verifySandboxCleanupAbsenceReceipt(signed(), cleanup, env, now)).toMatchObject(cleanup);
|
||||
for (const overrides of [
|
||||
...Object.keys(cleanup).map(key => ({ [key]: randomUUID() })),
|
||||
{ sub: "foreign-stack" }, { absent: false }, { aud: "paperclip-sandbox-absence/v1" }, { iss: "other" },
|
||||
{ iat: seconds + 31 }, { exp: seconds }, { exp: seconds + 61 },
|
||||
]) expect(verifySandboxCleanupAbsenceReceipt(signed(overrides), cleanup, env, now)).toBeNull();
|
||||
expect(verifySandboxCleanupAbsenceReceipt(signed().slice(0, -8) + "tampered", cleanup, env, now)).toBeNull();
|
||||
expect(verifySandboxCleanupAbsenceReceipt(receipt(), cleanup, env, now)).toBeNull();
|
||||
expect(verifySandboxAbsenceReceipt(signed(), binding, env, now)).toBeNull();
|
||||
});
|
||||
it("keeps old Clouds, outages, invalid replies and unconfigured instances inconclusive", async () => {
|
||||
for (const answer of [{ absent: false }, { absent: true }, { receipt: "forged" }, { receipt: signed({ heartbeatRunId: binding.runId }) }])
|
||||
expect(await inspectCloudSandboxCleanupAbsence(lease, { env, now, fetchImpl: vi.fn().mockResolvedValue(Response.json(answer)) })).toBeUndefined();
|
||||
for (const status of [400, 401, 403, 503]) expect(await inspectCloudSandboxCleanupAbsence(lease,
|
||||
{ env, now, fetchImpl: vi.fn().mockResolvedValue(new Response(null, { status })) })).toBeUndefined();
|
||||
expect(await inspectCloudSandboxCleanupAbsence(lease, { env, now, fetchImpl: vi.fn().mockRejectedValue(new Error("offline")) })).toBeUndefined();
|
||||
const fetchImpl = vi.fn();
|
||||
expect(await inspectCloudSandboxCleanupAbsence(lease, { env: {}, now, fetchImpl })).toBeUndefined();
|
||||
for (const change of [{ provider: "other" }, { environmentId: null }, { providerLeaseId: "invalid" }])
|
||||
expect(await inspectCloudSandboxCleanupAbsence({ ...lease, ...change }, { env, now, fetchImpl })).toBeUndefined();
|
||||
expect(fetchImpl).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
@@ -7,8 +7,18 @@ const bindingSchema = z.object({
|
||||
companyId: z.string().uuid(), issueId: z.string().uuid(), runId: z.string().uuid(),
|
||||
leaseId: z.string().uuid(), ownerToken: z.string().uuid(), provider: z.literal("daytona"),
|
||||
providerLeaseId: z.string().uuid(),
|
||||
});
|
||||
}).strict();
|
||||
const cleanupBindingSchema = z.object({
|
||||
purpose: z.literal("cleanup"), companyId: z.string().uuid(), environmentId: z.string().uuid(),
|
||||
leaseId: z.string().uuid(), heartbeatRunId: z.string().uuid().nullable(),
|
||||
provider: z.literal("daytona"), providerLeaseId: z.string().uuid(),
|
||||
}).strict();
|
||||
export type SandboxAbsenceBinding = z.infer<typeof bindingSchema>;
|
||||
export type SandboxCleanupAbsenceBinding = z.infer<typeof cleanupBindingSchema>;
|
||||
export type SandboxCleanupAbsenceProof = SandboxCleanupAbsenceBinding & {
|
||||
v: 1; iss: "paperclip-cloud"; aud: "paperclip-sandbox-cleanup-absence/v1"; sub: string;
|
||||
absent: true; iat: number; exp: number;
|
||||
};
|
||||
export type SandboxAbsenceProof = SandboxAbsenceBinding & {
|
||||
v: 1; iss: "paperclip-cloud"; aud: "paperclip-sandbox-absence/v1"; sub: string;
|
||||
absent: true; iat: number; exp: number;
|
||||
@@ -16,19 +26,30 @@ export type SandboxAbsenceProof = SandboxAbsenceBinding & {
|
||||
|
||||
export function verifySandboxAbsenceReceipt(receipt: string, binding: SandboxAbsenceBinding,
|
||||
env: NodeJS.ProcessEnv = process.env, now = new Date()): SandboxAbsenceProof | null {
|
||||
const parsed = bindingSchema.safeParse(binding);
|
||||
return parsed.success ? verifyReceipt(receipt, parsed.data, "sandbox-absence", env, now) as SandboxAbsenceProof | null : null;
|
||||
}
|
||||
|
||||
export function verifySandboxCleanupAbsenceReceipt(receipt: string, binding: SandboxCleanupAbsenceBinding,
|
||||
env: NodeJS.ProcessEnv = process.env, now = new Date()): SandboxCleanupAbsenceProof | null {
|
||||
const parsed = cleanupBindingSchema.safeParse(binding);
|
||||
return parsed.success ? verifyReceipt(receipt, parsed.data, "sandbox-cleanup-absence", env, now) as SandboxCleanupAbsenceProof | null : null;
|
||||
}
|
||||
|
||||
function verifyReceipt(receipt: string, binding: Record<string, unknown>, purpose: string,
|
||||
env: NodeJS.ProcessEnv, now: Date): Record<string, unknown> | null {
|
||||
try {
|
||||
const parsed = bindingSchema.safeParse(binding);
|
||||
if (receipt.length > 16_000 || !parsed.success) return null;
|
||||
if (receipt.length > 16_000) return null;
|
||||
const parts = receipt.split(".");
|
||||
if (parts.length !== 3 || parts.some(part => !/^[A-Za-z0-9_-]+$/.test(part))) return null;
|
||||
const header = JSON.parse(Buffer.from(parts[0], "base64url").toString("utf8"));
|
||||
if (header.alg !== "EdDSA" || header.typ !== "paperclip-sandbox-absence+jwt" || typeof header.kid !== "string") return null;
|
||||
if (header.alg !== "EdDSA" || header.typ !== `paperclip-${purpose}+jwt` || typeof header.kid !== "string") return null;
|
||||
if (!verify(null, Buffer.from(`${parts[0]}.${parts[1]}`, "ascii"), cloudRuntimePublicKeyForKid(env, header.kid), Buffer.from(parts[2], "base64url"))) return null;
|
||||
const proof = JSON.parse(Buffer.from(parts[1], "base64url").toString("utf8")) as SandboxAbsenceProof;
|
||||
const proof = JSON.parse(Buffer.from(parts[1], "base64url").toString("utf8"));
|
||||
const seconds = Math.floor(now.getTime() / 1000);
|
||||
if (proof.v !== 1 || proof.iss !== "paperclip-cloud" || proof.aud !== "paperclip-sandbox-absence/v1"
|
||||
if (proof.v !== 1 || proof.iss !== "paperclip-cloud" || proof.aud !== `paperclip-${purpose}/v1`
|
||||
|| !env.PAPERCLIP_CLOUD_STACK_ID || proof.sub !== env.PAPERCLIP_CLOUD_STACK_ID || proof.absent !== true
|
||||
|| Object.keys(parsed.data).some(key => proof[key as keyof SandboxAbsenceBinding] !== parsed.data[key as keyof SandboxAbsenceBinding])
|
||||
|| Object.keys(binding).some(key => proof[key] !== binding[key])
|
||||
|| !Number.isInteger(proof.iat) || !Number.isInteger(proof.exp) || proof.iat > seconds + 30
|
||||
|| proof.exp <= seconds || proof.exp <= proof.iat || proof.exp - proof.iat > 60) return null;
|
||||
return proof;
|
||||
@@ -38,11 +59,35 @@ export function verifySandboxAbsenceReceipt(receipt: string, binding: SandboxAbs
|
||||
export async function requestSandboxAbsenceProof(binding: SandboxAbsenceBinding, options: {
|
||||
env?: NodeJS.ProcessEnv; fetchImpl?: typeof fetch; now?: Date;
|
||||
} = {}): Promise<{ proof: SandboxAbsenceProof; receipt: string } | null> {
|
||||
const parsed = bindingSchema.safeParse(binding);
|
||||
if (!parsed.success) return null;
|
||||
const receipt = await requestReceipt(parsed.data, options);
|
||||
if (!receipt) return null;
|
||||
const proof = verifySandboxAbsenceReceipt(receipt, parsed.data, options.env, options.now);
|
||||
return proof ? { proof, receipt } : null;
|
||||
}
|
||||
|
||||
/** Cleanup-only observation: never grants continuation authority to a run. */
|
||||
export async function inspectCloudSandboxCleanupAbsence(lease: {
|
||||
id: string; companyId: string; environmentId: string | null; heartbeatRunId: string | null;
|
||||
provider: string | null; providerLeaseId: string | null;
|
||||
}, options: { env?: NodeJS.ProcessEnv; fetchImpl?: typeof fetch; now?: Date } = {}) {
|
||||
const parsed = cleanupBindingSchema.safeParse({ purpose: "cleanup", companyId: lease.companyId,
|
||||
environmentId: lease.environmentId, leaseId: lease.id, heartbeatRunId: lease.heartbeatRunId,
|
||||
provider: lease.provider, providerLeaseId: lease.providerLeaseId });
|
||||
if (!parsed.success) return undefined;
|
||||
const receipt = await requestReceipt(parsed.data, options);
|
||||
if (!receipt || !verifySandboxCleanupAbsenceReceipt(receipt, parsed.data, options.env, options.now)) return undefined;
|
||||
return { providerLeaseId: parsed.data.providerLeaseId, state: "destroyed" as const };
|
||||
}
|
||||
|
||||
async function requestReceipt(binding: Record<string, unknown>, options: {
|
||||
env?: NodeJS.ProcessEnv; fetchImpl?: typeof fetch;
|
||||
}): Promise<string | null> {
|
||||
const env = options.env ?? process.env;
|
||||
const context = getCloudStackContext(env);
|
||||
const secret = env.PAPERCLIP_AUTH_SECRET ?? env.BETTER_AUTH_SECRET;
|
||||
const parsed = bindingSchema.safeParse(binding);
|
||||
if (!context?.stackId || !context.primaryHost || !context.cloudOrigin || !secret?.trim() || !parsed.success) return null;
|
||||
if (!context?.stackId || !context.primaryHost || !context.cloudOrigin || !secret?.trim()) return null;
|
||||
try {
|
||||
const origin = new URL(context.cloudOrigin);
|
||||
const host = new URL(`https://${context.primaryHost}`);
|
||||
@@ -51,12 +96,11 @@ export async function requestSandboxAbsenceProof(binding: SandboxAbsenceBinding,
|
||||
const response = await (options.fetchImpl ?? fetch)(`${origin.origin}/v1/tenant/sandbox-absence`, {
|
||||
method: "POST", redirect: "error", signal: AbortSignal.timeout(10_000),
|
||||
headers: { authorization: `Bearer ${capability}`, "x-paperclip-cloud-tenant-host": host.hostname, "content-type": "application/json" },
|
||||
body: JSON.stringify(parsed.data),
|
||||
body: JSON.stringify(binding),
|
||||
});
|
||||
if (!response.ok) return null;
|
||||
const payload = await response.json() as { receipt?: unknown };
|
||||
if (typeof payload.receipt !== "string") return null;
|
||||
const proof = verifySandboxAbsenceReceipt(payload.receipt, binding, env, options.now);
|
||||
return proof ? { proof, receipt: payload.receipt } : null;
|
||||
return payload.receipt;
|
||||
} catch { return null; }
|
||||
}
|
||||
@@ -1,7 +1,9 @@
|
||||
import { notifyRunDispatch } from "./run-dispatch-notifications.js";
|
||||
import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js";
|
||||
|
||||
export type ScheduledRetryCancellationScope = { companyId: string; runId?: string };
|
||||
|
||||
export async function notifyScheduledRetryCancellation(transaction: object, scope?: ScheduledRetryCancellationScope) {
|
||||
await notifyRunDispatch(transaction, scope);
|
||||
await notifyDeliveryWork(transaction, DELIVERY_QUEUES.scheduledRetryCancellation, scope ? JSON.stringify(scope) : "*");
|
||||
}
|
||||
@@ -12,6 +12,8 @@ export function registerScopedDeadlineWork<Entry extends ScheduledEntry, Scope>(
|
||||
queue: DeliveryQueue;
|
||||
batchSize?: number;
|
||||
minimumPassIntervalMs?: number;
|
||||
reconstructionIntervalMs?: number;
|
||||
beforePass?(): Promise<void>;
|
||||
decodeScope(hint: string): Scope | undefined;
|
||||
matches(entry: Entry, scope?: Scope): boolean;
|
||||
load(scope?: Scope, signal?: AbortSignal): Promise<Entry[] | { rows: Entry[] | null; complete: boolean }>;
|
||||
@@ -25,6 +27,7 @@ export function registerScopedDeadlineWork<Entry extends ScheduledEntry, Scope>(
|
||||
const entries = new Map<string, Entry>();
|
||||
const failures = new Map<string, { count: number; retryAt: number }>();
|
||||
const scopeFailures = new Map<string, { count: number; retryAt: number }>();
|
||||
const reconstructAfter = new Map<string, number>();
|
||||
let earliest = 0;
|
||||
let lastCompany: string | null = null;
|
||||
function failed(map: typeof failures, id: string) {
|
||||
@@ -34,13 +37,14 @@ export function registerScopedDeadlineWork<Entry extends ScheduledEntry, Scope>(
|
||||
function dirtyScopes() {
|
||||
return [...dirty].filter(hint => hint !== "*" || !scopeFailures.has(hint))
|
||||
.filter(hint => !scans.has(hint) || scopeFailures.has(hint))
|
||||
.filter(hint => (reconstructAfter.get(hint) ?? 0) <= Date.now() || scopeFailures.has(hint))
|
||||
.map(hint => { try { return options.decodeScope(hint); } catch { return undefined; } });
|
||||
}
|
||||
function next() {
|
||||
if (!canRun()) return dirty.size > 0 || entries.size > 0 ? Date.now() + 30_000 : null;
|
||||
let nextAt: number | null = null;
|
||||
const take = (at: number) => { nextAt = nextAt === null ? at : Math.min(nextAt, at); };
|
||||
for (const hint of dirty) take(Math.max(earliest, scopeFailures.get(hint)?.retryAt ?? 0));
|
||||
for (const hint of dirty) take(Math.max(earliest, reconstructAfter.get(hint) ?? 0, scopeFailures.get(hint)?.retryAt ?? 0));
|
||||
const unavailable = dirtyScopes();
|
||||
for (const entry of entries.values()) if (!unavailable.some(scope => options.matches(entry, scope)))
|
||||
take(Math.max(earliest, entry.dueAt, failures.get(entry.id)?.retryAt ?? 0));
|
||||
@@ -54,9 +58,18 @@ export function registerScopedDeadlineWork<Entry extends ScheduledEntry, Scope>(
|
||||
} },
|
||||
run: async signal => {
|
||||
if (!canRun() || Date.now() < earliest) return;
|
||||
if ((options.reconstructionIntervalMs ?? 0) > 0) for (const [hint, until] of reconstructAfter) {
|
||||
if (until <= Date.now()) reconstructAfter.delete(hint);
|
||||
}
|
||||
const ready = [...dirty].filter(hint => Math.max(reconstructAfter.get(hint) ?? 0, scopeFailures.get(hint)?.retryAt ?? 0) <= Date.now());
|
||||
if (ready.length === 0 && ![...entries.values()].some(entry => Math.max(entry.dueAt, failures.get(entry.id)?.retryAt ?? 0) <= Date.now())) return;
|
||||
earliest = Date.now() + (options.minimumPassIntervalMs ?? 1000);
|
||||
try { await options.beforePass?.(); } catch (error) {
|
||||
for (const hint of ready) failed(scopeFailures, hint);
|
||||
for (const entry of entries.values()) if (entry.dueAt <= Date.now()) failed(failures, entry.id);
|
||||
throw error;
|
||||
}
|
||||
const errors: unknown[] = [];
|
||||
const ready = [...dirty].filter(hint => (scopeFailures.get(hint)?.retryAt ?? 0) <= Date.now());
|
||||
const requested = ready.includes("*")
|
||||
? ["*", ...ready.filter(hint => hint !== "*")].slice(0, 25)
|
||||
: ready.slice(0, 25);
|
||||
@@ -81,6 +94,9 @@ export function registerScopedDeadlineWork<Entry extends ScheduledEntry, Scope>(
|
||||
entries.delete(entry.id); failures.delete(entry.id);
|
||||
}
|
||||
scans.delete(hint);
|
||||
if ((options.reconstructionIntervalMs ?? 0) > 0) {
|
||||
reconstructAfter.set(hint, Date.now() + options.reconstructionIntervalMs!);
|
||||
}
|
||||
if ((versions.get(hint) ?? 0) !== scan.version) dirty.add(hint); else versions.delete(hint);
|
||||
if (!scope) scopeFailures.clear(); else scopeFailures.delete(hint);
|
||||
} catch (error) {
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
import { notifyAttentionRetentionWork } from "./retention-work-notifications.js";
|
||||
import { notifyStatusCardIssueChange } from "./status-card-notifications.js";
|
||||
import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js";
|
||||
import { notifyUnsafeWorkspaceExportRecovery } from "./unsafe-workspace-export-notifications.js";
|
||||
import { and, asc, desc, eq, gt, inArray, isNull, sql } from "drizzle-orm";
|
||||
@@ -126,6 +128,7 @@ export async function settleExternalConversation(db: Db, companyId: string, issu
|
||||
const ready = await readSettlement(tx, companyId, issueId, true);
|
||||
if (!ready) return false;
|
||||
const { binding, issue, run } = ready;
|
||||
await notifyAttentionRetentionWork(tx, { companyId });
|
||||
const changed = await tx.update(chatConversations).set({ state: "waiting", updatedAt: new Date() })
|
||||
.where(and(eq(chatConversations.id, binding.conversation.id), eq(chatConversations.companyId, companyId), eq(chatConversations.state, "active")))
|
||||
.returning({ id: chatConversations.id });
|
||||
@@ -133,6 +136,7 @@ export async function settleExternalConversation(db: Db, companyId: string, issu
|
||||
await notifyUnsafeWorkspaceExportRecovery(tx, { companyId, issueId });
|
||||
await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors);
|
||||
await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers);
|
||||
await notifyStatusCardIssueChange(tx, companyId);
|
||||
await tx.update(issues).set({ status: "in_review", updatedAt: new Date() })
|
||||
.where(and(eq(issues.id, issueId), eq(issues.companyId, companyId)));
|
||||
activity = (await persistActivity(tx as unknown as Db, {
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
import { notifyAttentionRetentionWork } from "./retention-work-notifications.js";
|
||||
import { notifyStatusCardIssueChange } from "./status-card-notifications.js";
|
||||
import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js";
|
||||
import { notifyExternalConversationSettlement } from "./external-conversation-settlement-notifications.js";
|
||||
import { chatConversations, issues, type Db } from "@paperclipai/db";
|
||||
@@ -45,10 +47,12 @@ export function nonIdleSlackIssueCondition() {
|
||||
|
||||
/** Called inside message admission while its transaction owns the issue lock. */
|
||||
export async function resumeExternalConversation(tx: Db, companyId: string, issueId: string) {
|
||||
await notifyAttentionRetentionWork(tx, { companyId });
|
||||
await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers);
|
||||
await notifyExternalConversationSettlement(tx, { companyId, issueId });
|
||||
// Only the verified idle convention may leave In Review automatically.
|
||||
// A real decision or delivery problem keeps the existing review semantics.
|
||||
await notifyStatusCardIssueChange(tx, companyId);
|
||||
await tx.update(issues).set({ status: "todo", updatedAt: new Date() })
|
||||
.where(and(eq(issues.companyId, companyId), eq(issues.id, issueId),
|
||||
sql`${externalConversationStateSql()} = 'waiting'`));
|
||||
|
||||
@@ -0,0 +1,98 @@
|
||||
import { and, asc, eq, isNotNull, isNull } from "drizzle-orm";
|
||||
import { companies, statusCards, type Db } from "@paperclipai/db";
|
||||
import { logger } from "../middleware/logger.js";
|
||||
import { registerDeadlineWork } from "./deadline-work.js";
|
||||
import type { createDeliveryWorkCoordinator } from "./delivery-work-coordinator.js";
|
||||
import { DELIVERY_QUEUES } from "./delivery-work-notifications.js";
|
||||
import { instanceSettingsService } from "./instance-settings.js";
|
||||
import { issueService } from "./issues.js";
|
||||
import { queueIssueAssignmentWakeup, type IssueAssignmentWakeupDeps } from "./issue-assignment-wakeup.js";
|
||||
import { statusCardService } from "./status-cards.js";
|
||||
|
||||
/** Feature-owned generation and assignment. Existing issue services retain
|
||||
* execution authority, durable wake dedupe and stalled-generation cleanup. */
|
||||
export function statusCardEvaluationService(db: Db, heartbeat: IssueAssignmentWakeupDeps) {
|
||||
const cards = statusCardService(db);
|
||||
const issues = issueService(db);
|
||||
const settings = instanceSettingsService(db);
|
||||
let reconstructReactive = true;
|
||||
let allChanges = false;
|
||||
const changes = new Set<string>();
|
||||
let enabled: boolean | null = null;
|
||||
let retryAt: number | null = null;
|
||||
|
||||
return {
|
||||
changed(hints?: ReadonlySet<string>) {
|
||||
for (const hint of hints ?? ["*"]) {
|
||||
if (hint === "*" || hint === "changes:*") { allChanges = true; if (hint === "*") enabled = null; }
|
||||
else if (hint.startsWith("changes:")) changes.add(hint.slice("changes:".length));
|
||||
}
|
||||
},
|
||||
async run(signal: AbortSignal) {
|
||||
if (enabled === null) enabled = (await settings.getExperimental()).enableStatusCards === true;
|
||||
if (!enabled || signal.aborted) return;
|
||||
const reconstruct = reconstructReactive;
|
||||
const evaluateAllChanges = allChanges;
|
||||
const companiesChanged = new Set(changes);
|
||||
changes.clear();
|
||||
reconstructReactive = false;
|
||||
allChanges = false;
|
||||
try {
|
||||
const result = await cards.tickDueStatusCards(new Date(), evaluateAllChanges ? "changed" : companiesChanged, reconstruct);
|
||||
retryAt = result.deferredCompanyIds.size ? Date.now() + 30_000 : null;
|
||||
for (const companyId of result.deferredCompanyIds) changes.add(companyId);
|
||||
const assignments = await Promise.allSettled(result.enqueued.map(async ({ cardId, generatingIssue }) => {
|
||||
// Once generated, dispatch or cancel even if shutdown begins: leaving
|
||||
// an unassigned claimed card is not a completed sweep.
|
||||
try {
|
||||
await queueIssueAssignmentWakeup({
|
||||
heartbeat, issue: generatingIssue,
|
||||
reason: "status_card_update_assigned",
|
||||
mutation: "status_card.scheduler_update_requested",
|
||||
contextSource: "status_card_scheduler",
|
||||
requestedByActorType: "system", taskKey: `status-card:${cardId}`,
|
||||
rethrowOnError: true,
|
||||
});
|
||||
} catch (error) {
|
||||
await issues.update(generatingIssue.id, { status: "cancelled" });
|
||||
throw error;
|
||||
}
|
||||
}));
|
||||
const failures = assignments.filter(result => result.status === "rejected");
|
||||
if (failures.length) throw new AggregateError(failures.map(result => result.reason), "Status-card assignment failed");
|
||||
if (result.evaluated || result.enqueued.length) {
|
||||
logger.info({ evaluated: result.evaluated, enqueued: result.enqueued.length }, "status-card evaluation complete");
|
||||
}
|
||||
} catch (error) {
|
||||
reconstructReactive ||= reconstruct;
|
||||
allChanges ||= evaluateAllChanges;
|
||||
for (const companyId of companiesChanged) changes.add(companyId);
|
||||
throw error;
|
||||
}
|
||||
},
|
||||
async next() {
|
||||
if (!enabled) return null;
|
||||
const [next] = await db.select({ at: statusCards.nextEvalAt }).from(statusCards)
|
||||
.innerJoin(companies, eq(companies.id, statusCards.companyId))
|
||||
.where(and(eq(companies.status, "active"), isNull(statusCards.archivedAt),
|
||||
isNull(statusCards.generatingIssueId), isNotNull(statusCards.nextEvalAt)))
|
||||
.orderBy(asc(statusCards.nextEvalAt)).limit(1);
|
||||
const deadline = next?.at?.getTime() ?? null;
|
||||
return retryAt === null ? deadline : Math.min(deadline ?? Infinity, retryAt);
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export function registerStatusCardEvaluationWork(
|
||||
coordinator: ReturnType<typeof createDeliveryWorkCoordinator>,
|
||||
service: ReturnType<typeof statusCardEvaluationService>,
|
||||
canRun: () => boolean,
|
||||
) {
|
||||
return registerDeadlineWork(coordinator, DELIVERY_QUEUES.statusCardEvaluation, {
|
||||
run: service.run, next: service.next, onSettled: service.changed, canRun, retryMs: 30_000,
|
||||
// Batch source hints for at most one second after reconciliation. Do not
|
||||
// use minimumSpacingMs: known policy/retry deadlines must preempt batching.
|
||||
// This window alone arms no timer when the feature has no pending work.
|
||||
notificationCoalesceMs: 1_000,
|
||||
});
|
||||
}
|
||||
@@ -1,3 +1,4 @@
|
||||
import { notifyStatusCardEvaluation } from "./status-card-notifications.js";
|
||||
import { and, eq, isNull } from "drizzle-orm";
|
||||
import type { Db } from "@paperclipai/db";
|
||||
import { statusCards, statusCardUpdates } from "@paperclipai/db";
|
||||
@@ -36,6 +37,7 @@ export async function finalizeStatusCardsForStalledGeneration(
|
||||
) {
|
||||
if (!STALLED_GENERATION_STATUSES.has(issue.status)) return [];
|
||||
|
||||
await notifyStatusCardEvaluation(dbOrTx, issue.companyId);
|
||||
const now = new Date();
|
||||
const failureReason = failureReasonForIssue(issue);
|
||||
const cards = await dbOrTx
|
||||
@@ -45,6 +47,7 @@ export async function finalizeStatusCardsForStalledGeneration(
|
||||
failureReason,
|
||||
generatingIssueId: null,
|
||||
nextEvalAt: null,
|
||||
nextEvalReason: null,
|
||||
updatedAt: now,
|
||||
})
|
||||
.where(
|
||||
|
||||
Loaded 100 of 201 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user