Merge commit 'd1af4fd95405b1c25e3008e935b0c0b6e5b0369a' into codex/sleep-09-aggregate

* commit 'd1af4fd95405b1c25e3008e935b0c0b6e5b0369a':
  Join receipt acceptance passes before advancing saved deadlines
  fix(server): retain live evidence beyond blocker graph admission
  Retain moved receipt deadlines and wake released execution gates
  Drive saved execution receipts from committed work and deadlines
This commit is contained in:
Dotta committed 2026-10-30 19:58:12 -05:00
commit f131ecd7e0
31 files changed
+822 -58

No files matched your search

+2
View File
@@ -1,4 +1,5 @@
import { notifyStatusCardIssueChange } from "./status-card-notifications.js";
import { notifyDeferredExecutionEligibility } from "./deferred-execution-notifications.js";
import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js";
import { notifyNativeStatusReassessment } from "./native-status-reassessment-notifications.js";
import { notifyUnsafeWorkspaceExportRecovery } from "./unsafe-workspace-export-notifications.js";
@@ -744,6 +745,7 @@ export function accessService(db: Db) {
sql`${issues.status} not in ('done', 'cancelled')`,
);
await notifyNativeStatusReassessment(tx, { companyId });
await notifyDeferredExecutionEligibility(tx, { companyId });
await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors);
await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers);
await notifyStatusCardIssueChange(tx, companyId);
@@ -1,3 +1,4 @@
import { notifyDeferredExecutionEligibility } from "./deferred-execution-notifications.js";
import { notifyRunDispatch } from "./run-dispatch-notifications.js";
import { notifyAttentionRetentionWork } from "./retention-work-notifications.js";
import { notifySkillVersionPinChange } from "./skill-version-retention-notifications.js";
@@ -87,6 +88,7 @@ export async function prepareAgentConfiguration(db: Db, existing: AgentConfigura
);
}
await notifyRunDispatch(db, { companyId: existing.companyId });
await notifyDeferredExecutionEligibility(db, { companyId: existing.companyId });
if (normalizedPatch.runtimeConfig !== undefined || normalizedPatch.reportsTo !== undefined) {
await notifyDeliveryWork(db, DELIVERY_QUEUES.agentTimers);
}
+2
View File
@@ -1,3 +1,4 @@
import { notifyDeferredExecutionEligibility } from "./deferred-execution-notifications.js";
import { notifyRunDispatch } from "./run-dispatch-notifications.js";
import { notifyAttentionRetentionWork } from "./retention-work-notifications.js";
import { notifyNativeReviewResponse } from "./native-review-response-notifications.js";
@@ -43,6 +44,7 @@ export function createAgentLifecycleEffects(_hooks: BudgetServiceHooks = {}): li
await notifyAttentionRetentionWork(tx, { companyId });
if (before !== after) {
await notifyRunDispatch(tx, { companyId });
await notifyDeferredExecutionEligibility(tx, { companyId });
await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers);
}
if (!isAgentStatusInvokable(before) && isAgentStatusInvokable(after)) {
+5
View File
@@ -1,3 +1,4 @@
import { notifyDeferredExecutionEligibility } from "./deferred-execution-notifications.js";
import { notifyRunDispatch } from "./run-dispatch-notifications.js";
import { notifyStatusCardIssueChange } from "./status-card-notifications.js";
import { notifyExternalObjectRefresh } from "./external-object-refresh-notifications.js";
@@ -291,6 +292,7 @@ export function budgetServiceInTransaction(db: Db, publications: ActivityPublica
if (policy.scopeType === "project") {
await notifyRunDispatch(db, { companyId: policy.companyId });
await notifyDeferredExecutionEligibility(db, { companyId: policy.companyId });
await db
.update(projects)
.set({
@@ -305,6 +307,7 @@ export function budgetServiceInTransaction(db: Db, publications: ActivityPublica
const scope = await resolveScopeRecord(db, "company", policy.scopeId);
if (scope.paused) return;
await notifyRunDispatch(db, { companyId: policy.companyId });
await notifyDeferredExecutionEligibility(db, { companyId: policy.companyId });
await notifyCompanyRetentionWork(db, policy.companyId);
await notifyStatusCardIssueChange(db, policy.companyId);
await notifyExternalObjectRefresh(db, policy.companyId);
@@ -354,6 +357,7 @@ export function budgetServiceInTransaction(db: Db, publications: ActivityPublica
if (policy.scopeType === "project") {
await notifyRunDispatch(db, { companyId: policy.companyId });
await notifyDeferredExecutionEligibility(db, { companyId: policy.companyId });
await db
.update(projects)
.set({
@@ -369,6 +373,7 @@ export function budgetServiceInTransaction(db: Db, publications: ActivityPublica
const scope = await resolveScopeRecord(db, "company", policy.scopeId);
if (scope.pauseReason !== "budget") return;
await notifyRunDispatch(db, { companyId: policy.companyId });
await notifyDeferredExecutionEligibility(db, { companyId: policy.companyId });
await notifyCompanyRetentionWork(db, policy.companyId);
await notifyStatusCardIssueChange(db, policy.companyId);
await notifyExternalObjectRefresh(db, policy.companyId);
@@ -1,3 +1,4 @@
import { notifyDeferredExecutionWork, notifyTaskRetrySettlement } from "../deferred-execution-notifications.js";
import { notifyNativeResponsePresentation } from "../native-response-presentation-notifications.js";
import { notifyChatInboundWakeNotices } from "../chat-inbound-wake-notice-notifications.js";
import { createFailedRetryNotices } from "./failed-retry-notices.js";
@@ -1424,6 +1425,7 @@ export function createChatFailedRunRetry(db: Db, options: ChatFailedRunRetryOpti
// Source authorization above checked every external receipt and its owner.
// Reset only this certified unstarted batch under the task's admission lock.
const skippedIds = source.taskRetryInput?.skippedActionIds ?? [];
await notifyDeferredExecutionWork(tx, { companyId: input.companyId });
await tx.update(agentWakeupRequests).set({ status: "deferred_issue_execution", runId: null, finishedAt: null,
payload: sql`${agentWakeupRequests.payload} - 'coalescedIntoWakeupRequestId'`, updatedAt: new Date(),
}).where(and(eq(agentWakeupRequests.companyId, input.companyId), eq(agentWakeupRequests.agentId, input.agentId),
@@ -1784,6 +1786,9 @@ export function createChatFailedRunRetry(db: Db, options: ChatFailedRunRetryOpti
const attemptCount = Number(candidate.result?.attemptCount ?? 0) + 1;
const settle = async (status: string, result: Record<string, unknown>) => {
const [current] = await db.transaction(async (tx) => {
if (typeof claimed.payload.taskRetryIntentId === "string") await notifyTaskRetrySettlement(tx, {
companyId: claimed.companyId, requestId: claimed.payload.taskRetryIntentId,
});
await notifyChatActionWork(tx, "failed_run_retry");
if (status === "processed") await notifyNativeResponsePresentation(tx, {
companyId: claimed.companyId, issueId: String(claimed.payload.issueId),
@@ -1,3 +1,4 @@
import { notifyDeferredExecutionWork } from "../deferred-execution-notifications.js";
import { z } from "zod";
import { and, asc, eq, gt, inArray, isNull, or, sql } from "drizzle-orm";
import { agentWakeupRequests, chatActions, chatConversations, chatEndpoints, heartbeatRuns, issues, type Db } from "@paperclipai/db";
@@ -91,6 +92,7 @@ export async function releaseUnadmittedTaskRetryInput(db: Db, input: { companyId
)).for("update");
if (receipt?.runId) return;
await notifyChatInboundWakeNotices(tx, { companyId: input.companyId, issueId: issue.id });
await notifyDeferredExecutionWork(tx, { companyId: input.companyId });
await tx.update(agentWakeupRequests).set({ status: "deferred_issue_execution", finishedAt: null,
payload: sql`${agentWakeupRequests.payload} - 'coalescedIntoWakeupRequestId'`, updatedAt: new Date(),
}).where(and(eq(agentWakeupRequests.companyId, input.companyId), eq(agentWakeupRequests.status, "coalesced"),
+3
View File
@@ -1,3 +1,4 @@
import { notifyDeferredExecutionEligibility } from "./deferred-execution-notifications.js";
import { notifyRunDispatch } from "./run-dispatch-notifications.js";
import { notifyStatusCardIssueChange } from "./status-card-notifications.js";
import { notifyGitHubCompanyChange } from "./github-connection-notifications.js";
@@ -338,6 +339,7 @@ export function companyService(db: Db, _budgetHooks: BudgetServiceHooks = {}) {
if (companyPatch.status !== undefined && companyPatch.status !== existing.status) {
await notifyRunDispatch(tx, { companyId: id });
await notifyGitHubCompanyChange(tx, id);
await notifyDeferredExecutionEligibility(tx, { companyId: id });
await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers);
await notifyDeliveryWork(tx, DELIVERY_QUEUES.skillSourceSync);
await notifyExternalObjectRefresh(tx, id);
@@ -498,6 +500,7 @@ export function companyService(db: Db, _budgetHooks: BudgetServiceHooks = {}) {
if (!wasAlreadyArchived) {
await notifyRunDispatch(tx, { companyId: id });
await notifyGitHubCompanyChange(tx, id);
await notifyDeferredExecutionEligibility(tx, { companyId: id });
await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers);
await notifyDeliveryWork(tx, DELIVERY_QUEUES.skillSourceSync);
await notifyExternalObjectRefresh(tx, id);
@@ -0,0 +1,34 @@
import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js";
export type DeferredExecutionScope = { companyId: string; requestId?: string };
export type DeferredReceiptRecoveryOptions = {
companyId?: string;
requestId?: string;
onError?: (error: unknown) => void;
};
function hint(scope: DeferredExecutionScope) {
return JSON.stringify({ companyId: scope.companyId, requestId: scope.requestId });
}
export async function notifyDeferredExecutionWork(tx: object, scope: DeferredExecutionScope) {
await notifyDeliveryWork(tx, DELIVERY_QUEUES.deferredExecutionReceipts, hint(scope));
}
export async function notifyNativeStatusWakeWork(tx: object, scope: DeferredExecutionScope) {
await notifyDeliveryWork(tx, DELIVERY_QUEUES.nativeStatusWakeIntents, hint(scope));
}
export async function notifyTaskRetrySettlement(tx: object, scope: DeferredExecutionScope) {
await notifyDeliveryWork(tx, DELIVERY_QUEUES.taskRetryReceiptSettlement, hint(scope));
}
export async function notifyDeferredExecutionEligibility(tx: object, scope?: { companyId: string }) {
if (scope) {
await notifyDeferredExecutionWork(tx, scope);
await notifyNativeStatusWakeWork(tx, scope);
} else {
await notifyDeliveryWork(tx, DELIVERY_QUEUES.deferredExecutionReceipts, "*");
await notifyDeliveryWork(tx, DELIVERY_QUEUES.nativeStatusWakeIntents, "*");
}
}
@@ -0,0 +1,24 @@
import { and, eq, sql } from "drizzle-orm";
import { agentWakeupRequests, type Db } from "@paperclipai/db";
import { notifyDeferredExecutionWork } from "./deferred-execution-notifications.js";
export async function writeDeferredExecutionWait(db: Db, scope: { companyId: string; requestId: string },
condition: Record<string, unknown>, options: { replace?: boolean } = {}) {
return db.transaction(async tx => {
const [current] = await tx.select({ payload: agentWakeupRequests.payload }).from(agentWakeupRequests)
.where(and(eq(agentWakeupRequests.companyId, scope.companyId), eq(agentWakeupRequests.id, scope.requestId),
eq(agentWakeupRequests.status, "deferred_issue_execution"))).for("update");
if (!current) return;
const previous = current.payload?.executionWait;
const previousReason = previous && typeof previous === "object" && !Array.isArray(previous)
? (previous as Record<string, unknown>).reason : undefined;
if (!previous || previousReason !== condition.reason) await notifyDeferredExecutionWork(tx, scope);
await tx.update(agentWakeupRequests).set({
payload: sql`jsonb_set(coalesce(${agentWakeupRequests.payload}, '{}'::jsonb), '{executionWait}',
${options.replace ? sql`'{}'::jsonb` : sql`coalesce(${agentWakeupRequests.payload}->'executionWait', '{}'::jsonb)`}
|| ${JSON.stringify(condition)}::jsonb)`,
updatedAt: new Date(),
}).where(and(eq(agentWakeupRequests.companyId, scope.companyId), eq(agentWakeupRequests.id, scope.requestId),
eq(agentWakeupRequests.status, "deferred_issue_execution")));
});
}
@@ -2,6 +2,9 @@ import { signalDatabaseWork, subscribeDatabaseWork } from "@paperclipai/db";
/** Topics for existing durable queues; register intent before their writes. */
export const DELIVERY_QUEUES = {
deferredExecutionReceipts: "deferred-execution-receipts",
taskRetryReceiptSettlement: "task-retry-receipt-settlement",
nativeStatusWakeIntents: "native-status-wake-intents",
orphanBlockers: "orphan-blockers",
activeRecoveryActions: "active-recovery-actions",
strandedAssignments: "stranded-assignments",
@@ -1,4 +1,5 @@
import { notifyAttentionRetentionWork } from "./retention-work-notifications.js";
import { notifyDeferredExecutionEligibility } from "./deferred-execution-notifications.js";
import { notifyRunDispatch } from "./run-dispatch-notifications.js";
import { notifyStatusCardIssueChange } from "./status-card-notifications.js";
import { notifyRoutineExecutionStatus } from "./routine-execution-status-notifications.js";
@@ -12,8 +13,9 @@ export async function notifyExecutionRunWork(transaction: object, scope?: Execut
await notifyAttentionRetentionWork(transaction, scope);
await notifyRunDispatch(transaction, scope);
await notifyStatusCardIssueChange(transaction, scope.companyId);
await notifyDeferredExecutionEligibility(transaction, scope);
await notifyRoutineExecutionStatus(transaction, scope.issueId, scope.companyId);
}
if (scope) await notifyRoutineExecutionStatus(transaction, scope.issueId, scope.companyId);
for (const queue of [DELIVERY_QUEUES.executionStatus, DELIVERY_QUEUES.executionFinalization,
DELIVERY_QUEUES.executionReplacement, DELIVERY_QUEUES.executionDisposition, DELIVERY_QUEUES.issueMonitors]) {
await notifyDeliveryWork(transaction, queue);
@@ -31,6 +33,7 @@ export function writeExecutionControl<T>(
if (scope) await notifyStatusCardIssueChange(transaction, scope.companyId);
if (scope) await notifyAttentionRetentionWork(transaction, scope);
if (scope) await notifyRoutineExecutionStatus(transaction, scope.issueId, scope.companyId);
if (scope) await notifyDeferredExecutionEligibility(transaction, scope);
await notifyDeliveryWork(transaction, DELIVERY_QUEUES.issueMonitors);
for (const topic of typeof queue === "string" ? [queue] : queue) await notifyDeliveryWork(transaction, topic);
return write(transaction as unknown as Db);
+9
View File
@@ -1,3 +1,5 @@
import { registerDeferredReceiptWork } from "./heartbeat/deferred-receipt-work.js";
import { reconcileTaskChatRetryReceipts } from "./task-retry.js";
import { registerIssueRecoveryWork } from "./recovery/work.js";
import { registerRunDispatchWork } from "./heartbeat/dispatch-work.js";
import { externalConversationSettlementWork } from "./external-conversation-settlement-work.js";
@@ -4243,6 +4245,13 @@ export function heartbeatService(
computeTaskDrain,
applyTaskDrain,
registerDeferredReceiptWork: (coordinator: Parameters<typeof registerDeferredReceiptWork>[1], canRun: () => boolean) =>
registerDeferredReceiptWork(db, coordinator, {
resume: scope => heartbeatQueue.resumeDeferredExecutionReceipts(scope),
settleTaskChat: scope => reconcileTaskChatRetryReceipts(db, { ...scope, settleOnly: true }),
dispatchNative: dispatchPendingNativeStatusWakeups,
schedulingSuppressed: async () => (await getSchedulingSuppression(true)).suppressed,
}, () => canRun() && readTaskDrain(new Date()) === null),
registerIssueStateRecoveryWork: (coordinator: Parameters<typeof registerIssueRecoveryWork>[1], canRun: () => boolean) =>
registerIssueRecoveryWork(db, coordinator, {
stranded: scope => reconcileStrandedAssignedIssues({ ...scope, skipExternalConversationSettlement: true, skipRelatedRecovery: true }),
@@ -0,0 +1,118 @@
import { and, asc, eq, gt, inArray, isNull, or, sql } from "drizzle-orm";
import { agentWakeupRequests, type Db } from "@paperclipai/db";
import type { createDeliveryWorkCoordinator } from "../delivery-work-coordinator.js";
import { DELIVERY_QUEUES } from "../delivery-work-notifications.js";
import type { DeferredExecutionScope, DeferredReceiptRecoveryOptions } from "../deferred-execution-notifications.js";
import { registerScopedDeadlineWork } from "../scoped-deadline-work.js";
type Entry = { id: string; companyId: string; dueAt: number };
type NativeIntentOptions = DeferredExecutionScope & {
onDeferredDeadline(at: number): void;
onError(error: unknown): void;
};
export function registerDeferredReceiptWork(
db: Db,
coordinator: ReturnType<typeof createDeliveryWorkCoordinator>,
owner: {
resume(scope: DeferredReceiptRecoveryOptions): Promise<unknown>;
settleTaskChat(scope: DeferredExecutionScope): Promise<unknown>;
dispatchNative(scope: NativeIntentOptions): Promise<unknown>;
schedulingSuppressed(): Promise<boolean>;
},
canRun: () => boolean,
) {
function register(kind: "deferred" | "task_chat" | "native") {
const cursors = new Map<string, string>();
let enabled = false;
const queue = kind === "deferred" ? DELIVERY_QUEUES.deferredExecutionReceipts
: kind === "task_chat" ? DELIVERY_QUEUES.taskRetryReceiptSettlement : DELIVERY_QUEUES.nativeStatusWakeIntents;
return registerScopedDeadlineWork<Entry, DeferredExecutionScope>(coordinator, {
queue,
reconstructionIntervalMs: 30_000,
beforePass: async () => { enabled = !(await owner.schedulingSuppressed()); },
decodeScope: hint => hint === "*" ? undefined : JSON.parse(hint),
matches: (entry, scope) => !scope || entry.companyId === scope.companyId && (!scope.requestId || entry.id === scope.requestId),
load: async scope => {
const key = scope ? JSON.stringify({ companyId: scope.companyId, requestId: scope.requestId }) : "*";
if (!enabled) { cursors.delete(key); return []; }
const cursor = cursors.get(key);
const rows = await db.select({ id: agentWakeupRequests.id, companyId: agentWakeupRequests.companyId,
status: agentWakeupRequests.status, updatedAt: agentWakeupRequests.updatedAt,
claimedAt: agentWakeupRequests.claimedAt, error: agentWakeupRequests.error,
delayed: sql<boolean>`${agentWakeupRequests.payload}->'queuedCommentInterrupt' is not null
or ${agentWakeupRequests.payload}->'executionWait' is not null`,
interruptedLegacy: sql<boolean>`exists (select 1 from heartbeat_runs r
where r.company_id = ${agentWakeupRequests.companyId} and r.agent_id = ${agentWakeupRequests.agentId}
and r.runtime_mode = 'legacy' and r.status = 'cancelled'
and r.result_json->>'queuedCommentInterruptQueueId' = ${agentWakeupRequests.id}::text)`,
}).from(agentWakeupRequests).where(and(
scope ? eq(agentWakeupRequests.companyId, scope.companyId) : undefined,
scope?.requestId ? eq(agentWakeupRequests.id, scope.requestId) : undefined,
cursor ? gt(agentWakeupRequests.id, cursor) : undefined,
kind === "native" ? and(
eq(agentWakeupRequests.requestedByActorType, "system"),
eq(agentWakeupRequests.requestedByActorId, "native-status-committer"),
inArray(agentWakeupRequests.status, ["queued", "claimed"]), isNull(agentWakeupRequests.runId),
) : and(eq(agentWakeupRequests.status, "deferred_issue_execution"),
kind === "task_chat" ? and(eq(agentWakeupRequests.reason, "task_retry"),
sql`${agentWakeupRequests.payload}#>>'{taskRetry,chatActionId}' is not null`) : or(
eq(agentWakeupRequests.reason, "task_retry"),
sql`${agentWakeupRequests.payload}->'queuedCommentInterrupt' is not null`,
sql`${agentWakeupRequests.payload}->'executionWait' is not null`,
sql`jsonb_typeof(${agentWakeupRequests.payload}#>'{_paperclipWakeContext,wakeCommentIds}') = 'array'
and ${agentWakeupRequests.payload}#>'{_paperclipWakeContext,wakeCommentIds}' <> '[]'::jsonb`,
sql`${agentWakeupRequests.payload}->>'mutation' = 'interaction'`,
sql`exists (select 1 from heartbeat_runs r where r.company_id = ${agentWakeupRequests.companyId}
and r.runtime_mode = 'legacy' and r.status = 'cancelled'
and r.result_json->>'queuedCommentInterruptQueueId' = ${agentWakeupRequests.id}::text)`,
)),
)).orderBy(asc(agentWakeupRequests.id)).limit(100);
const complete = rows.length < 100;
if (complete) cursors.delete(key); else cursors.set(key, rows.at(-1)!.id);
return { rows: rows.map(row => ({ id: row.id, companyId: row.companyId,
dueAt: kind === "native" && row.status === "claimed" && row.claimedAt ? row.claimedAt.getTime() + 60_000
: kind === "native" && row.error ? row.updatedAt.getTime() + 30_000
: kind === "deferred" && row.delayed && !row.interruptedLegacy ? row.updatedAt.getTime() + 30_000
: Date.now(),
})), complete };
},
check: async entry => {
if (!enabled) return null;
if (kind !== "task_chat") {
const [current] = await db.select({ status: agentWakeupRequests.status, payload: agentWakeupRequests.payload,
updatedAt: agentWakeupRequests.updatedAt, claimedAt: agentWakeupRequests.claimedAt, error: agentWakeupRequests.error,
interruptedLegacy: sql<boolean>`exists (select 1 from heartbeat_runs r
where r.company_id = ${agentWakeupRequests.companyId} and r.agent_id = ${agentWakeupRequests.agentId}
and r.runtime_mode = 'legacy' and r.status = 'cancelled'
and r.result_json->>'queuedCommentInterruptQueueId' = ${agentWakeupRequests.id}::text)`,
}).from(agentWakeupRequests).where(and(eq(agentWakeupRequests.companyId, entry.companyId),
eq(agentWakeupRequests.id, entry.id)));
if (!current) return null;
const due = kind === "native" && current.status === "claimed" && current.claimedAt
? current.claimedAt.getTime() + 60_000
: kind === "native" && current.status === "queued" && current.error ? current.updatedAt.getTime() + 30_000
: kind === "deferred" && current.status === "deferred_issue_execution" && !current.interruptedLegacy
&& (current.payload?.queuedCommentInterrupt || current.payload?.executionWait)
? current.updatedAt.getTime() + 30_000 : null;
if (due !== null && due > Date.now()) return { ...entry, dueAt: due };
}
const errors: unknown[] = [];
let next: number | null = null;
const scope = { companyId: entry.companyId, requestId: entry.id };
if (kind === "native") await owner.dispatchNative({ ...scope,
onDeferredDeadline: at => { next = next === null ? at : Math.min(next, at); },
onError: error => { errors.push(error); },
});
else if (kind === "task_chat") await owner.settleTaskChat(scope);
else await owner.resume({ ...scope, onError: error => { errors.push(error); } });
if (errors.length) throw new AggregateError(errors, `Deferred receipt ${entry.id} could not settle`);
return next === null ? null : { ...entry, dueAt: next };
},
}, canRun);
}
const workers = [register("deferred"), register("task_chat"), register("native")];
return { ready: Promise.all(workers.map(worker => worker.ready)),
wake() { for (const worker of workers) worker.invalidate(); },
async stop() { await Promise.all(workers.map(worker => worker.stop())); } };
}
+40 -21
View File
@@ -1,3 +1,6 @@
import { writeDeferredExecutionWait } from "../deferred-execution-wait.js";
import { notifyDeferredExecutionWork } from "../deferred-execution-notifications.js";
import type { DeferredReceiptRecoveryOptions } from "../deferred-execution-notifications.js";
import { notifyRunDispatch } from "../run-dispatch-notifications.js";
import { notifyStatusCardIssueChange } from "../status-card-notifications.js";
import { notifyRoutineExecutionStatus } from "../routine-execution-status-notifications.js";
@@ -224,7 +227,7 @@ export interface HeartbeatQueueDependencies extends Pick<HeartbeatRetryDependenc
publishRunLifecyclePluginEvent: (run: HeartbeatRun) => void;
cancelRunInternal: (runId: string, reason?: string, options?: { errorCode?: string }) => Promise<unknown>;
cancelActiveForAgentInternal: (agentId: string, reason: string) => Promise<unknown>;
resumeExecutionWaitComments: () => Promise<unknown>;
resumeExecutionWaitComments: (scope?: DeferredReceiptRecoveryOptions) => Promise<unknown>;
resumeQueuedCommentInterrupt: (companyId: string, queueId: string) => Promise<unknown>;
resumeSavedLegacyComments: (companyId: string, queueId: string) => Promise<unknown>;
formatIssueIdentifierLink: (identifier: string | null, fallback: string) => string;
@@ -1854,13 +1857,15 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
return cancelled;
}
async function resumeQueuedRuns() {
async function resumeDeferredExecutionReceipts(scope: DeferredReceiptRecoveryOptions = {}) {
if ((await getSchedulingSuppression()).suppressed) return;
await resumeExecutionWaitComments();
await resumeExecutionWaitComments(scope);
const cutoff = await getWorktreeExecutionCutoff();
const pendingInterrupts = await db.select({ id: agentWakeupRequests.id, companyId: agentWakeupRequests.companyId })
.from(agentWakeupRequests).innerJoin(companies, eq(companies.id, agentWakeupRequests.companyId))
.where(and(eq(agentWakeupRequests.status, "deferred_issue_execution"),
.where(and(scope.companyId ? eq(agentWakeupRequests.companyId, scope.companyId) : undefined,
scope.requestId ? eq(agentWakeupRequests.id, scope.requestId) : undefined,
eq(agentWakeupRequests.status, "deferred_issue_execution"),
eq(companies.status, "active"),
sql`${agentWakeupRequests.payload}->'queuedCommentInterrupt' is not null`,
lte(agentWakeupRequests.updatedAt, new Date(Date.now() - 30_000)),
@@ -1871,6 +1876,7 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
eq(agentWakeupRequests.id, wake.id), eq(agentWakeupRequests.status, "deferred_issue_execution"),
));
await resumeQueuedCommentInterrupt(wake.companyId, wake.id).catch(err => {
scope.onError?.(err);
logger.warn({ err, queueId: wake.id }, "failed to resume interrupted comment queue");
});
}
@@ -1884,7 +1890,9 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
or(eq(issues.assigneeAgentId, agentWakeupRequests.agentId),
sql`${agentWakeupRequests.payload} #>> '{_paperclipWakeContext,wakeReason}' = 'issue_comment_mentioned'`)))
.innerJoin(companies, and(eq(companies.id, issues.companyId), eq(companies.status, "active")))
.where(and(eq(agentWakeupRequests.status, "deferred_issue_execution"),
.where(and(scope.companyId ? eq(agentWakeupRequests.companyId, scope.companyId) : undefined,
scope.requestId ? eq(agentWakeupRequests.id, scope.requestId) : undefined,
eq(agentWakeupRequests.status, "deferred_issue_execution"),
isNull(issues.executionRunId),
or(and(
sql`jsonb_typeof(${agentWakeupRequests.payload} #> '{_paperclipWakeContext,wakeCommentIds}') = 'array'`,
@@ -1923,7 +1931,8 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
if (latest.runtimeMode !== "legacy" && latest.runtimeMode !== "native") continue;
if (wake.idempotencyKey?.startsWith("chat-inbound:")) {
await resumeDeferredChatWake(db, { companyId: wake.companyId, agentId: wake.agentId,
issueId: String(wake.payload?.issueId), requestId: wake.id }).catch(() => {
issueId: String(wake.payload?.issueId), requestId: wake.id }).catch(err => {
scope.onError?.(err);
logger.warn({ requestId: wake.id }, "saved chat source could not resume after recovery");
});
continue;
@@ -1935,6 +1944,7 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
latest.errorCode === "execution_reconciliation_required";
if ((latest.status !== "cancelled" || cancelledAdmission) && await getExecutionBlocker(db, wake.companyId, String(wake.payload?.issueId))) {
await resumeSavedLegacyComments(wake.companyId, wake.id).catch(err => {
scope.onError?.(err);
logger.warn({ err, queueId: wake.id }, "failed to deliver saved legacy comment after recovery stopped");
});
}
@@ -1942,6 +1952,7 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
await releaseIssueExecutionAndPromote(latest, { suppressImmediateRecovery: true, resumeStrandedQueue: true }).then(() => {
if (latest.runtimeMode === "native") releasedNativeSources.add(releaseSource);
}).catch(err => {
scope.onError?.(err);
logger.warn({ err, queueId: wake.id }, "failed to promote stranded comments");
});
}
@@ -1959,6 +1970,8 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
))
.innerJoin(companies, eq(companies.id, heartbeatRuns.companyId))
.where(and(
scope.companyId ? eq(agentWakeupRequests.companyId, scope.companyId) : undefined,
scope.requestId ? eq(agentWakeupRequests.id, scope.requestId) : undefined,
eq(agentWakeupRequests.status, "deferred_issue_execution"),
eq(heartbeatRuns.status, "cancelled"),
eq(heartbeatRuns.runtimeMode, "legacy"),
@@ -1967,10 +1980,15 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
));
for (const run of interruptedQueues) {
await releaseIssueExecutionAndPromote(run, { suppressImmediateRecovery: true }).catch((err) => {
scope.onError?.(err);
logger.error({ err, runId: run.id }, "failed to retry interrupted comment queue");
});
}
}
async function resumeQueuedRuns() {
await resumeDeferredExecutionReceipts();
await resumeQueuedRunsAfterDrain();
}
@@ -2509,17 +2527,11 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
// its one durable receipt ID on a skipped wake.
if (caseRequest) return { created: false };
if (executionWaitRequestId) {
await db.update(agentWakeupRequests).set({
payload: sql`jsonb_set(coalesce(${agentWakeupRequests.payload}, '{}'::jsonb), '{executionWait}',
coalesce(${agentWakeupRequests.payload}->'executionWait', '{}'::jsonb) || ${JSON.stringify({
reason: skipReason, message: patch.error ?? (skipReason === "issue_tree_hold_active"
? "This task is paused. Resume it to send your saved message."
: "Waiting for task execution to be enabled. Your message is saved."),
})}::jsonb)`,
updatedAt: new Date(),
}).where(and(eq(agentWakeupRequests.id, executionWaitRequestId),
eq(agentWakeupRequests.companyId, agent.companyId), eq(agentWakeupRequests.agentId, agentId),
eq(agentWakeupRequests.status, "deferred_issue_execution")));
await writeDeferredExecutionWait(db, { companyId: agent.companyId, requestId: executionWaitRequestId }, {
reason: skipReason, message: patch.error ?? (skipReason === "issue_tree_hold_active"
? "This task is paused. Resume it to send your saved message."
: "Waiting for task execution to be enabled. Your message is saved."),
});
return { created: false };
}
const request = {
@@ -3407,15 +3419,14 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
opts.issueStateGuard?.onExecutionDeferred?.(executionBlocker.cause);
const condition = { recoveryActionId: executionBlocker.recoveryActionId, ...continuationWait };
if (executionWaitRequestId) {
await tx.update(agentWakeupRequests).set({
payload: sql`jsonb_set(coalesce(${agentWakeupRequests.payload}, '{}'::jsonb), '{executionWait}', ${JSON.stringify(condition)}::jsonb)`,
updatedAt: new Date(),
}).where(eq(agentWakeupRequests.id, executionWaitRequestId));
await writeDeferredExecutionWait(tx as unknown as Db, { companyId: agent.companyId,
requestId: executionWaitRequestId }, condition, { replace: true });
return { kind: "deferred" as const, admitted: true };
}
if (durableRequest || wakeCommentId ||
hasInteractionContinuationWakeContext(enrichedContextSnapshot) ||
readNonEmptyString(enrichedContextSnapshot.nativeStatusWakeIntentId)) {
await notifyDeferredExecutionWork(tx, { companyId: agent.companyId });
await tx.insert(agentWakeupRequests).values({
...durableReceiptFields,
companyId: agent.companyId, agentId, source, triggerDetail, reason,
@@ -4035,6 +4046,7 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
}));
}
if (followUp) {
if (!followUp.consumed) await notifyDeferredExecutionWork(tx, { companyId: agent.companyId });
const assignmentId = followUp.assignmentId;
const pending = followUp.consumed ? undefined : pendingFollowUps.find(wake =>
wake.payload?.[followUpPayloadKey] === assignmentId &&
@@ -4773,6 +4785,9 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
companyId?: string;
limit?: number;
staleClaimMs?: number;
requestId?: string;
onDeferredDeadline?: (at: number) => void;
onError?: (error: unknown) => void;
} = {},
) {
const now = new Date();
@@ -4785,6 +4800,7 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
input.companyId
? eq(agentWakeupRequests.companyId, input.companyId)
: undefined,
input.requestId ? eq(agentWakeupRequests.id, input.requestId) : undefined,
eq(agentWakeupRequests.requestedByActorType, "system"),
eq(agentWakeupRequests.requestedByActorId, "native-status-committer"),
inArray(agentWakeupRequests.status, ["queued", "claimed"]),
@@ -4861,6 +4877,7 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
candidate.claimedAt &&
now.getTime() - candidate.claimedAt.getTime() < staleClaimMs
) {
input.onDeferredDeadline?.(candidate.claimedAt.getTime() + staleClaimMs);
deferred += 1;
continue;
}
@@ -5035,6 +5052,7 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
if (wakeRun) dispatched += 1;
else deferred += 1;
} catch (error) {
input.onError?.(error);
await db
.update(agentWakeupRequests)
.set({
@@ -5078,6 +5096,7 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
withChatControlRecoveryGate,
trackWakeup,
resumeQueuedRuns,
resumeDeferredExecutionReceipts,
resumeQueuedRunsAfterDrain,
parseHeartbeatPolicy,
nextTimerHeartbeatAt,
+2 -2
View File
@@ -1979,7 +1979,7 @@ export function createHeartbeatRecovery(db: Db, dependencies: HeartbeatRecoveryD
let settledProcessesDiscovered = false;
async function reapOrphanedRuns(opts?: { staleThresholdMs?: number; skipScheduledRetryCancellation?: boolean; skipCapturedInstructionCommit?: boolean; skipInstructionDirectoryCleanup?: boolean; skipStoppedInstructionCollection?: boolean; skipCancelledWorkspacePreservation?: boolean; skipObsoletePolicyReviews?: boolean; skipAutomaticCompletionReviews?: boolean; skipUnsafeWorkspaceExports?: boolean; skipNativePendingEffects?: boolean; skipNativeResponsePresentation?: boolean; skipNativeReviewResponse?: boolean; skipNativeStatusReassessment?: boolean; skipRetainedNativeSessionCleanup?: boolean; skipNativeWorkspaceFinalization?: boolean; skipNativeResultCommit?: boolean; skipNativeRunProjection?: boolean; skipNativeRecoveryNotices?: boolean; skipExternalConversationSettlement?: boolean; skipExternalAgentSettlement?: boolean; skipNativeEnvironmentSettlement?: boolean }) {
async function reapOrphanedRuns(opts?: { staleThresholdMs?: number; skipNativeStatusWakeIntents?: boolean; skipScheduledRetryCancellation?: boolean; skipCapturedInstructionCommit?: boolean; skipInstructionDirectoryCleanup?: boolean; skipStoppedInstructionCollection?: boolean; skipCancelledWorkspacePreservation?: boolean; skipObsoletePolicyReviews?: boolean; skipAutomaticCompletionReviews?: boolean; skipUnsafeWorkspaceExports?: boolean; skipNativePendingEffects?: boolean; skipNativeResponsePresentation?: boolean; skipNativeReviewResponse?: boolean; skipNativeStatusReassessment?: boolean; skipRetainedNativeSessionCleanup?: boolean; skipNativeWorkspaceFinalization?: boolean; skipNativeResultCommit?: boolean; skipNativeRunProjection?: boolean; skipNativeRecoveryNotices?: boolean; skipExternalConversationSettlement?: boolean; skipExternalAgentSettlement?: boolean; skipNativeEnvironmentSettlement?: boolean }) {
if (!opts?.skipCancelledWorkspacePreservation) await preserveCancelledNativeWorkspaces(db);
const staleThresholdMs = opts?.staleThresholdMs ?? 0;
const now = new Date();
@@ -2023,7 +2023,7 @@ export function createHeartbeatRecovery(db: Db, dependencies: HeartbeatRecoveryD
});
if (!opts?.skipRetainedNativeSessionCleanup) scheduleRetainedNativeSessionCleanup();
await reconcileRetainedWorkspaces(db, environmentRuntime, resumeRemoteStopComments);
await dispatchPendingNativeStatusWakeups().catch((error) => {
if (!opts?.skipNativeStatusWakeIntents) await dispatchPendingNativeStatusWakeups().catch((error) => {
logger.warn(
{ err: error },
"failed to dispatch persisted native status wake intents before orphan reaping",
+16 -11
View File
@@ -1,3 +1,5 @@
import { writeDeferredExecutionWait } from "../deferred-execution-wait.js";
import type { DeferredReceiptRecoveryOptions } from "../deferred-execution-notifications.js";
import { observeStoppedWorkspaceRecovery } from "../workspace-recovery.js";
import { notifyScheduledRetryCancellation } from "../scheduled-retry-cancellation-notifications.js";
import { notifyChatInboundWakeNotices } from "../chat-inbound-wake-notice-notifications.js";
@@ -616,12 +618,7 @@ export function createHeartbeatRunControl(db: Db, dependencies: HeartbeatRunCont
onBlocked: (reason, message) => { wait = { reason, message }; },
});
if (!admitted) {
await db.update(agentWakeupRequests).set({
payload: sql`jsonb_set(coalesce(${agentWakeupRequests.payload}, '{}'::jsonb), '{executionWait}',
coalesce(${agentWakeupRequests.payload}->'executionWait', '{}'::jsonb) || ${JSON.stringify(wait)}::jsonb)`,
updatedAt: new Date(),
}).where(and(eq(agentWakeupRequests.id, wake.id), eq(agentWakeupRequests.companyId, run.companyId),
eq(agentWakeupRequests.status, "deferred_issue_execution")));
await writeDeferredExecutionWait(db, { companyId: run.companyId, requestId: wake.id }, wait);
continue;
}
}
@@ -777,21 +774,25 @@ export function createHeartbeatRunControl(db: Db, dependencies: HeartbeatRunCont
idempotencyKey: `task-retry-dispatch:${wake.id}` }, wake.id);
}
async function resumeExecutionWaitComments() {
async function resumeExecutionWaitComments(scope: DeferredReceiptRecoveryOptions = {}) {
if ((await getSchedulingSuppression()).suppressed) return;
await reconcileTaskChatRetryReceipts(db);
await reconcileTaskChatRetryReceipts(db, scope);
const retries = await db.select({ id: agentWakeupRequests.id, companyId: agentWakeupRequests.companyId })
.from(agentWakeupRequests).innerJoin(issues, and(eq(issues.companyId, agentWakeupRequests.companyId),
sql`${issues.id}::text = ${agentWakeupRequests.payload}->>'issueId'`,
eq(issues.assigneeAgentId, agentWakeupRequests.agentId)))
.innerJoin(companies, and(eq(companies.id, issues.companyId), eq(companies.status, "active")))
.where(and(eq(agentWakeupRequests.reason, "task_retry"), eq(agentWakeupRequests.status, "deferred_issue_execution"),
.where(and(scope.companyId ? eq(agentWakeupRequests.companyId, scope.companyId) : undefined,
scope.requestId ? eq(agentWakeupRequests.id, scope.requestId) : undefined,
eq(agentWakeupRequests.reason, "task_retry"), eq(agentWakeupRequests.status, "deferred_issue_execution"),
sql`${agentWakeupRequests.payload}->'taskRetry'->>'chatActionId' is null`,
notInArray(issues.status, ["done", "cancelled"])))
.orderBy(asc(agentWakeupRequests.updatedAt)).limit(50);
for (const retry of retries) {
await db.update(agentWakeupRequests).set({ updatedAt: new Date() }).where(eq(agentWakeupRequests.id, retry.id));
await resumeTaskRetry(retry.companyId, retry.id).catch(err => logger.warn({ err, requestId: retry.id }, "Task retry dispatch deferred"));
await resumeTaskRetry(retry.companyId, retry.id).catch(err => {
scope.onError?.(err); logger.warn({ err, requestId: retry.id }, "Task retry dispatch deferred");
});
}
const waits = await db.select({ wake: agentWakeupRequests })
.from(agentWakeupRequests)
@@ -799,7 +800,9 @@ export function createHeartbeatRunControl(db: Db, dependencies: HeartbeatRunCont
sql`${issues.id}::text = ${agentWakeupRequests.payload}->>'issueId'`,
eq(issues.assigneeAgentId, agentWakeupRequests.agentId)))
.innerJoin(companies, and(eq(companies.id, issues.companyId), eq(companies.status, "active")))
.where(and(exists(db.select({ id: issueRecoveryActions.id }).from(issueRecoveryActions).where(and(
.where(and(scope.companyId ? eq(agentWakeupRequests.companyId, scope.companyId) : undefined,
scope.requestId ? eq(agentWakeupRequests.id, scope.requestId) : undefined,
exists(db.select({ id: issueRecoveryActions.id }).from(issueRecoveryActions).where(and(
eq(issueRecoveryActions.companyId, issues.companyId), eq(issueRecoveryActions.sourceIssueId, issues.id),
executionBlockerPredicate(),
))), eq(agentWakeupRequests.status, "deferred_issue_execution"),
@@ -831,11 +834,13 @@ export function createHeartbeatRunControl(db: Db, dependencies: HeartbeatRunCont
if (!run || run.companyId !== wake.companyId || run.agentId !== wake.agentId) continue;
if (canContinueCancelledRun(run)) {
await resumeSavedComments(wake.companyId, wake.id).catch(err => {
scope.onError?.(err);
logger.warn({ err, runId: run.id }, "failed to resume saved input after provider cancellation");
});
continue;
}
await resumeRemoteStopComments(run, wake.id).catch(err => {
scope.onError?.(err);
logger.warn({ err, runId: run.id }, "failed to resume saved execution-wait message");
});
}
+2
View File
@@ -1,3 +1,4 @@
import { notifyDeferredExecutionEligibility } from "./deferred-execution-notifications.js";
import { notifyRunDispatch } from "./run-dispatch-notifications.js";
import { notifyStatusCardEvaluation } from "./status-card-notifications.js";
import { notifySkillVersionRetentionReconstruction } from "./skill-version-retention-notifications.js";
@@ -636,6 +637,7 @@ export function instanceSettingsService(db: Db, options: InstanceSettingsService
if (patch.enableWorktreeRunExecution !== undefined) await notifyScheduledRetryCancellation(tx);
if (patch.enableWorktreeRunExecution !== undefined) {
await notifyRunDispatch(tx);
await notifyDeferredExecutionEligibility(tx);
await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers);
await notifyExternalAgentSettlement(tx);
await notifyInstructionCommit(tx);
@@ -1,5 +1,6 @@
import { notifyStatusCardIssueChange } from "./status-card-notifications.js";
import { notifyRoutineExecutionStatus } from "./routine-execution-status-notifications.js";
import { notifyDeferredExecutionEligibility } from "./deferred-execution-notifications.js";
import { notifyIssueRecovery } from "./issue-recovery-notifications.js";
import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js";
import { notifyNativeStatusReassessment } from "./native-status-reassessment-notifications.js";
@@ -762,6 +763,7 @@ export function issueTreeControlService(db: Db) {
const { hold: resumeHold } = await db.transaction(async (tx) => {
await notifyIssueRecovery(tx, { companyId });
await notifyDeferredExecutionEligibility(tx, { companyId });
const [createdHold] = await tx
.insert(issueTreeHolds)
.values({
@@ -841,6 +843,7 @@ export function issueTreeControlService(db: Db) {
const { hold, members } = await db.transaction(async (tx) => {
await notifyIssueRecovery(tx, { companyId });
await notifyDeferredExecutionEligibility(tx, { companyId });
const [createdHold] = await tx
.insert(issueTreeHolds)
.values({
@@ -1005,6 +1008,7 @@ export function issueTreeControlService(db: Db) {
const now = new Date();
const releasedCancelHoldIds = activeCancelHolds.map((hold) => hold.id);
const updatedIssues = await db.transaction(async (tx) => {
await notifyDeferredExecutionEligibility(tx, { companyId });
await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors);
await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers);
const restored: TreeStatusUpdateResult["updatedIssues"] = [];
@@ -1186,6 +1190,7 @@ export function issueTreeControlService(db: Db) {
const [updated] = await db.transaction(async tx => {
await notifyIssueRecovery(tx, { companyId });
await notifyDeferredExecutionEligibility(tx, { companyId });
return tx
.update(issueTreeHolds)
.set({
+32 -11
View File
@@ -1,3 +1,4 @@
import { notifyDeferredExecutionEligibility } from "./deferred-execution-notifications.js";
import { notifyIssueRecovery } from "./issue-recovery-notifications.js";
import { notifyRunDispatch } from "./run-dispatch-notifications.js";
import { notifyStatusCardIssueChange } from "./status-card-notifications.js";
@@ -3857,6 +3858,10 @@ async function readIssueBlockerAttentionBatch(
const nodesById = new Map<string, IssueBlockerAttentionNode>();
const edgesByIssueId = new Map<string, IssueBlockerAttentionEdge[]>();
// Returned rows are already bounded by the residual fetch budget. Keep their
// positive live-work evidence even when they cannot enter the traversal graph.
const observedNodes = new Map<string, IssueBlockerAttentionNode>();
const observedChildrenByParent = new Map<string, Set<string>>();
const admittedRoots = roots.slice(0, BLOCKER_ATTENTION_MAX_NODES);
for (const root of admittedRoots) nodesById.set(root.id, { ...root });
@@ -3953,6 +3958,10 @@ async function readIssueBlockerAttentionBatch(
);
for (const row of [...unresolvedExplicitBlockerRows, ...childRows]) {
if (!row.issueId) continue;
observedNodes.set(row.blockerIssueId, row);
const observed = observedChildrenByParent.get(row.issueId) ?? new Set<string>();
observed.add(row.blockerIssueId);
observedChildrenByParent.set(row.issueId, observed);
if (!nodesById.has(row.blockerIssueId)) {
if (nodesById.size >= BLOCKER_ATTENTION_MAX_NODES) { complete = false; continue; }
nodesById.set(row.blockerIssueId, {
@@ -4000,11 +4009,12 @@ async function readIssueBlockerAttentionBatch(
if (traversal.frontier.length > 0) traversal.truncated = true;
}
const nodeIds = [...nodesById.keys()];
const evidenceNodes = new Map([...nodesById, ...observedNodes]);
const nodeIds = [...evidenceNodes.keys()];
const activeIssueIds = new Set<string>();
const agentIds = new Set<string>();
const issueIdByExecutionRunId = new Map<string, string>();
for (const node of nodesById.values()) {
for (const node of evidenceNodes.values()) {
if (node.assigneeAgentId) agentIds.add(node.assigneeAgentId);
if (node.executionRunId)
issueIdByExecutionRunId.set(node.executionRunId, node.id);
@@ -4062,7 +4072,7 @@ async function readIssueBlockerAttentionBatch(
}
}
const explicitWaitCandidateIds = [...nodesById.values()]
const explicitWaitCandidateIds = [...evidenceNodes.values()]
.filter((node) => node.status !== "done")
.map((node) => node.id);
const explicitWaitingIssueIds = new Set<string>();
@@ -4407,19 +4417,22 @@ async function readIssueBlockerAttentionBatch(
};
};
const nodeHasLiveWork = (node: IssueBlockerAttentionNode) =>
node.status === "in_progress" || activeIssueIds.has(node.id) ||
explicitWaitingIssueIds.has(node.id) || Boolean(node.assigneeUserId);
const observedChildHasLiveWork = (parentId: string) =>
[...observedChildrenByParent.get(parentId) ?? []].some(id => {
const node = observedNodes.get(id);
return node?.companyId === companyId && nodeHasLiveWork(node);
});
const pathHasLiveWork = (nodeId: string, seen: Set<string>, traversal: Traversal): boolean => {
if (seen.has(nodeId) || !traversal.nodes.has(nodeId)) return false;
const node = nodesById.get(nodeId);
if (!node || node.companyId !== companyId) return false;
if (
node.status === "in_progress" ||
activeIssueIds.has(node.id) ||
explicitWaitingIssueIds.has(node.id) ||
Boolean(node.assigneeUserId)
)
return true;
if (nodeHasLiveWork(node)) return true;
if (!traversal.expanded.has(nodeId)) return false;
if (observedChildHasLiveWork(nodeId)) return true;
const nextSeen = new Set(seen);
nextSeen.add(nodeId);
return (edgesByIssueId.get(node.id) ?? []).some((edge) => {
@@ -4546,7 +4559,7 @@ async function readIssueBlockerAttentionBatch(
stalledEntry?.result.sampleStalledBlockerIdentifier ??
sampleStalledFromChain ??
null,
blockingTreeLive: topLevelEdges.some((edge) =>
blockingTreeLive: observedChildHasLiveWork(root.id) || topLevelEdges.some((edge) =>
pathHasLiveWork(edge.blockerIssueId, new Set([root.id]), traversal),
),
directBlockerIssueId: sampleEntry?.edge.blockerIssueId ?? null,
@@ -7653,6 +7666,7 @@ export function issueService(db: Db) {
await notifyIssueAttentionRetentionWork(dbOrTx, { companyId, issueId });
await notifyIssueRecovery(dbOrTx, { companyId });
await notifyDeferredExecutionEligibility(dbOrTx, { companyId });
await dbOrTx
.delete(issueRelations)
.where(
@@ -10643,6 +10657,7 @@ export function issueService(db: Db) {
}
if (values.monitorNextCheckAt) await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors);
await notifyRunDispatch(tx, { companyId: values.companyId });
await notifyDeferredExecutionEligibility(tx, { companyId: values.companyId });
await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers);
values.id ??= randomUUID();
await notifyStatusCardIssueChange(tx, companyId);
@@ -11548,6 +11563,7 @@ export function issueService(db: Db) {
|| patch.executionState !== undefined || patch.executionRunId !== undefined || patch.checkoutRunId !== undefined
|| blockedByIssueIds !== undefined) {
await notifyRunDispatch(tx, { companyId: receiptExisting.companyId });
await notifyDeferredExecutionEligibility(tx, { companyId: receiptExisting.companyId });
await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers);
await notifyRoutineExecutionStatus(tx, id);
}
@@ -12023,6 +12039,7 @@ export function issueService(db: Db) {
await notifyDecisionTargetIssueRemoval(tx, id);
await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors);
await notifyRunDispatch(tx, owner ? { companyId: owner.companyId } : undefined);
if (owner) await notifyDeferredExecutionEligibility(tx, owner);
await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers);
await notifyRoutineExecutionStatus(tx, id);
removedIssue = await tx
@@ -12155,6 +12172,7 @@ export function issueService(db: Db) {
await notifyIssueAttentionRetentionWork(tx, { companyId: issueCompany.companyId, issueId: id });
await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors);
await notifyRunDispatch(tx, { companyId: issueCompany.companyId });
await notifyDeferredExecutionEligibility(tx, { companyId: issueCompany.companyId });
await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers);
await notifyRoutineExecutionStatus(tx, id);
return tx
@@ -12291,6 +12309,7 @@ export function issueService(db: Db) {
await notifyIssueAttentionRetentionWork(tx, { companyId: issueCompany.companyId, issueId: id });
await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors);
await notifyRunDispatch(tx, { companyId: issueCompany.companyId });
await notifyDeferredExecutionEligibility(tx, { companyId: issueCompany.companyId });
await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers);
await notifyRoutineExecutionStatus(tx, id);
return tx
@@ -12549,6 +12568,7 @@ export function issueService(db: Db) {
await notifyIssueAttentionRetentionWork(tx, { companyId: existing.companyId, issueId: id });
await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors);
await notifyRunDispatch(tx, { companyId: existing.companyId });
await notifyDeferredExecutionEligibility(tx, { companyId: existing.companyId });
await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers);
await notifyRoutineExecutionStatus(tx, id);
const updated = await tx
@@ -12613,6 +12633,7 @@ export function issueService(db: Db) {
}
await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors);
await notifyRunDispatch(tx, { companyId: existing.companyId });
await notifyDeferredExecutionEligibility(tx, { companyId: existing.companyId });
await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers);
await notifyRoutineExecutionStatus(tx, id);
const updated = await tx
@@ -1,3 +1,4 @@
import { notifyNativeStatusWakeWork } from "../deferred-execution-notifications.js";
import { notifyExternalAgentSettlement } from "../../modules/external-agents/index.js";
import { lockCurrentNativeWorkAssessmentEvidence } from "./work-assessments.js";
import { hasNativeWorkspaceFinalizationOwner } from "./native-workspace-finalization-recovery-state.js";
@@ -478,6 +479,7 @@ async function enqueueWake(input: {
.limit(1)
.then((rows) => rows[0] ?? null);
if (existing) return existing.id;
await notifyNativeStatusWakeWork(input.tx, { companyId: input.companyId });
const inserted = await input.tx
.insert(agentWakeupRequests)
.values({
+4 -2
View File
@@ -339,8 +339,10 @@ export async function requestTaskRetry(input: {
/** Chat owns admission and authorization. The task receipt follows that
* durable outcome; it never launches a parallel generic board execution. */
export async function reconcileTaskChatRetryReceipts(db: Db) {
export async function reconcileTaskChatRetryReceipts(db: Db, scope: { companyId?: string; requestId?: string; settleOnly?: boolean } = {}) {
const pending = await db.select().from(agentWakeupRequests).where(and(
scope.companyId ? eq(agentWakeupRequests.companyId, scope.companyId) : undefined,
scope.requestId ? eq(agentWakeupRequests.id, scope.requestId) : undefined,
eq(agentWakeupRequests.reason, "task_retry"), eq(agentWakeupRequests.status, "deferred_issue_execution"),
sql`${agentWakeupRequests.payload}#>>'{taskRetry,chatActionId}' is not null`,
)).orderBy(agentWakeupRequests.updatedAt).limit(50);
@@ -356,7 +358,7 @@ export async function reconcileTaskChatRetryReceipts(db: Db) {
// A deferred chat receipt is admitted input, not a completed dispatch.
// Re-stage that exact action through chat's owner; it rechecks authority
// and normal queue policy before updating the existing receipt.
if (chat?.status === "processed" && chat.payload.taskRetryIntentId === wake.id &&
if (!scope.settleOnly && chat?.status === "processed" && chat.payload.taskRetryIntentId === wake.id &&
chat.payload.failedRunId === intent.sourceRunId && receipt?.status === "deferred_issue_execution" &&
!receipt.runId && receipt.payload?.taskRetryRequestId === wake.id) {
await db.transaction(async tx => {