mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-11-03 17:51:58 +01:00
Merge commit 'd1af4fd95405b1c25e3008e935b0c0b6e5b0369a' into codex/sleep-09-aggregate
* commit 'd1af4fd95405b1c25e3008e935b0c0b6e5b0369a': Join receipt acceptance passes before advancing saved deadlines fix(server): retain live evidence beyond blocker graph admission Retain moved receipt deadlines and wake released execution gates Drive saved execution receipts from committed work and deadlines
This commit is contained in:
commit
f131ecd7e0
31 files changed
+822
-58
No files matched your search
@@ -1,4 +1,5 @@
|
||||
import { notifyStatusCardIssueChange } from "./status-card-notifications.js";
|
||||
import { notifyDeferredExecutionEligibility } from "./deferred-execution-notifications.js";
|
||||
import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js";
|
||||
import { notifyNativeStatusReassessment } from "./native-status-reassessment-notifications.js";
|
||||
import { notifyUnsafeWorkspaceExportRecovery } from "./unsafe-workspace-export-notifications.js";
|
||||
@@ -744,6 +745,7 @@ export function accessService(db: Db) {
|
||||
sql`${issues.status} not in ('done', 'cancelled')`,
|
||||
);
|
||||
await notifyNativeStatusReassessment(tx, { companyId });
|
||||
await notifyDeferredExecutionEligibility(tx, { companyId });
|
||||
await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors);
|
||||
await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers);
|
||||
await notifyStatusCardIssueChange(tx, companyId);
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { notifyDeferredExecutionEligibility } from "./deferred-execution-notifications.js";
|
||||
import { notifyRunDispatch } from "./run-dispatch-notifications.js";
|
||||
import { notifyAttentionRetentionWork } from "./retention-work-notifications.js";
|
||||
import { notifySkillVersionPinChange } from "./skill-version-retention-notifications.js";
|
||||
@@ -87,6 +88,7 @@ export async function prepareAgentConfiguration(db: Db, existing: AgentConfigura
|
||||
);
|
||||
}
|
||||
await notifyRunDispatch(db, { companyId: existing.companyId });
|
||||
await notifyDeferredExecutionEligibility(db, { companyId: existing.companyId });
|
||||
if (normalizedPatch.runtimeConfig !== undefined || normalizedPatch.reportsTo !== undefined) {
|
||||
await notifyDeliveryWork(db, DELIVERY_QUEUES.agentTimers);
|
||||
}
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { notifyDeferredExecutionEligibility } from "./deferred-execution-notifications.js";
|
||||
import { notifyRunDispatch } from "./run-dispatch-notifications.js";
|
||||
import { notifyAttentionRetentionWork } from "./retention-work-notifications.js";
|
||||
import { notifyNativeReviewResponse } from "./native-review-response-notifications.js";
|
||||
@@ -43,6 +44,7 @@ export function createAgentLifecycleEffects(_hooks: BudgetServiceHooks = {}): li
|
||||
await notifyAttentionRetentionWork(tx, { companyId });
|
||||
if (before !== after) {
|
||||
await notifyRunDispatch(tx, { companyId });
|
||||
await notifyDeferredExecutionEligibility(tx, { companyId });
|
||||
await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers);
|
||||
}
|
||||
if (!isAgentStatusInvokable(before) && isAgentStatusInvokable(after)) {
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { notifyDeferredExecutionEligibility } from "./deferred-execution-notifications.js";
|
||||
import { notifyRunDispatch } from "./run-dispatch-notifications.js";
|
||||
import { notifyStatusCardIssueChange } from "./status-card-notifications.js";
|
||||
import { notifyExternalObjectRefresh } from "./external-object-refresh-notifications.js";
|
||||
@@ -291,6 +292,7 @@ export function budgetServiceInTransaction(db: Db, publications: ActivityPublica
|
||||
|
||||
if (policy.scopeType === "project") {
|
||||
await notifyRunDispatch(db, { companyId: policy.companyId });
|
||||
await notifyDeferredExecutionEligibility(db, { companyId: policy.companyId });
|
||||
await db
|
||||
.update(projects)
|
||||
.set({
|
||||
@@ -305,6 +307,7 @@ export function budgetServiceInTransaction(db: Db, publications: ActivityPublica
|
||||
const scope = await resolveScopeRecord(db, "company", policy.scopeId);
|
||||
if (scope.paused) return;
|
||||
await notifyRunDispatch(db, { companyId: policy.companyId });
|
||||
await notifyDeferredExecutionEligibility(db, { companyId: policy.companyId });
|
||||
await notifyCompanyRetentionWork(db, policy.companyId);
|
||||
await notifyStatusCardIssueChange(db, policy.companyId);
|
||||
await notifyExternalObjectRefresh(db, policy.companyId);
|
||||
@@ -354,6 +357,7 @@ export function budgetServiceInTransaction(db: Db, publications: ActivityPublica
|
||||
|
||||
if (policy.scopeType === "project") {
|
||||
await notifyRunDispatch(db, { companyId: policy.companyId });
|
||||
await notifyDeferredExecutionEligibility(db, { companyId: policy.companyId });
|
||||
await db
|
||||
.update(projects)
|
||||
.set({
|
||||
@@ -369,6 +373,7 @@ export function budgetServiceInTransaction(db: Db, publications: ActivityPublica
|
||||
const scope = await resolveScopeRecord(db, "company", policy.scopeId);
|
||||
if (scope.pauseReason !== "budget") return;
|
||||
await notifyRunDispatch(db, { companyId: policy.companyId });
|
||||
await notifyDeferredExecutionEligibility(db, { companyId: policy.companyId });
|
||||
await notifyCompanyRetentionWork(db, policy.companyId);
|
||||
await notifyStatusCardIssueChange(db, policy.companyId);
|
||||
await notifyExternalObjectRefresh(db, policy.companyId);
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { notifyDeferredExecutionWork, notifyTaskRetrySettlement } from "../deferred-execution-notifications.js";
|
||||
import { notifyNativeResponsePresentation } from "../native-response-presentation-notifications.js";
|
||||
import { notifyChatInboundWakeNotices } from "../chat-inbound-wake-notice-notifications.js";
|
||||
import { createFailedRetryNotices } from "./failed-retry-notices.js";
|
||||
@@ -1424,6 +1425,7 @@ export function createChatFailedRunRetry(db: Db, options: ChatFailedRunRetryOpti
|
||||
// Source authorization above checked every external receipt and its owner.
|
||||
// Reset only this certified unstarted batch under the task's admission lock.
|
||||
const skippedIds = source.taskRetryInput?.skippedActionIds ?? [];
|
||||
await notifyDeferredExecutionWork(tx, { companyId: input.companyId });
|
||||
await tx.update(agentWakeupRequests).set({ status: "deferred_issue_execution", runId: null, finishedAt: null,
|
||||
payload: sql`${agentWakeupRequests.payload} - 'coalescedIntoWakeupRequestId'`, updatedAt: new Date(),
|
||||
}).where(and(eq(agentWakeupRequests.companyId, input.companyId), eq(agentWakeupRequests.agentId, input.agentId),
|
||||
@@ -1784,6 +1786,9 @@ export function createChatFailedRunRetry(db: Db, options: ChatFailedRunRetryOpti
|
||||
const attemptCount = Number(candidate.result?.attemptCount ?? 0) + 1;
|
||||
const settle = async (status: string, result: Record<string, unknown>) => {
|
||||
const [current] = await db.transaction(async (tx) => {
|
||||
if (typeof claimed.payload.taskRetryIntentId === "string") await notifyTaskRetrySettlement(tx, {
|
||||
companyId: claimed.companyId, requestId: claimed.payload.taskRetryIntentId,
|
||||
});
|
||||
await notifyChatActionWork(tx, "failed_run_retry");
|
||||
if (status === "processed") await notifyNativeResponsePresentation(tx, {
|
||||
companyId: claimed.companyId, issueId: String(claimed.payload.issueId),
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { notifyDeferredExecutionWork } from "../deferred-execution-notifications.js";
|
||||
import { z } from "zod";
|
||||
import { and, asc, eq, gt, inArray, isNull, or, sql } from "drizzle-orm";
|
||||
import { agentWakeupRequests, chatActions, chatConversations, chatEndpoints, heartbeatRuns, issues, type Db } from "@paperclipai/db";
|
||||
@@ -91,6 +92,7 @@ export async function releaseUnadmittedTaskRetryInput(db: Db, input: { companyId
|
||||
)).for("update");
|
||||
if (receipt?.runId) return;
|
||||
await notifyChatInboundWakeNotices(tx, { companyId: input.companyId, issueId: issue.id });
|
||||
await notifyDeferredExecutionWork(tx, { companyId: input.companyId });
|
||||
await tx.update(agentWakeupRequests).set({ status: "deferred_issue_execution", finishedAt: null,
|
||||
payload: sql`${agentWakeupRequests.payload} - 'coalescedIntoWakeupRequestId'`, updatedAt: new Date(),
|
||||
}).where(and(eq(agentWakeupRequests.companyId, input.companyId), eq(agentWakeupRequests.status, "coalesced"),
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { notifyDeferredExecutionEligibility } from "./deferred-execution-notifications.js";
|
||||
import { notifyRunDispatch } from "./run-dispatch-notifications.js";
|
||||
import { notifyStatusCardIssueChange } from "./status-card-notifications.js";
|
||||
import { notifyGitHubCompanyChange } from "./github-connection-notifications.js";
|
||||
@@ -338,6 +339,7 @@ export function companyService(db: Db, _budgetHooks: BudgetServiceHooks = {}) {
|
||||
if (companyPatch.status !== undefined && companyPatch.status !== existing.status) {
|
||||
await notifyRunDispatch(tx, { companyId: id });
|
||||
await notifyGitHubCompanyChange(tx, id);
|
||||
await notifyDeferredExecutionEligibility(tx, { companyId: id });
|
||||
await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers);
|
||||
await notifyDeliveryWork(tx, DELIVERY_QUEUES.skillSourceSync);
|
||||
await notifyExternalObjectRefresh(tx, id);
|
||||
@@ -498,6 +500,7 @@ export function companyService(db: Db, _budgetHooks: BudgetServiceHooks = {}) {
|
||||
if (!wasAlreadyArchived) {
|
||||
await notifyRunDispatch(tx, { companyId: id });
|
||||
await notifyGitHubCompanyChange(tx, id);
|
||||
await notifyDeferredExecutionEligibility(tx, { companyId: id });
|
||||
await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers);
|
||||
await notifyDeliveryWork(tx, DELIVERY_QUEUES.skillSourceSync);
|
||||
await notifyExternalObjectRefresh(tx, id);
|
||||
|
||||
@@ -0,0 +1,34 @@
|
||||
import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js";
|
||||
|
||||
export type DeferredExecutionScope = { companyId: string; requestId?: string };
|
||||
export type DeferredReceiptRecoveryOptions = {
|
||||
companyId?: string;
|
||||
requestId?: string;
|
||||
onError?: (error: unknown) => void;
|
||||
};
|
||||
|
||||
function hint(scope: DeferredExecutionScope) {
|
||||
return JSON.stringify({ companyId: scope.companyId, requestId: scope.requestId });
|
||||
}
|
||||
|
||||
export async function notifyDeferredExecutionWork(tx: object, scope: DeferredExecutionScope) {
|
||||
await notifyDeliveryWork(tx, DELIVERY_QUEUES.deferredExecutionReceipts, hint(scope));
|
||||
}
|
||||
|
||||
export async function notifyNativeStatusWakeWork(tx: object, scope: DeferredExecutionScope) {
|
||||
await notifyDeliveryWork(tx, DELIVERY_QUEUES.nativeStatusWakeIntents, hint(scope));
|
||||
}
|
||||
|
||||
export async function notifyTaskRetrySettlement(tx: object, scope: DeferredExecutionScope) {
|
||||
await notifyDeliveryWork(tx, DELIVERY_QUEUES.taskRetryReceiptSettlement, hint(scope));
|
||||
}
|
||||
|
||||
export async function notifyDeferredExecutionEligibility(tx: object, scope?: { companyId: string }) {
|
||||
if (scope) {
|
||||
await notifyDeferredExecutionWork(tx, scope);
|
||||
await notifyNativeStatusWakeWork(tx, scope);
|
||||
} else {
|
||||
await notifyDeliveryWork(tx, DELIVERY_QUEUES.deferredExecutionReceipts, "*");
|
||||
await notifyDeliveryWork(tx, DELIVERY_QUEUES.nativeStatusWakeIntents, "*");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
import { and, eq, sql } from "drizzle-orm";
|
||||
import { agentWakeupRequests, type Db } from "@paperclipai/db";
|
||||
import { notifyDeferredExecutionWork } from "./deferred-execution-notifications.js";
|
||||
|
||||
export async function writeDeferredExecutionWait(db: Db, scope: { companyId: string; requestId: string },
|
||||
condition: Record<string, unknown>, options: { replace?: boolean } = {}) {
|
||||
return db.transaction(async tx => {
|
||||
const [current] = await tx.select({ payload: agentWakeupRequests.payload }).from(agentWakeupRequests)
|
||||
.where(and(eq(agentWakeupRequests.companyId, scope.companyId), eq(agentWakeupRequests.id, scope.requestId),
|
||||
eq(agentWakeupRequests.status, "deferred_issue_execution"))).for("update");
|
||||
if (!current) return;
|
||||
const previous = current.payload?.executionWait;
|
||||
const previousReason = previous && typeof previous === "object" && !Array.isArray(previous)
|
||||
? (previous as Record<string, unknown>).reason : undefined;
|
||||
if (!previous || previousReason !== condition.reason) await notifyDeferredExecutionWork(tx, scope);
|
||||
await tx.update(agentWakeupRequests).set({
|
||||
payload: sql`jsonb_set(coalesce(${agentWakeupRequests.payload}, '{}'::jsonb), '{executionWait}',
|
||||
${options.replace ? sql`'{}'::jsonb` : sql`coalesce(${agentWakeupRequests.payload}->'executionWait', '{}'::jsonb)`}
|
||||
|| ${JSON.stringify(condition)}::jsonb)`,
|
||||
updatedAt: new Date(),
|
||||
}).where(and(eq(agentWakeupRequests.companyId, scope.companyId), eq(agentWakeupRequests.id, scope.requestId),
|
||||
eq(agentWakeupRequests.status, "deferred_issue_execution")));
|
||||
});
|
||||
}
|
||||
@@ -2,6 +2,9 @@ import { signalDatabaseWork, subscribeDatabaseWork } from "@paperclipai/db";
|
||||
|
||||
/** Topics for existing durable queues; register intent before their writes. */
|
||||
export const DELIVERY_QUEUES = {
|
||||
deferredExecutionReceipts: "deferred-execution-receipts",
|
||||
taskRetryReceiptSettlement: "task-retry-receipt-settlement",
|
||||
nativeStatusWakeIntents: "native-status-wake-intents",
|
||||
orphanBlockers: "orphan-blockers",
|
||||
activeRecoveryActions: "active-recovery-actions",
|
||||
strandedAssignments: "stranded-assignments",
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import { notifyAttentionRetentionWork } from "./retention-work-notifications.js";
|
||||
import { notifyDeferredExecutionEligibility } from "./deferred-execution-notifications.js";
|
||||
import { notifyRunDispatch } from "./run-dispatch-notifications.js";
|
||||
import { notifyStatusCardIssueChange } from "./status-card-notifications.js";
|
||||
import { notifyRoutineExecutionStatus } from "./routine-execution-status-notifications.js";
|
||||
@@ -12,8 +13,9 @@ export async function notifyExecutionRunWork(transaction: object, scope?: Execut
|
||||
await notifyAttentionRetentionWork(transaction, scope);
|
||||
await notifyRunDispatch(transaction, scope);
|
||||
await notifyStatusCardIssueChange(transaction, scope.companyId);
|
||||
await notifyDeferredExecutionEligibility(transaction, scope);
|
||||
await notifyRoutineExecutionStatus(transaction, scope.issueId, scope.companyId);
|
||||
}
|
||||
if (scope) await notifyRoutineExecutionStatus(transaction, scope.issueId, scope.companyId);
|
||||
for (const queue of [DELIVERY_QUEUES.executionStatus, DELIVERY_QUEUES.executionFinalization,
|
||||
DELIVERY_QUEUES.executionReplacement, DELIVERY_QUEUES.executionDisposition, DELIVERY_QUEUES.issueMonitors]) {
|
||||
await notifyDeliveryWork(transaction, queue);
|
||||
@@ -31,6 +33,7 @@ export function writeExecutionControl<T>(
|
||||
if (scope) await notifyStatusCardIssueChange(transaction, scope.companyId);
|
||||
if (scope) await notifyAttentionRetentionWork(transaction, scope);
|
||||
if (scope) await notifyRoutineExecutionStatus(transaction, scope.issueId, scope.companyId);
|
||||
if (scope) await notifyDeferredExecutionEligibility(transaction, scope);
|
||||
await notifyDeliveryWork(transaction, DELIVERY_QUEUES.issueMonitors);
|
||||
for (const topic of typeof queue === "string" ? [queue] : queue) await notifyDeliveryWork(transaction, topic);
|
||||
return write(transaction as unknown as Db);
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
import { registerDeferredReceiptWork } from "./heartbeat/deferred-receipt-work.js";
|
||||
import { reconcileTaskChatRetryReceipts } from "./task-retry.js";
|
||||
import { registerIssueRecoveryWork } from "./recovery/work.js";
|
||||
import { registerRunDispatchWork } from "./heartbeat/dispatch-work.js";
|
||||
import { externalConversationSettlementWork } from "./external-conversation-settlement-work.js";
|
||||
@@ -4243,6 +4245,13 @@ export function heartbeatService(
|
||||
computeTaskDrain,
|
||||
applyTaskDrain,
|
||||
|
||||
registerDeferredReceiptWork: (coordinator: Parameters<typeof registerDeferredReceiptWork>[1], canRun: () => boolean) =>
|
||||
registerDeferredReceiptWork(db, coordinator, {
|
||||
resume: scope => heartbeatQueue.resumeDeferredExecutionReceipts(scope),
|
||||
settleTaskChat: scope => reconcileTaskChatRetryReceipts(db, { ...scope, settleOnly: true }),
|
||||
dispatchNative: dispatchPendingNativeStatusWakeups,
|
||||
schedulingSuppressed: async () => (await getSchedulingSuppression(true)).suppressed,
|
||||
}, () => canRun() && readTaskDrain(new Date()) === null),
|
||||
registerIssueStateRecoveryWork: (coordinator: Parameters<typeof registerIssueRecoveryWork>[1], canRun: () => boolean) =>
|
||||
registerIssueRecoveryWork(db, coordinator, {
|
||||
stranded: scope => reconcileStrandedAssignedIssues({ ...scope, skipExternalConversationSettlement: true, skipRelatedRecovery: true }),
|
||||
|
||||
@@ -0,0 +1,118 @@
|
||||
import { and, asc, eq, gt, inArray, isNull, or, sql } from "drizzle-orm";
|
||||
import { agentWakeupRequests, type Db } from "@paperclipai/db";
|
||||
import type { createDeliveryWorkCoordinator } from "../delivery-work-coordinator.js";
|
||||
import { DELIVERY_QUEUES } from "../delivery-work-notifications.js";
|
||||
import type { DeferredExecutionScope, DeferredReceiptRecoveryOptions } from "../deferred-execution-notifications.js";
|
||||
import { registerScopedDeadlineWork } from "../scoped-deadline-work.js";
|
||||
|
||||
type Entry = { id: string; companyId: string; dueAt: number };
|
||||
type NativeIntentOptions = DeferredExecutionScope & {
|
||||
onDeferredDeadline(at: number): void;
|
||||
onError(error: unknown): void;
|
||||
};
|
||||
|
||||
export function registerDeferredReceiptWork(
|
||||
db: Db,
|
||||
coordinator: ReturnType<typeof createDeliveryWorkCoordinator>,
|
||||
owner: {
|
||||
resume(scope: DeferredReceiptRecoveryOptions): Promise<unknown>;
|
||||
settleTaskChat(scope: DeferredExecutionScope): Promise<unknown>;
|
||||
dispatchNative(scope: NativeIntentOptions): Promise<unknown>;
|
||||
schedulingSuppressed(): Promise<boolean>;
|
||||
},
|
||||
canRun: () => boolean,
|
||||
) {
|
||||
function register(kind: "deferred" | "task_chat" | "native") {
|
||||
const cursors = new Map<string, string>();
|
||||
let enabled = false;
|
||||
const queue = kind === "deferred" ? DELIVERY_QUEUES.deferredExecutionReceipts
|
||||
: kind === "task_chat" ? DELIVERY_QUEUES.taskRetryReceiptSettlement : DELIVERY_QUEUES.nativeStatusWakeIntents;
|
||||
return registerScopedDeadlineWork<Entry, DeferredExecutionScope>(coordinator, {
|
||||
queue,
|
||||
reconstructionIntervalMs: 30_000,
|
||||
beforePass: async () => { enabled = !(await owner.schedulingSuppressed()); },
|
||||
decodeScope: hint => hint === "*" ? undefined : JSON.parse(hint),
|
||||
matches: (entry, scope) => !scope || entry.companyId === scope.companyId && (!scope.requestId || entry.id === scope.requestId),
|
||||
load: async scope => {
|
||||
const key = scope ? JSON.stringify({ companyId: scope.companyId, requestId: scope.requestId }) : "*";
|
||||
if (!enabled) { cursors.delete(key); return []; }
|
||||
const cursor = cursors.get(key);
|
||||
const rows = await db.select({ id: agentWakeupRequests.id, companyId: agentWakeupRequests.companyId,
|
||||
status: agentWakeupRequests.status, updatedAt: agentWakeupRequests.updatedAt,
|
||||
claimedAt: agentWakeupRequests.claimedAt, error: agentWakeupRequests.error,
|
||||
delayed: sql<boolean>`${agentWakeupRequests.payload}->'queuedCommentInterrupt' is not null
|
||||
or ${agentWakeupRequests.payload}->'executionWait' is not null`,
|
||||
interruptedLegacy: sql<boolean>`exists (select 1 from heartbeat_runs r
|
||||
where r.company_id = ${agentWakeupRequests.companyId} and r.agent_id = ${agentWakeupRequests.agentId}
|
||||
and r.runtime_mode = 'legacy' and r.status = 'cancelled'
|
||||
and r.result_json->>'queuedCommentInterruptQueueId' = ${agentWakeupRequests.id}::text)`,
|
||||
}).from(agentWakeupRequests).where(and(
|
||||
scope ? eq(agentWakeupRequests.companyId, scope.companyId) : undefined,
|
||||
scope?.requestId ? eq(agentWakeupRequests.id, scope.requestId) : undefined,
|
||||
cursor ? gt(agentWakeupRequests.id, cursor) : undefined,
|
||||
kind === "native" ? and(
|
||||
eq(agentWakeupRequests.requestedByActorType, "system"),
|
||||
eq(agentWakeupRequests.requestedByActorId, "native-status-committer"),
|
||||
inArray(agentWakeupRequests.status, ["queued", "claimed"]), isNull(agentWakeupRequests.runId),
|
||||
) : and(eq(agentWakeupRequests.status, "deferred_issue_execution"),
|
||||
kind === "task_chat" ? and(eq(agentWakeupRequests.reason, "task_retry"),
|
||||
sql`${agentWakeupRequests.payload}#>>'{taskRetry,chatActionId}' is not null`) : or(
|
||||
eq(agentWakeupRequests.reason, "task_retry"),
|
||||
sql`${agentWakeupRequests.payload}->'queuedCommentInterrupt' is not null`,
|
||||
sql`${agentWakeupRequests.payload}->'executionWait' is not null`,
|
||||
sql`jsonb_typeof(${agentWakeupRequests.payload}#>'{_paperclipWakeContext,wakeCommentIds}') = 'array'
|
||||
and ${agentWakeupRequests.payload}#>'{_paperclipWakeContext,wakeCommentIds}' <> '[]'::jsonb`,
|
||||
sql`${agentWakeupRequests.payload}->>'mutation' = 'interaction'`,
|
||||
sql`exists (select 1 from heartbeat_runs r where r.company_id = ${agentWakeupRequests.companyId}
|
||||
and r.runtime_mode = 'legacy' and r.status = 'cancelled'
|
||||
and r.result_json->>'queuedCommentInterruptQueueId' = ${agentWakeupRequests.id}::text)`,
|
||||
)),
|
||||
)).orderBy(asc(agentWakeupRequests.id)).limit(100);
|
||||
const complete = rows.length < 100;
|
||||
if (complete) cursors.delete(key); else cursors.set(key, rows.at(-1)!.id);
|
||||
return { rows: rows.map(row => ({ id: row.id, companyId: row.companyId,
|
||||
dueAt: kind === "native" && row.status === "claimed" && row.claimedAt ? row.claimedAt.getTime() + 60_000
|
||||
: kind === "native" && row.error ? row.updatedAt.getTime() + 30_000
|
||||
: kind === "deferred" && row.delayed && !row.interruptedLegacy ? row.updatedAt.getTime() + 30_000
|
||||
: Date.now(),
|
||||
})), complete };
|
||||
},
|
||||
check: async entry => {
|
||||
if (!enabled) return null;
|
||||
if (kind !== "task_chat") {
|
||||
const [current] = await db.select({ status: agentWakeupRequests.status, payload: agentWakeupRequests.payload,
|
||||
updatedAt: agentWakeupRequests.updatedAt, claimedAt: agentWakeupRequests.claimedAt, error: agentWakeupRequests.error,
|
||||
interruptedLegacy: sql<boolean>`exists (select 1 from heartbeat_runs r
|
||||
where r.company_id = ${agentWakeupRequests.companyId} and r.agent_id = ${agentWakeupRequests.agentId}
|
||||
and r.runtime_mode = 'legacy' and r.status = 'cancelled'
|
||||
and r.result_json->>'queuedCommentInterruptQueueId' = ${agentWakeupRequests.id}::text)`,
|
||||
}).from(agentWakeupRequests).where(and(eq(agentWakeupRequests.companyId, entry.companyId),
|
||||
eq(agentWakeupRequests.id, entry.id)));
|
||||
if (!current) return null;
|
||||
const due = kind === "native" && current.status === "claimed" && current.claimedAt
|
||||
? current.claimedAt.getTime() + 60_000
|
||||
: kind === "native" && current.status === "queued" && current.error ? current.updatedAt.getTime() + 30_000
|
||||
: kind === "deferred" && current.status === "deferred_issue_execution" && !current.interruptedLegacy
|
||||
&& (current.payload?.queuedCommentInterrupt || current.payload?.executionWait)
|
||||
? current.updatedAt.getTime() + 30_000 : null;
|
||||
if (due !== null && due > Date.now()) return { ...entry, dueAt: due };
|
||||
}
|
||||
const errors: unknown[] = [];
|
||||
let next: number | null = null;
|
||||
const scope = { companyId: entry.companyId, requestId: entry.id };
|
||||
if (kind === "native") await owner.dispatchNative({ ...scope,
|
||||
onDeferredDeadline: at => { next = next === null ? at : Math.min(next, at); },
|
||||
onError: error => { errors.push(error); },
|
||||
});
|
||||
else if (kind === "task_chat") await owner.settleTaskChat(scope);
|
||||
else await owner.resume({ ...scope, onError: error => { errors.push(error); } });
|
||||
if (errors.length) throw new AggregateError(errors, `Deferred receipt ${entry.id} could not settle`);
|
||||
return next === null ? null : { ...entry, dueAt: next };
|
||||
},
|
||||
}, canRun);
|
||||
}
|
||||
const workers = [register("deferred"), register("task_chat"), register("native")];
|
||||
return { ready: Promise.all(workers.map(worker => worker.ready)),
|
||||
wake() { for (const worker of workers) worker.invalidate(); },
|
||||
async stop() { await Promise.all(workers.map(worker => worker.stop())); } };
|
||||
}
|
||||
@@ -1,3 +1,6 @@
|
||||
import { writeDeferredExecutionWait } from "../deferred-execution-wait.js";
|
||||
import { notifyDeferredExecutionWork } from "../deferred-execution-notifications.js";
|
||||
import type { DeferredReceiptRecoveryOptions } from "../deferred-execution-notifications.js";
|
||||
import { notifyRunDispatch } from "../run-dispatch-notifications.js";
|
||||
import { notifyStatusCardIssueChange } from "../status-card-notifications.js";
|
||||
import { notifyRoutineExecutionStatus } from "../routine-execution-status-notifications.js";
|
||||
@@ -224,7 +227,7 @@ export interface HeartbeatQueueDependencies extends Pick<HeartbeatRetryDependenc
|
||||
publishRunLifecyclePluginEvent: (run: HeartbeatRun) => void;
|
||||
cancelRunInternal: (runId: string, reason?: string, options?: { errorCode?: string }) => Promise<unknown>;
|
||||
cancelActiveForAgentInternal: (agentId: string, reason: string) => Promise<unknown>;
|
||||
resumeExecutionWaitComments: () => Promise<unknown>;
|
||||
resumeExecutionWaitComments: (scope?: DeferredReceiptRecoveryOptions) => Promise<unknown>;
|
||||
resumeQueuedCommentInterrupt: (companyId: string, queueId: string) => Promise<unknown>;
|
||||
resumeSavedLegacyComments: (companyId: string, queueId: string) => Promise<unknown>;
|
||||
formatIssueIdentifierLink: (identifier: string | null, fallback: string) => string;
|
||||
@@ -1854,13 +1857,15 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
|
||||
return cancelled;
|
||||
}
|
||||
|
||||
async function resumeQueuedRuns() {
|
||||
async function resumeDeferredExecutionReceipts(scope: DeferredReceiptRecoveryOptions = {}) {
|
||||
if ((await getSchedulingSuppression()).suppressed) return;
|
||||
await resumeExecutionWaitComments();
|
||||
await resumeExecutionWaitComments(scope);
|
||||
const cutoff = await getWorktreeExecutionCutoff();
|
||||
const pendingInterrupts = await db.select({ id: agentWakeupRequests.id, companyId: agentWakeupRequests.companyId })
|
||||
.from(agentWakeupRequests).innerJoin(companies, eq(companies.id, agentWakeupRequests.companyId))
|
||||
.where(and(eq(agentWakeupRequests.status, "deferred_issue_execution"),
|
||||
.where(and(scope.companyId ? eq(agentWakeupRequests.companyId, scope.companyId) : undefined,
|
||||
scope.requestId ? eq(agentWakeupRequests.id, scope.requestId) : undefined,
|
||||
eq(agentWakeupRequests.status, "deferred_issue_execution"),
|
||||
eq(companies.status, "active"),
|
||||
sql`${agentWakeupRequests.payload}->'queuedCommentInterrupt' is not null`,
|
||||
lte(agentWakeupRequests.updatedAt, new Date(Date.now() - 30_000)),
|
||||
@@ -1871,6 +1876,7 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
|
||||
eq(agentWakeupRequests.id, wake.id), eq(agentWakeupRequests.status, "deferred_issue_execution"),
|
||||
));
|
||||
await resumeQueuedCommentInterrupt(wake.companyId, wake.id).catch(err => {
|
||||
scope.onError?.(err);
|
||||
logger.warn({ err, queueId: wake.id }, "failed to resume interrupted comment queue");
|
||||
});
|
||||
}
|
||||
@@ -1884,7 +1890,9 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
|
||||
or(eq(issues.assigneeAgentId, agentWakeupRequests.agentId),
|
||||
sql`${agentWakeupRequests.payload} #>> '{_paperclipWakeContext,wakeReason}' = 'issue_comment_mentioned'`)))
|
||||
.innerJoin(companies, and(eq(companies.id, issues.companyId), eq(companies.status, "active")))
|
||||
.where(and(eq(agentWakeupRequests.status, "deferred_issue_execution"),
|
||||
.where(and(scope.companyId ? eq(agentWakeupRequests.companyId, scope.companyId) : undefined,
|
||||
scope.requestId ? eq(agentWakeupRequests.id, scope.requestId) : undefined,
|
||||
eq(agentWakeupRequests.status, "deferred_issue_execution"),
|
||||
isNull(issues.executionRunId),
|
||||
or(and(
|
||||
sql`jsonb_typeof(${agentWakeupRequests.payload} #> '{_paperclipWakeContext,wakeCommentIds}') = 'array'`,
|
||||
@@ -1923,7 +1931,8 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
|
||||
if (latest.runtimeMode !== "legacy" && latest.runtimeMode !== "native") continue;
|
||||
if (wake.idempotencyKey?.startsWith("chat-inbound:")) {
|
||||
await resumeDeferredChatWake(db, { companyId: wake.companyId, agentId: wake.agentId,
|
||||
issueId: String(wake.payload?.issueId), requestId: wake.id }).catch(() => {
|
||||
issueId: String(wake.payload?.issueId), requestId: wake.id }).catch(err => {
|
||||
scope.onError?.(err);
|
||||
logger.warn({ requestId: wake.id }, "saved chat source could not resume after recovery");
|
||||
});
|
||||
continue;
|
||||
@@ -1935,6 +1944,7 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
|
||||
latest.errorCode === "execution_reconciliation_required";
|
||||
if ((latest.status !== "cancelled" || cancelledAdmission) && await getExecutionBlocker(db, wake.companyId, String(wake.payload?.issueId))) {
|
||||
await resumeSavedLegacyComments(wake.companyId, wake.id).catch(err => {
|
||||
scope.onError?.(err);
|
||||
logger.warn({ err, queueId: wake.id }, "failed to deliver saved legacy comment after recovery stopped");
|
||||
});
|
||||
}
|
||||
@@ -1942,6 +1952,7 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
|
||||
await releaseIssueExecutionAndPromote(latest, { suppressImmediateRecovery: true, resumeStrandedQueue: true }).then(() => {
|
||||
if (latest.runtimeMode === "native") releasedNativeSources.add(releaseSource);
|
||||
}).catch(err => {
|
||||
scope.onError?.(err);
|
||||
logger.warn({ err, queueId: wake.id }, "failed to promote stranded comments");
|
||||
});
|
||||
}
|
||||
@@ -1959,6 +1970,8 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
|
||||
))
|
||||
.innerJoin(companies, eq(companies.id, heartbeatRuns.companyId))
|
||||
.where(and(
|
||||
scope.companyId ? eq(agentWakeupRequests.companyId, scope.companyId) : undefined,
|
||||
scope.requestId ? eq(agentWakeupRequests.id, scope.requestId) : undefined,
|
||||
eq(agentWakeupRequests.status, "deferred_issue_execution"),
|
||||
eq(heartbeatRuns.status, "cancelled"),
|
||||
eq(heartbeatRuns.runtimeMode, "legacy"),
|
||||
@@ -1967,10 +1980,15 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
|
||||
));
|
||||
for (const run of interruptedQueues) {
|
||||
await releaseIssueExecutionAndPromote(run, { suppressImmediateRecovery: true }).catch((err) => {
|
||||
scope.onError?.(err);
|
||||
logger.error({ err, runId: run.id }, "failed to retry interrupted comment queue");
|
||||
});
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
async function resumeQueuedRuns() {
|
||||
await resumeDeferredExecutionReceipts();
|
||||
await resumeQueuedRunsAfterDrain();
|
||||
}
|
||||
|
||||
@@ -2509,17 +2527,11 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
|
||||
// its one durable receipt ID on a skipped wake.
|
||||
if (caseRequest) return { created: false };
|
||||
if (executionWaitRequestId) {
|
||||
await db.update(agentWakeupRequests).set({
|
||||
payload: sql`jsonb_set(coalesce(${agentWakeupRequests.payload}, '{}'::jsonb), '{executionWait}',
|
||||
coalesce(${agentWakeupRequests.payload}->'executionWait', '{}'::jsonb) || ${JSON.stringify({
|
||||
reason: skipReason, message: patch.error ?? (skipReason === "issue_tree_hold_active"
|
||||
? "This task is paused. Resume it to send your saved message."
|
||||
: "Waiting for task execution to be enabled. Your message is saved."),
|
||||
})}::jsonb)`,
|
||||
updatedAt: new Date(),
|
||||
}).where(and(eq(agentWakeupRequests.id, executionWaitRequestId),
|
||||
eq(agentWakeupRequests.companyId, agent.companyId), eq(agentWakeupRequests.agentId, agentId),
|
||||
eq(agentWakeupRequests.status, "deferred_issue_execution")));
|
||||
await writeDeferredExecutionWait(db, { companyId: agent.companyId, requestId: executionWaitRequestId }, {
|
||||
reason: skipReason, message: patch.error ?? (skipReason === "issue_tree_hold_active"
|
||||
? "This task is paused. Resume it to send your saved message."
|
||||
: "Waiting for task execution to be enabled. Your message is saved."),
|
||||
});
|
||||
return { created: false };
|
||||
}
|
||||
const request = {
|
||||
@@ -3407,15 +3419,14 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
|
||||
opts.issueStateGuard?.onExecutionDeferred?.(executionBlocker.cause);
|
||||
const condition = { recoveryActionId: executionBlocker.recoveryActionId, ...continuationWait };
|
||||
if (executionWaitRequestId) {
|
||||
await tx.update(agentWakeupRequests).set({
|
||||
payload: sql`jsonb_set(coalesce(${agentWakeupRequests.payload}, '{}'::jsonb), '{executionWait}', ${JSON.stringify(condition)}::jsonb)`,
|
||||
updatedAt: new Date(),
|
||||
}).where(eq(agentWakeupRequests.id, executionWaitRequestId));
|
||||
await writeDeferredExecutionWait(tx as unknown as Db, { companyId: agent.companyId,
|
||||
requestId: executionWaitRequestId }, condition, { replace: true });
|
||||
return { kind: "deferred" as const, admitted: true };
|
||||
}
|
||||
if (durableRequest || wakeCommentId ||
|
||||
hasInteractionContinuationWakeContext(enrichedContextSnapshot) ||
|
||||
readNonEmptyString(enrichedContextSnapshot.nativeStatusWakeIntentId)) {
|
||||
await notifyDeferredExecutionWork(tx, { companyId: agent.companyId });
|
||||
await tx.insert(agentWakeupRequests).values({
|
||||
...durableReceiptFields,
|
||||
companyId: agent.companyId, agentId, source, triggerDetail, reason,
|
||||
@@ -4035,6 +4046,7 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
|
||||
}));
|
||||
}
|
||||
if (followUp) {
|
||||
if (!followUp.consumed) await notifyDeferredExecutionWork(tx, { companyId: agent.companyId });
|
||||
const assignmentId = followUp.assignmentId;
|
||||
const pending = followUp.consumed ? undefined : pendingFollowUps.find(wake =>
|
||||
wake.payload?.[followUpPayloadKey] === assignmentId &&
|
||||
@@ -4773,6 +4785,9 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
|
||||
companyId?: string;
|
||||
limit?: number;
|
||||
staleClaimMs?: number;
|
||||
requestId?: string;
|
||||
onDeferredDeadline?: (at: number) => void;
|
||||
onError?: (error: unknown) => void;
|
||||
} = {},
|
||||
) {
|
||||
const now = new Date();
|
||||
@@ -4785,6 +4800,7 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
|
||||
input.companyId
|
||||
? eq(agentWakeupRequests.companyId, input.companyId)
|
||||
: undefined,
|
||||
input.requestId ? eq(agentWakeupRequests.id, input.requestId) : undefined,
|
||||
eq(agentWakeupRequests.requestedByActorType, "system"),
|
||||
eq(agentWakeupRequests.requestedByActorId, "native-status-committer"),
|
||||
inArray(agentWakeupRequests.status, ["queued", "claimed"]),
|
||||
@@ -4861,6 +4877,7 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
|
||||
candidate.claimedAt &&
|
||||
now.getTime() - candidate.claimedAt.getTime() < staleClaimMs
|
||||
) {
|
||||
input.onDeferredDeadline?.(candidate.claimedAt.getTime() + staleClaimMs);
|
||||
deferred += 1;
|
||||
continue;
|
||||
}
|
||||
@@ -5035,6 +5052,7 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
|
||||
if (wakeRun) dispatched += 1;
|
||||
else deferred += 1;
|
||||
} catch (error) {
|
||||
input.onError?.(error);
|
||||
await db
|
||||
.update(agentWakeupRequests)
|
||||
.set({
|
||||
@@ -5078,6 +5096,7 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
|
||||
withChatControlRecoveryGate,
|
||||
trackWakeup,
|
||||
resumeQueuedRuns,
|
||||
resumeDeferredExecutionReceipts,
|
||||
resumeQueuedRunsAfterDrain,
|
||||
parseHeartbeatPolicy,
|
||||
nextTimerHeartbeatAt,
|
||||
|
||||
@@ -1979,7 +1979,7 @@ export function createHeartbeatRecovery(db: Db, dependencies: HeartbeatRecoveryD
|
||||
|
||||
let settledProcessesDiscovered = false;
|
||||
|
||||
async function reapOrphanedRuns(opts?: { staleThresholdMs?: number; skipScheduledRetryCancellation?: boolean; skipCapturedInstructionCommit?: boolean; skipInstructionDirectoryCleanup?: boolean; skipStoppedInstructionCollection?: boolean; skipCancelledWorkspacePreservation?: boolean; skipObsoletePolicyReviews?: boolean; skipAutomaticCompletionReviews?: boolean; skipUnsafeWorkspaceExports?: boolean; skipNativePendingEffects?: boolean; skipNativeResponsePresentation?: boolean; skipNativeReviewResponse?: boolean; skipNativeStatusReassessment?: boolean; skipRetainedNativeSessionCleanup?: boolean; skipNativeWorkspaceFinalization?: boolean; skipNativeResultCommit?: boolean; skipNativeRunProjection?: boolean; skipNativeRecoveryNotices?: boolean; skipExternalConversationSettlement?: boolean; skipExternalAgentSettlement?: boolean; skipNativeEnvironmentSettlement?: boolean }) {
|
||||
async function reapOrphanedRuns(opts?: { staleThresholdMs?: number; skipNativeStatusWakeIntents?: boolean; skipScheduledRetryCancellation?: boolean; skipCapturedInstructionCommit?: boolean; skipInstructionDirectoryCleanup?: boolean; skipStoppedInstructionCollection?: boolean; skipCancelledWorkspacePreservation?: boolean; skipObsoletePolicyReviews?: boolean; skipAutomaticCompletionReviews?: boolean; skipUnsafeWorkspaceExports?: boolean; skipNativePendingEffects?: boolean; skipNativeResponsePresentation?: boolean; skipNativeReviewResponse?: boolean; skipNativeStatusReassessment?: boolean; skipRetainedNativeSessionCleanup?: boolean; skipNativeWorkspaceFinalization?: boolean; skipNativeResultCommit?: boolean; skipNativeRunProjection?: boolean; skipNativeRecoveryNotices?: boolean; skipExternalConversationSettlement?: boolean; skipExternalAgentSettlement?: boolean; skipNativeEnvironmentSettlement?: boolean }) {
|
||||
if (!opts?.skipCancelledWorkspacePreservation) await preserveCancelledNativeWorkspaces(db);
|
||||
const staleThresholdMs = opts?.staleThresholdMs ?? 0;
|
||||
const now = new Date();
|
||||
@@ -2023,7 +2023,7 @@ export function createHeartbeatRecovery(db: Db, dependencies: HeartbeatRecoveryD
|
||||
});
|
||||
if (!opts?.skipRetainedNativeSessionCleanup) scheduleRetainedNativeSessionCleanup();
|
||||
await reconcileRetainedWorkspaces(db, environmentRuntime, resumeRemoteStopComments);
|
||||
await dispatchPendingNativeStatusWakeups().catch((error) => {
|
||||
if (!opts?.skipNativeStatusWakeIntents) await dispatchPendingNativeStatusWakeups().catch((error) => {
|
||||
logger.warn(
|
||||
{ err: error },
|
||||
"failed to dispatch persisted native status wake intents before orphan reaping",
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
import { writeDeferredExecutionWait } from "../deferred-execution-wait.js";
|
||||
import type { DeferredReceiptRecoveryOptions } from "../deferred-execution-notifications.js";
|
||||
import { observeStoppedWorkspaceRecovery } from "../workspace-recovery.js";
|
||||
import { notifyScheduledRetryCancellation } from "../scheduled-retry-cancellation-notifications.js";
|
||||
import { notifyChatInboundWakeNotices } from "../chat-inbound-wake-notice-notifications.js";
|
||||
@@ -616,12 +618,7 @@ export function createHeartbeatRunControl(db: Db, dependencies: HeartbeatRunCont
|
||||
onBlocked: (reason, message) => { wait = { reason, message }; },
|
||||
});
|
||||
if (!admitted) {
|
||||
await db.update(agentWakeupRequests).set({
|
||||
payload: sql`jsonb_set(coalesce(${agentWakeupRequests.payload}, '{}'::jsonb), '{executionWait}',
|
||||
coalesce(${agentWakeupRequests.payload}->'executionWait', '{}'::jsonb) || ${JSON.stringify(wait)}::jsonb)`,
|
||||
updatedAt: new Date(),
|
||||
}).where(and(eq(agentWakeupRequests.id, wake.id), eq(agentWakeupRequests.companyId, run.companyId),
|
||||
eq(agentWakeupRequests.status, "deferred_issue_execution")));
|
||||
await writeDeferredExecutionWait(db, { companyId: run.companyId, requestId: wake.id }, wait);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
@@ -777,21 +774,25 @@ export function createHeartbeatRunControl(db: Db, dependencies: HeartbeatRunCont
|
||||
idempotencyKey: `task-retry-dispatch:${wake.id}` }, wake.id);
|
||||
}
|
||||
|
||||
async function resumeExecutionWaitComments() {
|
||||
async function resumeExecutionWaitComments(scope: DeferredReceiptRecoveryOptions = {}) {
|
||||
if ((await getSchedulingSuppression()).suppressed) return;
|
||||
await reconcileTaskChatRetryReceipts(db);
|
||||
await reconcileTaskChatRetryReceipts(db, scope);
|
||||
const retries = await db.select({ id: agentWakeupRequests.id, companyId: agentWakeupRequests.companyId })
|
||||
.from(agentWakeupRequests).innerJoin(issues, and(eq(issues.companyId, agentWakeupRequests.companyId),
|
||||
sql`${issues.id}::text = ${agentWakeupRequests.payload}->>'issueId'`,
|
||||
eq(issues.assigneeAgentId, agentWakeupRequests.agentId)))
|
||||
.innerJoin(companies, and(eq(companies.id, issues.companyId), eq(companies.status, "active")))
|
||||
.where(and(eq(agentWakeupRequests.reason, "task_retry"), eq(agentWakeupRequests.status, "deferred_issue_execution"),
|
||||
.where(and(scope.companyId ? eq(agentWakeupRequests.companyId, scope.companyId) : undefined,
|
||||
scope.requestId ? eq(agentWakeupRequests.id, scope.requestId) : undefined,
|
||||
eq(agentWakeupRequests.reason, "task_retry"), eq(agentWakeupRequests.status, "deferred_issue_execution"),
|
||||
sql`${agentWakeupRequests.payload}->'taskRetry'->>'chatActionId' is null`,
|
||||
notInArray(issues.status, ["done", "cancelled"])))
|
||||
.orderBy(asc(agentWakeupRequests.updatedAt)).limit(50);
|
||||
for (const retry of retries) {
|
||||
await db.update(agentWakeupRequests).set({ updatedAt: new Date() }).where(eq(agentWakeupRequests.id, retry.id));
|
||||
await resumeTaskRetry(retry.companyId, retry.id).catch(err => logger.warn({ err, requestId: retry.id }, "Task retry dispatch deferred"));
|
||||
await resumeTaskRetry(retry.companyId, retry.id).catch(err => {
|
||||
scope.onError?.(err); logger.warn({ err, requestId: retry.id }, "Task retry dispatch deferred");
|
||||
});
|
||||
}
|
||||
const waits = await db.select({ wake: agentWakeupRequests })
|
||||
.from(agentWakeupRequests)
|
||||
@@ -799,7 +800,9 @@ export function createHeartbeatRunControl(db: Db, dependencies: HeartbeatRunCont
|
||||
sql`${issues.id}::text = ${agentWakeupRequests.payload}->>'issueId'`,
|
||||
eq(issues.assigneeAgentId, agentWakeupRequests.agentId)))
|
||||
.innerJoin(companies, and(eq(companies.id, issues.companyId), eq(companies.status, "active")))
|
||||
.where(and(exists(db.select({ id: issueRecoveryActions.id }).from(issueRecoveryActions).where(and(
|
||||
.where(and(scope.companyId ? eq(agentWakeupRequests.companyId, scope.companyId) : undefined,
|
||||
scope.requestId ? eq(agentWakeupRequests.id, scope.requestId) : undefined,
|
||||
exists(db.select({ id: issueRecoveryActions.id }).from(issueRecoveryActions).where(and(
|
||||
eq(issueRecoveryActions.companyId, issues.companyId), eq(issueRecoveryActions.sourceIssueId, issues.id),
|
||||
executionBlockerPredicate(),
|
||||
))), eq(agentWakeupRequests.status, "deferred_issue_execution"),
|
||||
@@ -831,11 +834,13 @@ export function createHeartbeatRunControl(db: Db, dependencies: HeartbeatRunCont
|
||||
if (!run || run.companyId !== wake.companyId || run.agentId !== wake.agentId) continue;
|
||||
if (canContinueCancelledRun(run)) {
|
||||
await resumeSavedComments(wake.companyId, wake.id).catch(err => {
|
||||
scope.onError?.(err);
|
||||
logger.warn({ err, runId: run.id }, "failed to resume saved input after provider cancellation");
|
||||
});
|
||||
continue;
|
||||
}
|
||||
await resumeRemoteStopComments(run, wake.id).catch(err => {
|
||||
scope.onError?.(err);
|
||||
logger.warn({ err, runId: run.id }, "failed to resume saved execution-wait message");
|
||||
});
|
||||
}
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { notifyDeferredExecutionEligibility } from "./deferred-execution-notifications.js";
|
||||
import { notifyRunDispatch } from "./run-dispatch-notifications.js";
|
||||
import { notifyStatusCardEvaluation } from "./status-card-notifications.js";
|
||||
import { notifySkillVersionRetentionReconstruction } from "./skill-version-retention-notifications.js";
|
||||
@@ -636,6 +637,7 @@ export function instanceSettingsService(db: Db, options: InstanceSettingsService
|
||||
if (patch.enableWorktreeRunExecution !== undefined) await notifyScheduledRetryCancellation(tx);
|
||||
if (patch.enableWorktreeRunExecution !== undefined) {
|
||||
await notifyRunDispatch(tx);
|
||||
await notifyDeferredExecutionEligibility(tx);
|
||||
await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers);
|
||||
await notifyExternalAgentSettlement(tx);
|
||||
await notifyInstructionCommit(tx);
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import { notifyStatusCardIssueChange } from "./status-card-notifications.js";
|
||||
import { notifyRoutineExecutionStatus } from "./routine-execution-status-notifications.js";
|
||||
import { notifyDeferredExecutionEligibility } from "./deferred-execution-notifications.js";
|
||||
import { notifyIssueRecovery } from "./issue-recovery-notifications.js";
|
||||
import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js";
|
||||
import { notifyNativeStatusReassessment } from "./native-status-reassessment-notifications.js";
|
||||
@@ -762,6 +763,7 @@ export function issueTreeControlService(db: Db) {
|
||||
|
||||
const { hold: resumeHold } = await db.transaction(async (tx) => {
|
||||
await notifyIssueRecovery(tx, { companyId });
|
||||
await notifyDeferredExecutionEligibility(tx, { companyId });
|
||||
const [createdHold] = await tx
|
||||
.insert(issueTreeHolds)
|
||||
.values({
|
||||
@@ -841,6 +843,7 @@ export function issueTreeControlService(db: Db) {
|
||||
|
||||
const { hold, members } = await db.transaction(async (tx) => {
|
||||
await notifyIssueRecovery(tx, { companyId });
|
||||
await notifyDeferredExecutionEligibility(tx, { companyId });
|
||||
const [createdHold] = await tx
|
||||
.insert(issueTreeHolds)
|
||||
.values({
|
||||
@@ -1005,6 +1008,7 @@ export function issueTreeControlService(db: Db) {
|
||||
const now = new Date();
|
||||
const releasedCancelHoldIds = activeCancelHolds.map((hold) => hold.id);
|
||||
const updatedIssues = await db.transaction(async (tx) => {
|
||||
await notifyDeferredExecutionEligibility(tx, { companyId });
|
||||
await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors);
|
||||
await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers);
|
||||
const restored: TreeStatusUpdateResult["updatedIssues"] = [];
|
||||
@@ -1186,6 +1190,7 @@ export function issueTreeControlService(db: Db) {
|
||||
|
||||
const [updated] = await db.transaction(async tx => {
|
||||
await notifyIssueRecovery(tx, { companyId });
|
||||
await notifyDeferredExecutionEligibility(tx, { companyId });
|
||||
return tx
|
||||
.update(issueTreeHolds)
|
||||
.set({
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { notifyDeferredExecutionEligibility } from "./deferred-execution-notifications.js";
|
||||
import { notifyIssueRecovery } from "./issue-recovery-notifications.js";
|
||||
import { notifyRunDispatch } from "./run-dispatch-notifications.js";
|
||||
import { notifyStatusCardIssueChange } from "./status-card-notifications.js";
|
||||
@@ -3857,6 +3858,10 @@ async function readIssueBlockerAttentionBatch(
|
||||
|
||||
const nodesById = new Map<string, IssueBlockerAttentionNode>();
|
||||
const edgesByIssueId = new Map<string, IssueBlockerAttentionEdge[]>();
|
||||
// Returned rows are already bounded by the residual fetch budget. Keep their
|
||||
// positive live-work evidence even when they cannot enter the traversal graph.
|
||||
const observedNodes = new Map<string, IssueBlockerAttentionNode>();
|
||||
const observedChildrenByParent = new Map<string, Set<string>>();
|
||||
const admittedRoots = roots.slice(0, BLOCKER_ATTENTION_MAX_NODES);
|
||||
for (const root of admittedRoots) nodesById.set(root.id, { ...root });
|
||||
|
||||
@@ -3953,6 +3958,10 @@ async function readIssueBlockerAttentionBatch(
|
||||
);
|
||||
for (const row of [...unresolvedExplicitBlockerRows, ...childRows]) {
|
||||
if (!row.issueId) continue;
|
||||
observedNodes.set(row.blockerIssueId, row);
|
||||
const observed = observedChildrenByParent.get(row.issueId) ?? new Set<string>();
|
||||
observed.add(row.blockerIssueId);
|
||||
observedChildrenByParent.set(row.issueId, observed);
|
||||
if (!nodesById.has(row.blockerIssueId)) {
|
||||
if (nodesById.size >= BLOCKER_ATTENTION_MAX_NODES) { complete = false; continue; }
|
||||
nodesById.set(row.blockerIssueId, {
|
||||
@@ -4000,11 +4009,12 @@ async function readIssueBlockerAttentionBatch(
|
||||
if (traversal.frontier.length > 0) traversal.truncated = true;
|
||||
}
|
||||
|
||||
const nodeIds = [...nodesById.keys()];
|
||||
const evidenceNodes = new Map([...nodesById, ...observedNodes]);
|
||||
const nodeIds = [...evidenceNodes.keys()];
|
||||
const activeIssueIds = new Set<string>();
|
||||
const agentIds = new Set<string>();
|
||||
const issueIdByExecutionRunId = new Map<string, string>();
|
||||
for (const node of nodesById.values()) {
|
||||
for (const node of evidenceNodes.values()) {
|
||||
if (node.assigneeAgentId) agentIds.add(node.assigneeAgentId);
|
||||
if (node.executionRunId)
|
||||
issueIdByExecutionRunId.set(node.executionRunId, node.id);
|
||||
@@ -4062,7 +4072,7 @@ async function readIssueBlockerAttentionBatch(
|
||||
}
|
||||
}
|
||||
|
||||
const explicitWaitCandidateIds = [...nodesById.values()]
|
||||
const explicitWaitCandidateIds = [...evidenceNodes.values()]
|
||||
.filter((node) => node.status !== "done")
|
||||
.map((node) => node.id);
|
||||
const explicitWaitingIssueIds = new Set<string>();
|
||||
@@ -4407,19 +4417,22 @@ async function readIssueBlockerAttentionBatch(
|
||||
};
|
||||
};
|
||||
|
||||
const nodeHasLiveWork = (node: IssueBlockerAttentionNode) =>
|
||||
node.status === "in_progress" || activeIssueIds.has(node.id) ||
|
||||
explicitWaitingIssueIds.has(node.id) || Boolean(node.assigneeUserId);
|
||||
const observedChildHasLiveWork = (parentId: string) =>
|
||||
[...observedChildrenByParent.get(parentId) ?? []].some(id => {
|
||||
const node = observedNodes.get(id);
|
||||
return node?.companyId === companyId && nodeHasLiveWork(node);
|
||||
});
|
||||
const pathHasLiveWork = (nodeId: string, seen: Set<string>, traversal: Traversal): boolean => {
|
||||
if (seen.has(nodeId) || !traversal.nodes.has(nodeId)) return false;
|
||||
const node = nodesById.get(nodeId);
|
||||
if (!node || node.companyId !== companyId) return false;
|
||||
if (
|
||||
node.status === "in_progress" ||
|
||||
activeIssueIds.has(node.id) ||
|
||||
explicitWaitingIssueIds.has(node.id) ||
|
||||
Boolean(node.assigneeUserId)
|
||||
)
|
||||
return true;
|
||||
if (nodeHasLiveWork(node)) return true;
|
||||
|
||||
if (!traversal.expanded.has(nodeId)) return false;
|
||||
if (observedChildHasLiveWork(nodeId)) return true;
|
||||
const nextSeen = new Set(seen);
|
||||
nextSeen.add(nodeId);
|
||||
return (edgesByIssueId.get(node.id) ?? []).some((edge) => {
|
||||
@@ -4546,7 +4559,7 @@ async function readIssueBlockerAttentionBatch(
|
||||
stalledEntry?.result.sampleStalledBlockerIdentifier ??
|
||||
sampleStalledFromChain ??
|
||||
null,
|
||||
blockingTreeLive: topLevelEdges.some((edge) =>
|
||||
blockingTreeLive: observedChildHasLiveWork(root.id) || topLevelEdges.some((edge) =>
|
||||
pathHasLiveWork(edge.blockerIssueId, new Set([root.id]), traversal),
|
||||
),
|
||||
directBlockerIssueId: sampleEntry?.edge.blockerIssueId ?? null,
|
||||
@@ -7653,6 +7666,7 @@ export function issueService(db: Db) {
|
||||
|
||||
await notifyIssueAttentionRetentionWork(dbOrTx, { companyId, issueId });
|
||||
await notifyIssueRecovery(dbOrTx, { companyId });
|
||||
await notifyDeferredExecutionEligibility(dbOrTx, { companyId });
|
||||
await dbOrTx
|
||||
.delete(issueRelations)
|
||||
.where(
|
||||
@@ -10643,6 +10657,7 @@ export function issueService(db: Db) {
|
||||
}
|
||||
if (values.monitorNextCheckAt) await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors);
|
||||
await notifyRunDispatch(tx, { companyId: values.companyId });
|
||||
await notifyDeferredExecutionEligibility(tx, { companyId: values.companyId });
|
||||
await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers);
|
||||
values.id ??= randomUUID();
|
||||
await notifyStatusCardIssueChange(tx, companyId);
|
||||
@@ -11548,6 +11563,7 @@ export function issueService(db: Db) {
|
||||
|| patch.executionState !== undefined || patch.executionRunId !== undefined || patch.checkoutRunId !== undefined
|
||||
|| blockedByIssueIds !== undefined) {
|
||||
await notifyRunDispatch(tx, { companyId: receiptExisting.companyId });
|
||||
await notifyDeferredExecutionEligibility(tx, { companyId: receiptExisting.companyId });
|
||||
await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers);
|
||||
await notifyRoutineExecutionStatus(tx, id);
|
||||
}
|
||||
@@ -12023,6 +12039,7 @@ export function issueService(db: Db) {
|
||||
await notifyDecisionTargetIssueRemoval(tx, id);
|
||||
await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors);
|
||||
await notifyRunDispatch(tx, owner ? { companyId: owner.companyId } : undefined);
|
||||
if (owner) await notifyDeferredExecutionEligibility(tx, owner);
|
||||
await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers);
|
||||
await notifyRoutineExecutionStatus(tx, id);
|
||||
removedIssue = await tx
|
||||
@@ -12155,6 +12172,7 @@ export function issueService(db: Db) {
|
||||
await notifyIssueAttentionRetentionWork(tx, { companyId: issueCompany.companyId, issueId: id });
|
||||
await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors);
|
||||
await notifyRunDispatch(tx, { companyId: issueCompany.companyId });
|
||||
await notifyDeferredExecutionEligibility(tx, { companyId: issueCompany.companyId });
|
||||
await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers);
|
||||
await notifyRoutineExecutionStatus(tx, id);
|
||||
return tx
|
||||
@@ -12291,6 +12309,7 @@ export function issueService(db: Db) {
|
||||
await notifyIssueAttentionRetentionWork(tx, { companyId: issueCompany.companyId, issueId: id });
|
||||
await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors);
|
||||
await notifyRunDispatch(tx, { companyId: issueCompany.companyId });
|
||||
await notifyDeferredExecutionEligibility(tx, { companyId: issueCompany.companyId });
|
||||
await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers);
|
||||
await notifyRoutineExecutionStatus(tx, id);
|
||||
return tx
|
||||
@@ -12549,6 +12568,7 @@ export function issueService(db: Db) {
|
||||
await notifyIssueAttentionRetentionWork(tx, { companyId: existing.companyId, issueId: id });
|
||||
await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors);
|
||||
await notifyRunDispatch(tx, { companyId: existing.companyId });
|
||||
await notifyDeferredExecutionEligibility(tx, { companyId: existing.companyId });
|
||||
await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers);
|
||||
await notifyRoutineExecutionStatus(tx, id);
|
||||
const updated = await tx
|
||||
@@ -12613,6 +12633,7 @@ export function issueService(db: Db) {
|
||||
}
|
||||
await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors);
|
||||
await notifyRunDispatch(tx, { companyId: existing.companyId });
|
||||
await notifyDeferredExecutionEligibility(tx, { companyId: existing.companyId });
|
||||
await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers);
|
||||
await notifyRoutineExecutionStatus(tx, id);
|
||||
const updated = await tx
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { notifyNativeStatusWakeWork } from "../deferred-execution-notifications.js";
|
||||
import { notifyExternalAgentSettlement } from "../../modules/external-agents/index.js";
|
||||
import { lockCurrentNativeWorkAssessmentEvidence } from "./work-assessments.js";
|
||||
import { hasNativeWorkspaceFinalizationOwner } from "./native-workspace-finalization-recovery-state.js";
|
||||
@@ -478,6 +479,7 @@ async function enqueueWake(input: {
|
||||
.limit(1)
|
||||
.then((rows) => rows[0] ?? null);
|
||||
if (existing) return existing.id;
|
||||
await notifyNativeStatusWakeWork(input.tx, { companyId: input.companyId });
|
||||
const inserted = await input.tx
|
||||
.insert(agentWakeupRequests)
|
||||
.values({
|
||||
|
||||
@@ -339,8 +339,10 @@ export async function requestTaskRetry(input: {
|
||||
|
||||
/** Chat owns admission and authorization. The task receipt follows that
|
||||
* durable outcome; it never launches a parallel generic board execution. */
|
||||
export async function reconcileTaskChatRetryReceipts(db: Db) {
|
||||
export async function reconcileTaskChatRetryReceipts(db: Db, scope: { companyId?: string; requestId?: string; settleOnly?: boolean } = {}) {
|
||||
const pending = await db.select().from(agentWakeupRequests).where(and(
|
||||
scope.companyId ? eq(agentWakeupRequests.companyId, scope.companyId) : undefined,
|
||||
scope.requestId ? eq(agentWakeupRequests.id, scope.requestId) : undefined,
|
||||
eq(agentWakeupRequests.reason, "task_retry"), eq(agentWakeupRequests.status, "deferred_issue_execution"),
|
||||
sql`${agentWakeupRequests.payload}#>>'{taskRetry,chatActionId}' is not null`,
|
||||
)).orderBy(agentWakeupRequests.updatedAt).limit(50);
|
||||
@@ -356,7 +358,7 @@ export async function reconcileTaskChatRetryReceipts(db: Db) {
|
||||
// A deferred chat receipt is admitted input, not a completed dispatch.
|
||||
// Re-stage that exact action through chat's owner; it rechecks authority
|
||||
// and normal queue policy before updating the existing receipt.
|
||||
if (chat?.status === "processed" && chat.payload.taskRetryIntentId === wake.id &&
|
||||
if (!scope.settleOnly && chat?.status === "processed" && chat.payload.taskRetryIntentId === wake.id &&
|
||||
chat.payload.failedRunId === intent.sourceRunId && receipt?.status === "deferred_issue_execution" &&
|
||||
!receipt.runId && receipt.payload?.taskRetryRequestId === wake.id) {
|
||||
await db.transaction(async tx => {
|
||||
|
||||
Reference in new issue
Block a user