mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-11-03 09:37:19 +01:00
Merge commit 'd1af4fd95405b1c25e3008e935b0c0b6e5b0369a' into codex/sleep-09-aggregate
* commit 'd1af4fd95405b1c25e3008e935b0c0b6e5b0369a': Join receipt acceptance passes before advancing saved deadlines fix(server): retain live evidence beyond blocker graph admission Retain moved receipt deadlines and wake released execution gates Drive saved execution receipts from committed work and deadlines
This commit is contained in:
commit
f131ecd7e0
31 files changed
+822
-58
No files matched your search
@@ -377,6 +377,37 @@ const delay = DEFAULT_DECISION_ARCHIVE_DAYS * 86_400_000;
|
||||
expect(await db.select().from(decisionArchiveNotificationOutbox)).toHaveLength(0);
|
||||
});
|
||||
|
||||
it.each(["in_progress", "run", "confirmation"])("preserves observed child live evidence (%s) after explicit blockers fill the node budget", async (kind) => {
|
||||
const root = await insertIssue("blocked");
|
||||
await db.execute(sql`with blockers as (
|
||||
insert into issues (id, company_id, title, status, assignee_agent_id, updated_at)
|
||||
select gen_random_uuid(), ${companyId}::uuid, 'explicit-leaf-' || n, 'backlog', ${agentId}::uuid, ${old().toISOString()}::timestamptz
|
||||
from generate_series(1, 1999) n returning id)
|
||||
insert into issue_relations (company_id, issue_id, related_issue_id, type)
|
||||
select ${companyId}::uuid, id, ${root.id}::uuid, 'blocks' from blockers`);
|
||||
const child = await insertIssue(kind === "in_progress" ? "in_progress" : "todo", { parentId: root.id, assigneeAgentId: agentId });
|
||||
if (kind === "run") {
|
||||
const [run] = await db.insert(heartbeatRuns).values({ companyId, agentId, status: "running" }).returning();
|
||||
await db.update(issues).set({ executionRunId: run.id }).where(eq(issues.id, child.id));
|
||||
} else if (kind === "confirmation") {
|
||||
await db.insert(issueThreadInteractions).values({ companyId, issueId: child.id, kind: "request_confirmation", status: "pending", payload: { prompt: "Approve?" } });
|
||||
}
|
||||
parameterCounts.length = 0;
|
||||
const batch = await issueService(db).listBlockerAttentionBatch(companyId, [root]);
|
||||
expect(batch.nodeCount).toBeLessThanOrEqual(BLOCKER_ATTENTION_MAX_NODES);
|
||||
expect(batch.edgeCount).toBeLessThanOrEqual(BLOCKER_ATTENTION_MAX_NODES * 2);
|
||||
expect(batch.fetchedRows).toBeLessThanOrEqual(BLOCKER_ATTENTION_MAX_NODES * 6);
|
||||
expect(parameterCounts.length).toBeLessThanOrEqual(100);
|
||||
expect(Math.max(...parameterCounts)).toBeLessThanOrEqual(1100);
|
||||
expect(batch.deferredRootIds.has(root.id)).toBe(true);
|
||||
expect(batch.attention.get(root.id)?.blockingTreeLive).toBe(true);
|
||||
const service = attentionRetentionService(db), page = await service.load({ companyId, sourceKind: "blocker_attention" });
|
||||
const entry = page.rows!.find(item => item.rootId === root.id && item.variant === "terminal")!;
|
||||
expect(await service.check(entry)).toBeNull();
|
||||
expect(await db.select().from(decisionRetention)).toHaveLength(0);
|
||||
expect(await db.select().from(decisionArchiveNotificationOutbox)).toHaveLength(0);
|
||||
});
|
||||
|
||||
it("disarms finite deferred-root checks when large retained graphs have no archive obligation", async () => {
|
||||
await db.execute(sql`insert into issues (id, company_id, title, status, updated_at)
|
||||
select gen_random_uuid(), ${companyId}::uuid, 'held-root-' || n, 'blocked', ${old().toISOString()}::timestamptz
|
||||
|
||||
@@ -0,0 +1,427 @@
|
||||
import { setImmediate as immediate } from "node:timers/promises";
|
||||
import { DELIVERY_QUEUES, subscribeDeliveryWork } from "../services/delivery-work-notifications.js";
|
||||
import { accessService } from "../services/access.js";
|
||||
import { writeDeferredExecutionWait } from "../services/deferred-execution-wait.js";
|
||||
import { createQueuedCommentIssueLockWriter } from "../modules/wake-queue/adapters/queued-comment-postgres.js";
|
||||
import { createHeartbeatQueue } from "../services/heartbeat/queue.js";
|
||||
import { createHeartbeatQueueTestDependencies } from "./helpers/heartbeat-queue.js";
|
||||
import { issueTreeControlService } from "../services/issue-tree-control.js";
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { and, eq, sql } from "drizzle-orm";
|
||||
import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { agents, agentWakeupRequests, authUsers, chatActions, chatEndpoints, toolApplications, toolConnections, companies, companyMemberships, createDb, reconcileDatabaseWork, heartbeatRuns, issueComments, issues } from "@paperclipai/db";
|
||||
import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js";
|
||||
import { startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js";
|
||||
import { createDeliveryWorkCoordinator } from "../services/delivery-work-coordinator.js";
|
||||
import { notifyDeferredExecutionEligibility, notifyDeferredExecutionWork, notifyNativeStatusWakeWork, notifyTaskRetrySettlement } from "../services/deferred-execution-notifications.js";
|
||||
import { registerDeferredReceiptWork } from "../services/heartbeat/deferred-receipt-work.js";
|
||||
import { heartbeatService } from "../services/heartbeat.js";
|
||||
import { requestTaskRetry } from "../services/task-retry.js";
|
||||
import { issueService } from "../services/issues.js";
|
||||
import { idleWorkSnapshot } from "../services/task-admission.js";
|
||||
|
||||
vi.setConfig({ testTimeout: 90_000 });
|
||||
describe("deferred receipt durable dispatch", () => {
|
||||
let temporary: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>> | undefined;
|
||||
let db: ReturnType<typeof createDb>;
|
||||
let heartbeat: ReturnType<typeof heartbeatService>;
|
||||
let coordinator: ReturnType<typeof createDeliveryWorkCoordinator>;
|
||||
let drain = false;
|
||||
const errors: unknown[] = [];
|
||||
const passes = new Set<Promise<unknown>>();
|
||||
beforeAll(async () => {
|
||||
const external = resolveExternalTestDatabaseUrl("PAPERCLIP_DEFERRED_RECEIPT_TEST_DATABASE_URL");
|
||||
if (!external) temporary = await startEmbeddedPostgresTestDatabase("deferred-receipts-");
|
||||
db = createDb(external ?? temporary!.connectionString, { maxConnections: 6, idleTimeoutSeconds: 0, maxLifetimeSeconds: 0 });
|
||||
await Promise.all(Array.from({ length: 6 }, () => db.execute(sql`select pg_sleep(0.01)`)));
|
||||
});
|
||||
beforeEach(async () => {
|
||||
await db.execute(sql`truncate companies cascade`); drain = false; errors.length = 0;
|
||||
heartbeat = heartbeatService(db);
|
||||
coordinator = createDeliveryWorkCoordinator({ owner: db, canRun: () => !drain, canReconcile: () => true,
|
||||
onError: error => errors.push(error) });
|
||||
const register = coordinator.register.bind(coordinator);
|
||||
coordinator.register = (queue, task) => register(queue, { ...task, run: signal => {
|
||||
const pass = Promise.resolve(task.run(signal)); passes.add(pass);
|
||||
void pass.then(() => passes.delete(pass), () => passes.delete(pass)); return pass;
|
||||
} });
|
||||
});
|
||||
afterEach(async () => {
|
||||
await coordinator.stop(); await heartbeat.drainActiveRunExecutions();
|
||||
vi.useRealTimers(); vi.restoreAllMocks(); expect(errors).toEqual([]);
|
||||
});
|
||||
afterAll(async () => { await db?.$client.end({ timeout: 1 }); await temporary?.cleanup(); });
|
||||
async function fixture() {
|
||||
const companyId = randomUUID(), actorId = "deferred-receipt-user";
|
||||
await db.insert(authUsers).values({ id: actorId, name: "Receipt operator", email: `${actorId}@example.test`,
|
||||
createdAt: new Date(), updatedAt: new Date() }).onConflictDoNothing();
|
||||
await db.insert(companies).values({ id: companyId, name: "Deferred dispatch", issuePrefix: `DE${companyId.slice(0, 6)}`,
|
||||
defaultResponsibleUserId: actorId, requireBoardApprovalForNewAgents: false });
|
||||
await db.insert(companyMemberships).values({ companyId, principalId: actorId, principalType: "user", status: "active", membershipRole: "owner" });
|
||||
const [agent] = await db.insert(agents).values({ companyId, name: "Receipt process", adapterType: "process", status: "idle",
|
||||
adapterConfig: { command: process.execPath, args: ["-e", "console.log('receipt-proof')"], cwd: process.cwd() },
|
||||
runtimeConfig: { heartbeat: { enabled: true, intervalSec: 0, wakeOnDemand: true } } }).returning();
|
||||
const [issue] = await db.insert(issues).values({ companyId, title: "Resume saved task", status: "todo", assigneeAgentId: agent.id }).returning();
|
||||
const [source] = await db.insert(heartbeatRuns).values({ companyId, agentId: agent.id, issueId: issue.id,
|
||||
status: "failed", finishedAt: new Date(), responsibleUserId: actorId, errorCode: "process_failed",
|
||||
contextSnapshot: { issueId: issue.id }, resultJson: { executionRecovery: { kind: "bootstrap", providerWorkStarted: false } } }).returning();
|
||||
return { companyId, agent, issue, source, actorId };
|
||||
}
|
||||
async function quiet() {
|
||||
await vi.waitFor(async () => { expect(coordinator.nextWakeAt()).toBeNull(); expect(idleWorkSnapshot().active).toBe(0); }, { timeout: 40_000 });
|
||||
const reads = vi.spyOn(db, "select"), executes = vi.spyOn(db, "execute");
|
||||
const selected = reads.mock.calls.length, executed = executes.mock.calls.length;
|
||||
if (!vi.isFakeTimers()) vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"] });
|
||||
await vi.advanceTimersByTimeAsync(20 * 60_000);
|
||||
expect(reads).toHaveBeenCalledTimes(selected); expect(executes).toHaveBeenCalledTimes(executed);
|
||||
}
|
||||
async function finish(issueId: string) {
|
||||
await issueService(db).update(issueId, { status: "done" });
|
||||
await vi.waitFor(async () => {
|
||||
await Promise.allSettled([...passes]); await heartbeat.drainActiveRunExecutions();
|
||||
expect(await db.select().from(heartbeatRuns).where(sql`${heartbeatRuns.status} in ('queued', 'running')`)).toHaveLength(0);
|
||||
}, { timeout: 15_000 });
|
||||
const saved = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.status, "scheduled_retry"));
|
||||
if (!vi.isFakeTimers()) vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"] });
|
||||
if (saved.length) {
|
||||
const due = Math.max(...saved.map(run => run.scheduledRetryAt!.getTime()));
|
||||
while (Date.now() <= due) {
|
||||
await vi.advanceTimersByTimeAsync(Math.min(500, due - Date.now() + 1));
|
||||
await Promise.allSettled([...passes]); await immediate();
|
||||
}
|
||||
}
|
||||
await vi.waitFor(async () => {
|
||||
await vi.advanceTimersByTimeAsync(500); await Promise.allSettled([...passes]);
|
||||
await heartbeat.drainActiveRunExecutions();
|
||||
expect(await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.status, "scheduled_retry"))).toHaveLength(0);
|
||||
});
|
||||
}
|
||||
it("is quiet after empty reconstruction and terminal receipt history", async () => {
|
||||
const f = await fixture();
|
||||
await db.insert(agentWakeupRequests).values({ companyId: f.companyId, agentId: f.agent.id,
|
||||
source: "on_demand", status: "completed", reason: "task_retry", payload: { issueId: f.issue.id } });
|
||||
await heartbeat.registerDeferredReceiptWork(coordinator, () => !drain).ready; await quiet();
|
||||
});
|
||||
it("dispatches an ordinary committed task Retry into saved process output", async () => {
|
||||
const f = await fixture();
|
||||
await heartbeat.registerDeferredReceiptWork(coordinator, () => !drain).ready;
|
||||
await heartbeat.registerRunDispatchWork(coordinator, () => !drain).ready;
|
||||
await heartbeat.registerScheduledRetryCancellationWork(coordinator, () => !drain).ready;
|
||||
const receipt = await requestTaskRetry({ db, companyId: f.companyId, issueId: f.issue.id, actorId: f.actorId,
|
||||
observedRunId: f.source.id, idempotencyKey: randomUUID() });
|
||||
expect(receipt.accepted).toBe(true);
|
||||
let runId: string | undefined;
|
||||
await vi.waitFor(async () => {
|
||||
const [wake] = await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, receipt.requestId!));
|
||||
expect(wake.runId, JSON.stringify(wake)).toBeTruthy(); runId = wake.runId!;
|
||||
}, { timeout: 15_000 });
|
||||
await vi.waitFor(async () => {
|
||||
await heartbeat.drainActiveRunExecutions();
|
||||
const [run] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, runId!));
|
||||
expect(run.status, JSON.stringify({ error: run.error, code: run.errorCode })).toBe("succeeded");
|
||||
expect(run.stdoutExcerpt).toContain("receipt-proof");
|
||||
}, { timeout: 15_000 });
|
||||
await finish(f.issue.id);
|
||||
const linked = await db.select().from(agentWakeupRequests).where(and(eq(agentWakeupRequests.reason, "task_retry"), sql`${agentWakeupRequests.runId} is not null`));
|
||||
expect(linked).toHaveLength(2);
|
||||
expect(new Set(linked.map(row => row.runId))).toEqual(new Set([runId]));
|
||||
expect(linked.map(row => row.idempotencyKey)).toContain(`task-retry-dispatch:${receipt.requestId}`);
|
||||
expect(linked.find(row => row.id === receipt.requestId)?.status).toBe("coalesced");
|
||||
expect(await db.select().from(heartbeatRuns).where(sql`${heartbeatRuns.contextSnapshot}->>'taskRetryRequestId' = ${receipt.requestId}`)).toHaveLength(1);
|
||||
await quiet();
|
||||
});
|
||||
it.each(["native claim", "execution wait"])("reconstructs an exact fresh %s deadline without probing early", async kind => {
|
||||
const f = await fixture(), updatedAt = new Date(), native = kind === "native claim";
|
||||
const [wake] = await db.insert(agentWakeupRequests).values({ companyId: f.companyId, agentId: f.agent.id,
|
||||
source: "automation", status: native ? "claimed" : "deferred_issue_execution", claimedAt: updatedAt, updatedAt,
|
||||
payload: native ? {} : { executionWait: { reason: "recovery" } },
|
||||
requestedByActorType: "system", requestedByActorId: native ? "native-status-committer" : null }).returning();
|
||||
vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"] });
|
||||
const check = vi.fn(async (scope: { companyId?: string; requestId?: string }) => {
|
||||
await db.update(agentWakeupRequests).set({ status: "skipped" }).where(eq(agentWakeupRequests.id, scope.requestId!));
|
||||
});
|
||||
await registerDeferredReceiptWork(db, coordinator, { resume: check, settleTaskChat: vi.fn(), dispatchNative: check,
|
||||
schedulingSuppressed: async () => false }, () => !drain).ready;
|
||||
const due = updatedAt.getTime() + (native ? 60_000 : 30_000);
|
||||
expect(coordinator.nextWakeAt()).toBe(due); expect(check).not.toHaveBeenCalled();
|
||||
await vi.advanceTimersByTimeAsync(due - Date.now() - 1); expect(check).not.toHaveBeenCalled();
|
||||
await vi.advanceTimersByTimeAsync(1);
|
||||
await vi.waitFor(() => expect(check).toHaveBeenCalledExactlyOnceWith(expect.objectContaining({ companyId: f.companyId, requestId: wake.id })));
|
||||
await quiet();
|
||||
});
|
||||
it("reconciles the first drained producer and waits for resume without domain SQL", async () => {
|
||||
const f = await fixture(); drain = true;
|
||||
const resume = vi.fn(async () => {}), dispatchNative = vi.fn(async () => {});
|
||||
await registerDeferredReceiptWork(db, coordinator, { resume, settleTaskChat: vi.fn(), dispatchNative,
|
||||
schedulingSuppressed: async () => false }, () => !drain).ready;
|
||||
await db.transaction(async tx => {
|
||||
await notifyDeferredExecutionWork(tx, { companyId: f.companyId });
|
||||
await tx.insert(agentWakeupRequests).values({ companyId: f.companyId, agentId: f.agent.id, source: "on_demand",
|
||||
status: "deferred_issue_execution", reason: "task_retry" });
|
||||
});
|
||||
vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"] });
|
||||
await vi.advanceTimersByTimeAsync(1000);
|
||||
expect(resume).not.toHaveBeenCalled(); expect(dispatchNative).not.toHaveBeenCalled();
|
||||
await vi.waitFor(() => expect(idleWorkSnapshot().active).toBe(0));
|
||||
drain = false; await vi.advanceTimersByTimeAsync(30_000);
|
||||
await vi.waitFor(() => expect(resume).toHaveBeenCalledTimes(1)); await quiet();
|
||||
});
|
||||
it.each([
|
||||
{ name: "saved comment", payload: { _paperclipWakeContext: { wakeCommentIds: ["saved-comment"] } } },
|
||||
{ name: "interaction", payload: { mutation: "interaction" } },
|
||||
{ name: "execution wait", payload: { executionWait: { reason: "recovery" } } },
|
||||
{ name: "interrupt", payload: { queuedCommentInterrupt: { actorId: "operator" } } },
|
||||
])("activates only the committed company's $name receipt after an empty start", async ({ payload }) => {
|
||||
const f = await fixture(), foreign = await fixture();
|
||||
const resume = vi.fn(async () => {});
|
||||
await registerDeferredReceiptWork(db, coordinator, { resume, settleTaskChat: vi.fn(), dispatchNative: vi.fn(),
|
||||
schedulingSuppressed: async () => false }, () => !drain).ready;
|
||||
const [other] = await db.insert(agentWakeupRequests).values({ companyId: foreign.companyId, agentId: foreign.agent.id,
|
||||
source: "on_demand", status: "deferred_issue_execution", payload }).returning();
|
||||
let requestId: string;
|
||||
await db.transaction(async tx => {
|
||||
await tx.transaction(async nested => {
|
||||
await notifyDeferredExecutionWork(nested, { companyId: f.companyId });
|
||||
const [row] = await nested.insert(agentWakeupRequests).values({ companyId: f.companyId, agentId: f.agent.id,
|
||||
source: "on_demand", status: "deferred_issue_execution", payload, updatedAt: new Date(Date.now() - 31_000) }).returning();
|
||||
requestId = row.id;
|
||||
});
|
||||
expect(resume).not.toHaveBeenCalled();
|
||||
});
|
||||
await vi.waitFor(() => expect(resume).toHaveBeenCalledExactlyOnceWith(expect.objectContaining({ companyId: f.companyId, requestId })), { timeout: 5_000 });
|
||||
expect((await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, other.id)))[0].status).toBe("deferred_issue_execution");
|
||||
await quiet();
|
||||
});
|
||||
it("reconstructs a cancelled legacy run's exact deferred queue even without comment payload", async () => {
|
||||
const f = await fixture();
|
||||
const [wake] = await db.insert(agentWakeupRequests).values({ companyId: f.companyId, agentId: f.agent.id,
|
||||
source: "on_demand", status: "deferred_issue_execution" }).returning();
|
||||
await db.update(heartbeatRuns).set({ status: "cancelled", resultJson: { queuedCommentInterruptQueueId: wake.id } })
|
||||
.where(eq(heartbeatRuns.id, f.source.id));
|
||||
const resume = vi.fn(async () => {});
|
||||
await registerDeferredReceiptWork(db, coordinator, { resume, settleTaskChat: vi.fn(), dispatchNative: vi.fn(),
|
||||
schedulingSuppressed: async () => false }, () => !drain).ready;
|
||||
expect(resume).toHaveBeenCalledExactlyOnceWith(expect.objectContaining({ companyId: f.companyId, requestId: wake.id }));
|
||||
await quiet();
|
||||
});
|
||||
it("settles a processed chat receipt without restaging it until an actual eligibility commit", async () => {
|
||||
const f = await fixture();
|
||||
await heartbeat.registerDeferredReceiptWork(coordinator, () => !drain).ready;
|
||||
const [application] = await db.insert(toolApplications).values({ companyId: f.companyId, name: "Receipt chat", type: "mcp_http" }).returning();
|
||||
const [connection] = await db.insert(toolConnections).values({ companyId: f.companyId, applicationId: application.id,
|
||||
uid: randomUUID(), name: "Receipt chat", transport: "rest_api" }).returning();
|
||||
const [endpoint] = await db.insert(chatEndpoints).values({ companyId: f.companyId, connectionId: connection.id,
|
||||
provider: "slack", publicId: randomUUID(), assignedAgentId: f.agent.id }).returning();
|
||||
const intentId = randomUUID(), actionId = randomUUID(), previous = new Date(Date.now() - 60_000);
|
||||
await db.insert(agentWakeupRequests).values([
|
||||
{ id: intentId, companyId: f.companyId, agentId: f.agent.id, source: "on_demand", reason: "task_retry",
|
||||
status: "deferred_issue_execution", updatedAt: previous, payload: { issueId: f.issue.id,
|
||||
taskRetry: { schema: "paperclip.task-retry.v1", generationId: intentId, sourceRunId: f.source.id,
|
||||
actorId: f.actorId, strategy: "evaluate", chatActionId: actionId } } },
|
||||
{ id: actionId, companyId: f.companyId, agentId: f.agent.id, source: "automation", status: "deferred_issue_execution",
|
||||
payload: { issueId: f.issue.id, taskRetryRequestId: intentId } },
|
||||
]);
|
||||
await db.transaction(async tx => {
|
||||
await notifyTaskRetrySettlement(tx, { companyId: f.companyId, requestId: intentId });
|
||||
await tx.insert(chatActions).values({ id: actionId, companyId: f.companyId, endpointId: endpoint.id,
|
||||
providerActionId: randomUUID(), kind: "failed_run_retry", status: "processed", updatedAt: new Date(),
|
||||
payload: { issueId: f.issue.id, taskRetryIntentId: intentId, failedRunId: f.source.id } });
|
||||
});
|
||||
await vi.waitFor(async () => {
|
||||
const [intent] = await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, intentId));
|
||||
expect(intent.updatedAt.getTime()).toBeGreaterThan(previous.getTime());
|
||||
}, { timeout: 5_000 });
|
||||
expect((await db.select().from(chatActions).where(eq(chatActions.id, actionId)))[0].status).toBe("processed");
|
||||
await db.transaction(tx => notifyDeferredExecutionEligibility(tx, { companyId: f.companyId }));
|
||||
await vi.waitFor(async () => expect((await db.select().from(chatActions).where(eq(chatActions.id, actionId)))[0].status)
|
||||
.toBe("issued"), { timeout: 5_000 });
|
||||
await quiet();
|
||||
});
|
||||
it("retains a moved interrupt grace deadline after an ordinary saved queue reorder", async () => {
|
||||
const f = await fixture(), initialAt = new Date();
|
||||
const comments = await db.insert(issueComments).values(["first", "second"].map(body => ({
|
||||
companyId: f.companyId, issueId: f.issue.id, body, authorType: "user", authorUserId: f.actorId,
|
||||
}))).returning();
|
||||
const payload = { issueId: f.issue.id, queuedCommentInterrupt: { actorId: f.actorId },
|
||||
_paperclipWakeContext: { wakeCommentIds: comments.map(row => row.id) } };
|
||||
const [wake] = await db.insert(agentWakeupRequests).values({ companyId: f.companyId, agentId: f.agent.id,
|
||||
source: "on_demand", status: "deferred_issue_execution", payload, updatedAt: initialAt }).returning();
|
||||
vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"] });
|
||||
const deps = createHeartbeatQueueTestDependencies(db), queue = createHeartbeatQueue(db, deps);
|
||||
await registerDeferredReceiptWork(db, coordinator, { resume: scope => queue.resumeDeferredExecutionReceipts(scope),
|
||||
settleTaskChat: async () => {}, dispatchNative: async () => {}, schedulingSuppressed: async () => false }, () => !drain).ready;
|
||||
await vi.advanceTimersByTimeAsync(initialAt.getTime() + 10_000 - Date.now());
|
||||
const movedAt = new Date();
|
||||
const writer = createQueuedCommentIssueLockWriter(db, { syncCommentReferences: async () => {},
|
||||
deleteCommentReferenceSource: async () => {}, syncCommentExternalObjectsSafely: async () => {} });
|
||||
await writer.withLockedQueue({ issue: f.issue, queueId: wake.id,
|
||||
actor: { actorType: "user", actorId: f.actorId, agentId: null, runId: null, agentApiKeyId: null } },
|
||||
async (_locked, tx) => { await tx.updateWakeQueuedCommentIds({ wakeId: wake.id, payload,
|
||||
ids: comments.map(row => row.id).reverse(), updatedAt: movedAt }); });
|
||||
await vi.advanceTimersByTimeAsync(initialAt.getTime() + 30_000 - Date.now());
|
||||
await vi.waitFor(() => expect(idleWorkSnapshot().active).toBe(0));
|
||||
expect(deps.resumeQueuedCommentInterrupt).not.toHaveBeenCalled();
|
||||
expect(coordinator.nextWakeAt()).toBe(movedAt.getTime() + 30_000);
|
||||
await vi.advanceTimersByTimeAsync(movedAt.getTime() + 30_000 - Date.now());
|
||||
await vi.waitFor(() => expect(deps.resumeQueuedCommentInterrupt).toHaveBeenCalledExactlyOnceWith(f.companyId, wake.id));
|
||||
await quiet();
|
||||
});
|
||||
it("reactivates a held task Retry when the authorized subtree hold is released without wakeAgents", async () => {
|
||||
const f = await fixture(), tree = issueTreeControlService(db);
|
||||
const actor = { actorType: "user" as const, actorId: f.actorId, userId: f.actorId };
|
||||
const hold = await tree.createHold(f.companyId, f.issue.id, { mode: "pause", actor });
|
||||
await heartbeat.registerDeferredReceiptWork(coordinator, () => !drain).ready;
|
||||
await heartbeat.registerRunDispatchWork(coordinator, () => !drain).ready;
|
||||
await heartbeat.registerScheduledRetryCancellationWork(coordinator, () => !drain).ready;
|
||||
const request = await requestTaskRetry({ db, companyId: f.companyId, issueId: f.issue.id, actorId: f.actorId,
|
||||
observedRunId: f.source.id, idempotencyKey: randomUUID() });
|
||||
expect(request.requestId).toBeTruthy();
|
||||
await vi.waitFor(async () => {
|
||||
const [wake] = await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, request.requestId!));
|
||||
expect(wake.payload?.executionWait).toMatchObject({ reason: "issue_tree_hold_active" });
|
||||
expect(wake.runId).toBeNull();
|
||||
}, { timeout: 5_000 });
|
||||
await vi.waitFor(() => { expect(coordinator.nextWakeAt()).toBeNull(); expect(idleWorkSnapshot().active).toBe(0); }, { timeout: 40_000 });
|
||||
await tree.releaseHold(f.companyId, f.issue.id, hold.hold.id, { actor, reason: "Resume saved Retry" });
|
||||
let runId: string | null = null;
|
||||
await vi.waitFor(async () => {
|
||||
const [wake] = await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, request.requestId!));
|
||||
expect(wake.runId, JSON.stringify(wake)).toBeTruthy(); runId = wake.runId;
|
||||
}, { timeout: 40_000 });
|
||||
await vi.waitFor(async () => {
|
||||
await heartbeat.drainActiveRunExecutions();
|
||||
const [run] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, runId!));
|
||||
expect(run.status, JSON.stringify({ error: run.error, code: run.errorCode })).toBe("succeeded");
|
||||
expect(run.stdoutExcerpt).toContain("receipt-proof");
|
||||
}, { timeout: 15_000 });
|
||||
expect(await db.select().from(heartbeatRuns).where(sql`${heartbeatRuns.contextSnapshot}->>'taskRetryRequestId' = ${request.requestId}`)).toHaveLength(1);
|
||||
await finish(f.issue.id); await quiet();
|
||||
});
|
||||
it("keeps a moved native dispatch error deadline instead of retrying at its old time", async () => {
|
||||
const f = await fixture(), initialAt = new Date();
|
||||
const [wake] = await db.insert(agentWakeupRequests).values({ companyId: f.companyId, agentId: f.agent.id,
|
||||
source: "automation", status: "queued", error: "Transient dispatch failure", updatedAt: initialAt,
|
||||
requestedByActorType: "system", requestedByActorId: "native-status-committer" }).returning();
|
||||
vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"] });
|
||||
const dispatchNative = vi.fn(async () => {});
|
||||
await registerDeferredReceiptWork(db, coordinator, { resume: async () => {}, settleTaskChat: async () => {},
|
||||
dispatchNative, schedulingSuppressed: async () => false }, () => !drain).ready;
|
||||
await vi.advanceTimersByTimeAsync(initialAt.getTime() + 10_000 - Date.now());
|
||||
const movedAt = new Date();
|
||||
await db.update(agentWakeupRequests).set({ updatedAt: movedAt }).where(eq(agentWakeupRequests.id, wake.id));
|
||||
await vi.advanceTimersByTimeAsync(initialAt.getTime() + 30_000 - Date.now());
|
||||
await vi.waitFor(() => expect(idleWorkSnapshot().active).toBe(0));
|
||||
expect(dispatchNative).not.toHaveBeenCalled(); expect(coordinator.nextWakeAt()).toBe(movedAt.getTime() + 30_000);
|
||||
await vi.advanceTimersByTimeAsync(movedAt.getTime() + 30_000 - Date.now());
|
||||
await vi.waitFor(() => expect(dispatchNative).toHaveBeenCalledTimes(1)); await quiet();
|
||||
});
|
||||
it("arms one new wait reason grace and stays quiet when that same gate is still closed", async () => {
|
||||
const f = await fixture(); let reason = "operator_pause";
|
||||
const resume = vi.fn(async (scope: { companyId?: string; requestId?: string }) => {
|
||||
await writeDeferredExecutionWait(db, { companyId: scope.companyId!, requestId: scope.requestId! }, { reason });
|
||||
});
|
||||
await registerDeferredReceiptWork(db, coordinator, { resume, settleTaskChat: async () => {}, dispatchNative: async () => {},
|
||||
schedulingSuppressed: async () => false }, () => !drain).ready;
|
||||
const [wake] = await db.insert(agentWakeupRequests).values({ companyId: f.companyId, agentId: f.agent.id,
|
||||
source: "on_demand", status: "deferred_issue_execution", reason: "task_retry" }).returning();
|
||||
vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"] });
|
||||
for (const nextReason of ["operator_pause", "budget_pause"]) {
|
||||
reason = nextReason;
|
||||
await writeDeferredExecutionWait(db, { companyId: f.companyId, requestId: wake.id }, { reason });
|
||||
const [current] = await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, wake.id));
|
||||
const due = current.updatedAt.getTime() + 30_000;
|
||||
await vi.advanceTimersByTimeAsync(1_000);
|
||||
await vi.waitFor(() => expect(coordinator.nextWakeAt()).toBe(due));
|
||||
const checked = resume.mock.calls.length;
|
||||
await vi.advanceTimersByTimeAsync(due - Date.now());
|
||||
await vi.waitFor(() => expect(resume).toHaveBeenCalledTimes(checked + 1));
|
||||
await quiet();
|
||||
}
|
||||
expect(resume).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
it("reactivates a saved Retry through ordinary restore cancelled task", async () => {
|
||||
const f = await fixture(), actor = { actorType: "user" as const, actorId: f.actorId, userId: f.actorId };
|
||||
const request = await requestTaskRetry({ db, companyId: f.companyId, issueId: f.issue.id, actorId: f.actorId,
|
||||
observedRunId: f.source.id, idempotencyKey: randomUUID() });
|
||||
const tree = issueTreeControlService(db);
|
||||
const cancel = await tree.createHold(f.companyId, f.issue.id, { mode: "cancel", actor });
|
||||
await tree.cancelIssueStatusesForHold(f.companyId, f.issue.id, cancel.hold.id);
|
||||
const restore = await tree.createHold(f.companyId, f.issue.id, { mode: "restore", actor });
|
||||
const commitEligibility = () => tree.restoreIssueStatusesForHold(f.companyId, f.issue.id, restore.hold.id, { actor });
|
||||
await heartbeat.registerDeferredReceiptWork(coordinator, () => !drain).ready;
|
||||
await heartbeat.registerRunDispatchWork(coordinator, () => !drain).ready;
|
||||
await heartbeat.registerScheduledRetryCancellationWork(coordinator, () => !drain).ready;
|
||||
await vi.waitFor(() => { expect(coordinator.nextWakeAt()).toBeNull(); expect(idleWorkSnapshot().active).toBe(0); }, { timeout: 5_000 });
|
||||
expect((await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, request.requestId!)))[0].runId).toBeNull();
|
||||
await commitEligibility();
|
||||
let runId: string | null = null;
|
||||
await vi.waitFor(async () => {
|
||||
const [wake] = await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, request.requestId!));
|
||||
expect(wake.runId, JSON.stringify(wake)).toBeTruthy(); runId = wake.runId;
|
||||
}, { timeout: 15_000 });
|
||||
await vi.waitFor(async () => {
|
||||
await heartbeat.drainActiveRunExecutions();
|
||||
const [run] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, runId!));
|
||||
expect(run.status, JSON.stringify({ error: run.error, code: run.errorCode })).toBe("succeeded");
|
||||
expect(run.stdoutExcerpt).toContain("receipt-proof");
|
||||
}, { timeout: 15_000 });
|
||||
expect(await db.select().from(heartbeatRuns).where(sql`${heartbeatRuns.contextSnapshot}->>'taskRetryRequestId' = ${request.requestId}`)).toHaveLength(1);
|
||||
await finish(f.issue.id); await quiet();
|
||||
});
|
||||
it("member archive publishes committed eligibility while preserving a cancelled Retry tombstone", async () => {
|
||||
const f = await fixture(), departing = `departing-${randomUUID()}`;
|
||||
const request = await requestTaskRetry({ db, companyId: f.companyId, issueId: f.issue.id, actorId: f.actorId,
|
||||
observedRunId: f.source.id, idempotencyKey: randomUUID() });
|
||||
await db.insert(authUsers).values({ id: departing, name: "Departing member", email: `${departing}@example.test`,
|
||||
createdAt: new Date(), updatedAt: new Date() });
|
||||
const [member] = await db.insert(companyMemberships).values({ companyId: f.companyId, principalId: departing,
|
||||
principalType: "user", status: "active", membershipRole: "member" }).returning();
|
||||
await issueService(db).update(f.issue.id, { assigneeUserId: departing, assigneeAgentId: null });
|
||||
const readIntent = async () => (await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, request.requestId!)))[0];
|
||||
expect(await readIntent()).toMatchObject({ status: "cancelled", runId: null });
|
||||
const deferred = vi.fn(), native = vi.fn();
|
||||
const unsubscribes = [subscribeDeliveryWork(db, DELIVERY_QUEUES.deferredExecutionReceipts, deferred),
|
||||
subscribeDeliveryWork(db, DELIVERY_QUEUES.nativeStatusWakeIntents, native)];
|
||||
try {
|
||||
await expect(db.transaction(async tx => {
|
||||
await accessService(tx as unknown as typeof db).archiveMember(f.companyId, member.id,
|
||||
{ reassignment: { assigneeAgentId: f.agent.id } });
|
||||
expect(deferred).not.toHaveBeenCalled(); expect(native).not.toHaveBeenCalled();
|
||||
throw new Error("roll back member archive");
|
||||
})).rejects.toThrow("roll back member archive");
|
||||
expect(deferred).toHaveBeenCalledTimes(1); expect(native).toHaveBeenCalledTimes(1);
|
||||
expect((await db.select().from(issues).where(eq(issues.id, f.issue.id)))[0].assigneeUserId).toBe(departing);
|
||||
await reconcileDatabaseWork(db, DELIVERY_QUEUES.deferredExecutionReceipts);
|
||||
await reconcileDatabaseWork(db, DELIVERY_QUEUES.nativeStatusWakeIntents);
|
||||
deferred.mockClear(); native.mockClear();
|
||||
await db.transaction(async tx => {
|
||||
await accessService(tx as unknown as typeof db).archiveMember(f.companyId, member.id,
|
||||
{ reassignment: { assigneeAgentId: f.agent.id } });
|
||||
expect(deferred).not.toHaveBeenCalled(); expect(native).not.toHaveBeenCalled();
|
||||
});
|
||||
expect(deferred).toHaveBeenCalledTimes(1); expect(native).toHaveBeenCalledTimes(1);
|
||||
expect((await db.select().from(issues).where(eq(issues.id, f.issue.id)))[0]).toMatchObject({
|
||||
assigneeUserId: null, assigneeAgentId: f.agent.id,
|
||||
});
|
||||
expect((await db.select().from(companyMemberships).where(eq(companyMemberships.id, member.id)))[0].status).toBe("archived");
|
||||
expect(await readIntent()).toMatchObject({ status: "cancelled", runId: null });
|
||||
expect(await db.select().from(heartbeatRuns).where(sql`${heartbeatRuns.contextSnapshot}->>'taskRetryRequestId' = ${request.requestId}`)).toHaveLength(0);
|
||||
} finally { for (const unsubscribe of unsubscribes) unsubscribe(); }
|
||||
await quiet();
|
||||
});
|
||||
it("outer rollback and nested savepoint rollback do not dispatch receipt intent", async () => {
|
||||
const f = await fixture(); const dispatchNative = vi.fn(async () => {});
|
||||
await registerDeferredReceiptWork(db, coordinator, { resume: vi.fn(), settleTaskChat: vi.fn(), dispatchNative,
|
||||
schedulingSuppressed: async () => false }, () => !drain).ready;
|
||||
await expect(db.transaction(async tx => {
|
||||
await notifyNativeStatusWakeWork(tx, { companyId: f.companyId });
|
||||
throw new Error("outer rollback");
|
||||
})).rejects.toThrow("outer rollback");
|
||||
await db.transaction(async tx => {
|
||||
await expect(tx.transaction(async nested => {
|
||||
await notifyNativeStatusWakeWork(nested, { companyId: f.companyId }); throw new Error("nested rollback");
|
||||
})).rejects.toThrow("nested rollback");
|
||||
});
|
||||
await quiet(); expect(dispatchNative).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
@@ -145,6 +145,7 @@ const {
|
||||
reconcileHotRestartAdoption: vi.fn(async () => ({ mode: "none" })),
|
||||
reapOrphanedRuns: vi.fn(async () => ({ reaped: 0, runIds: [] })),
|
||||
registerIssueStateRecoveryWork: vi.fn((_coordinator: unknown, _canRun: () => boolean) => ({ ready: Promise.resolve() })),
|
||||
registerDeferredReceiptWork: vi.fn((_coordinator: unknown, _canRun: () => boolean) => ({ ready: Promise.resolve() })),
|
||||
registerRunDispatchWork: vi.fn((_coordinator: unknown, _canRun: () => boolean) => ({ ready: Promise.resolve() })),
|
||||
registerScheduledRetryCancellationWork: vi.fn((_coordinator: unknown, _canRun: () => boolean) => ({ ready: Promise.resolve() })),
|
||||
registerStoppedInstructionCollectionWork: vi.fn((_coordinator: unknown, _canRun: () => boolean) => ({ ready: Promise.resolve() })),
|
||||
@@ -993,6 +994,8 @@ describe("startServer feedback export wiring", () => {
|
||||
await startServer(); await settleStartupHousekeeping();
|
||||
expect(heartbeatServiceMock.registerIssueStateRecoveryWork).toHaveBeenCalledTimes(enabled ? 1 : 0);
|
||||
expect(heartbeatServiceMock.registerRunDispatchWork).toHaveBeenCalledTimes(enabled ? 1 : 0);
|
||||
expect(heartbeatServiceMock.registerDeferredReceiptWork).toHaveBeenCalledTimes(enabled ? 1 : 0);
|
||||
expect(heartbeatServiceMock.resumeQueuedRuns).not.toHaveBeenCalled();
|
||||
if (enabled) {
|
||||
const [coordinator, admitted] = heartbeatServiceMock.registerRunDispatchWork.mock.calls[0];
|
||||
const app = await createAppMock.mock.results.at(-1)!.value;
|
||||
@@ -1436,7 +1439,7 @@ describe("startServer feedback export wiring", () => {
|
||||
intervalCallback?.();
|
||||
await new Promise<void>((resolve) => setImmediate(resolve));
|
||||
expect(heartbeatServiceMock.sweepStaleIssueLocks).not.toHaveBeenCalled();
|
||||
expect(heartbeatServiceMock.reapOrphanedRuns).toHaveBeenLastCalledWith({ staleThresholdMs: 300_000, skipScheduledRetryCancellation: true, skipCapturedInstructionCommit: true, skipInstructionDirectoryCleanup: true, skipStoppedInstructionCollection: true, skipCancelledWorkspacePreservation: true, skipObsoletePolicyReviews: true, skipAutomaticCompletionReviews: true, skipUnsafeWorkspaceExports: true, skipNativePendingEffects: true, skipNativeResponsePresentation: true, skipNativeReviewResponse: true, skipNativeStatusReassessment: true, skipRetainedNativeSessionCleanup: true, skipNativeWorkspaceFinalization: true, skipNativeResultCommit: true, skipNativeRunProjection: true, skipNativeRecoveryNotices: true, skipExternalConversationSettlement: true, skipExternalAgentSettlement: true, skipNativeEnvironmentSettlement: true });
|
||||
expect(heartbeatServiceMock.reapOrphanedRuns).toHaveBeenLastCalledWith({ staleThresholdMs: 300_000, skipNativeStatusWakeIntents: true, skipScheduledRetryCancellation: true, skipCapturedInstructionCommit: true, skipInstructionDirectoryCleanup: true, skipStoppedInstructionCollection: true, skipCancelledWorkspacePreservation: true, skipObsoletePolicyReviews: true, skipAutomaticCompletionReviews: true, skipUnsafeWorkspaceExports: true, skipNativePendingEffects: true, skipNativeResponsePresentation: true, skipNativeReviewResponse: true, skipNativeStatusReassessment: true, skipRetainedNativeSessionCleanup: true, skipNativeWorkspaceFinalization: true, skipNativeResultCommit: true, skipNativeRunProjection: true, skipNativeRecoveryNotices: true, skipExternalConversationSettlement: true, skipExternalAgentSettlement: true, skipNativeEnvironmentSettlement: true });
|
||||
expect(heartbeatServiceMock.reconcileCostAccounting).toHaveBeenCalledTimes(1);
|
||||
expect(heartbeatServiceMock.promoteDueScheduledRetries).not.toHaveBeenCalled();
|
||||
expect(retiredDetector).not.toHaveBeenCalled();
|
||||
|
||||
+3
-5
@@ -1704,7 +1704,6 @@ async function startServerWithDatabaseTeardown(
|
||||
}
|
||||
}
|
||||
|
||||
await timeStartupRecovery("queued_runs", () => heartbeat.resumeQueuedRuns());
|
||||
const recoveredGoalActions = await timeStartupRecovery("goal_actions", () => heartbeat.recoverPendingSessionGoalActions());
|
||||
if (
|
||||
recoveredGoalActions.enqueued > 0 ||
|
||||
@@ -1736,6 +1735,8 @@ async function startServerWithDatabaseTeardown(
|
||||
|
||||
if (config.heartbeatSchedulerEnabled) await heartbeat.registerIssueStateRecoveryWork(deliveryWork,
|
||||
() => !isIdleTaskDrainActive() && !isWarmStandby() && !heartbeatSchedulerStopped).ready;
|
||||
if (config.heartbeatSchedulerEnabled) await heartbeat.registerDeferredReceiptWork(deliveryWork,
|
||||
() => !isIdleTaskDrainActive() && !isWarmStandby() && !heartbeatSchedulerStopped).ready;
|
||||
if (config.heartbeatSchedulerEnabled) await heartbeat.registerRunDispatchWork(deliveryWork,
|
||||
() => !isIdleTaskDrainActive() && !isWarmStandby() && !heartbeatSchedulerStopped).ready;
|
||||
if (config.heartbeatSchedulerEnabled) await heartbeat.registerScheduledRetryCancellationWork(deliveryWork,
|
||||
@@ -1891,10 +1892,7 @@ async function startServerWithDatabaseTeardown(
|
||||
// Periodically reap orphaned runs (5-min staleness threshold) and make sure
|
||||
// persisted queued work is still being driven forward.
|
||||
runPeriodicRecovery(() => heartbeat
|
||||
.reapOrphanedRuns({ staleThresholdMs: 5 * 60 * 1000, skipScheduledRetryCancellation: true, skipCapturedInstructionCommit: true, skipInstructionDirectoryCleanup: true, skipStoppedInstructionCollection: true, skipCancelledWorkspacePreservation: true, skipObsoletePolicyReviews: true, skipAutomaticCompletionReviews: true, skipUnsafeWorkspaceExports: true, skipNativePendingEffects: true, skipNativeResponsePresentation: true, skipNativeReviewResponse: true, skipNativeStatusReassessment: true, skipRetainedNativeSessionCleanup: true, skipNativeWorkspaceFinalization: true, skipNativeResultCommit: true, skipNativeRunProjection: true, skipNativeRecoveryNotices: true, skipExternalConversationSettlement: true, skipExternalAgentSettlement: true, skipNativeEnvironmentSettlement: true })
|
||||
.then(async () => {
|
||||
await heartbeat.resumeQueuedRuns();
|
||||
})
|
||||
.reapOrphanedRuns({ staleThresholdMs: 5 * 60 * 1000, skipNativeStatusWakeIntents: true, skipScheduledRetryCancellation: true, skipCapturedInstructionCommit: true, skipInstructionDirectoryCleanup: true, skipStoppedInstructionCollection: true, skipCancelledWorkspacePreservation: true, skipObsoletePolicyReviews: true, skipAutomaticCompletionReviews: true, skipUnsafeWorkspaceExports: true, skipNativePendingEffects: true, skipNativeResponsePresentation: true, skipNativeReviewResponse: true, skipNativeStatusReassessment: true, skipRetainedNativeSessionCleanup: true, skipNativeWorkspaceFinalization: true, skipNativeResultCommit: true, skipNativeRunProjection: true, skipNativeRecoveryNotices: true, skipExternalConversationSettlement: true, skipExternalAgentSettlement: true, skipNativeEnvironmentSettlement: true })
|
||||
.then(async () => {
|
||||
const scanned = await heartbeat.scanSilentActiveRuns();
|
||||
if (scanned.created > 0 || scanned.escalated > 0) {
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { notifyDeferredExecutionWork } from "../../../services/deferred-execution-notifications.js";
|
||||
import { hasRequiredWorkspaceRecovery } from "../../../services/workspace-restore-recovery-state.js";
|
||||
import { readNativeWorkspaceSyncReference } from "../../../services/native-runtime/native-workspace-sync.js";
|
||||
import { isPreDispatchExecutionWaitVerified } from "../../../services/pre-dispatch-execution-wait.js";
|
||||
@@ -996,6 +997,7 @@ export function createPostgresRunDispatchAdapter(
|
||||
recordedCommentId && authors.get(recordedCommentId) === actorId ? [recordedCommentId] : [];
|
||||
}));
|
||||
if (commentIds.some(id => !recordedCommentIds.has(id))) return null;
|
||||
await notifyDeferredExecutionWork(tx, { companyId: run.companyId, requestId: wake.id });
|
||||
const [deferred] = await tx.update(agentWakeupRequests).set({
|
||||
status: "deferred_issue_execution", runId: null, claimedAt: null, finishedAt: null, error: null,
|
||||
payload: withQueuedCommentIdsInWakePayload({ ...payload, _paperclipWakeContext: context,
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { resolveExternalTestDatabaseUrl } from "../../../__tests__/helpers/external-test-database.js";
|
||||
import { DELIVERY_QUEUES } from "../../../services/delivery-work-notifications.js";
|
||||
import { instanceSettingsService } from "../../../services/instance-settings.js";
|
||||
import { randomBytes, randomUUID } from "node:crypto";
|
||||
@@ -37,7 +38,8 @@ import type { TransactionScope } from "../application/ports.js";
|
||||
// a foreign-company row is invisible to a read, and a deferred-status
|
||||
// compare-and-set that affects no row leaves no other trace. The decision
|
||||
// branching itself is proven against plain facts in `domain/policy.test.ts`.
|
||||
const embeddedPostgresSupport = await getEmbeddedPostgresTestSupport();
|
||||
const external = resolveExternalTestDatabaseUrl("PAPERCLIP_WAKE_QUEUE_TEST_DATABASE_URL");
|
||||
const embeddedPostgresSupport = external ? { supported: true, reason: undefined } : await getEmbeddedPostgresTestSupport();
|
||||
const describeEmbeddedPostgres = embeddedPostgresSupport.supported ? describe : describe.skip;
|
||||
|
||||
if (!embeddedPostgresSupport.supported) {
|
||||
@@ -57,8 +59,8 @@ describeEmbeddedPostgres("wake-queue postgres adapter", () => {
|
||||
};
|
||||
|
||||
beforeAll(async () => {
|
||||
tempDb = await startEmbeddedPostgresTestDatabase("paperclip-wake-queue-postgres-adapter-");
|
||||
db = createDb(tempDb.connectionString);
|
||||
if (!external) tempDb = await startEmbeddedPostgresTestDatabase("paperclip-wake-queue-postgres-adapter-");
|
||||
db = createDb(external ?? tempDb!.connectionString, { maxConnections: 6 });
|
||||
}, 20_000);
|
||||
|
||||
afterEach(async () => {
|
||||
@@ -78,6 +80,7 @@ describeEmbeddedPostgres("wake-queue postgres adapter", () => {
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
await db?.$client.end({ timeout: 1 });
|
||||
await tempDb?.cleanup();
|
||||
});
|
||||
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { notifyDeferredExecutionWork } from "../../../services/deferred-execution-notifications.js";
|
||||
import { notifyRunDispatch } from "../../../services/run-dispatch-notifications.js";
|
||||
import { notifyStatusCardIssueChange } from "../../../services/status-card-notifications.js";
|
||||
import { notifyExternalConversationSettlement } from "../../../services/external-conversation-settlement-notifications.js";
|
||||
@@ -1012,6 +1013,11 @@ export function createWakeAdmissionWriter(): WakeAdmissionWriter {
|
||||
)).for("update");
|
||||
if (!prior) throw new Error("wake-queue: the deferred wake to merge into was not found for this company");
|
||||
const priorPayload = parseObject(prior.payload), issueId = readNonEmptyString(priorPayload.issueId);
|
||||
const previousComments = new Set(queuedCommentIdsFromWakePayload(priorPayload));
|
||||
const newComments = queuedCommentIdsFromWakePayload(input.mergedPayload).some(id => !previousComments.has(id));
|
||||
const newInteraction = readNonEmptyString(input.mergedPayload.interactionId);
|
||||
if (newComments || newInteraction && newInteraction !== readNonEmptyString(priorPayload.interactionId))
|
||||
await notifyDeferredExecutionWork(tx, { companyId: input.companyId, requestId: prior.id });
|
||||
const priorCommentId = readNonEmptyString(priorPayload.commentId), priorActor = readNonEmptyString(prior.requestedByActorId);
|
||||
const [priorComment] = prior.requestedByActorType === "user" && priorActor && issueId && isUuidLike(issueId) &&
|
||||
priorCommentId && isUuidLike(priorCommentId) ? await tx.select({ id: issueComments.id, body: issueComments.body }).from(issueComments).where(and(
|
||||
@@ -1083,6 +1089,7 @@ export function createWakeAdmissionWriter(): WakeAdmissionWriter {
|
||||
|
||||
async insertNewDeferredWake(scope, input) {
|
||||
const tx = requireAdmissionTx(scope, input.companyId);
|
||||
await notifyDeferredExecutionWork(tx, { companyId: input.companyId });
|
||||
await tx.insert(agentWakeupRequests).values({
|
||||
...input.durableReceipt,
|
||||
companyId: input.companyId,
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { notifyDeferredExecutionWork } from "../services/deferred-execution-notifications.js";
|
||||
import { notifyNativeRecoveryNotices } from "../services/native-recovery-notice-notifications.js";
|
||||
import { notifyNativeStatusReassessment } from "../services/native-status-reassessment-notifications.js";
|
||||
import { notifyUnsafeWorkspaceExportRecovery } from "../services/unsafe-workspace-export-notifications.js";
|
||||
@@ -16674,6 +16675,7 @@ export function issueRoutes(
|
||||
}
|
||||
// The click is durable fresh user intent, including when the message
|
||||
// predates a failed run's stop. Keep its content and original attribution.
|
||||
await notifyDeferredExecutionWork(tx, { companyId: issue.companyId, requestId: locked.wake.id });
|
||||
await tx.update(agentWakeupRequests).set({
|
||||
payload: { ...readObject(locked.wake.payload), queuedCommentInterrupt: {
|
||||
actorId: actor.actorId, requestedAt: new Date().toISOString(),
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import { notifyStatusCardIssueChange } from "./status-card-notifications.js";
|
||||
import { notifyDeferredExecutionEligibility } from "./deferred-execution-notifications.js";
|
||||
import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js";
|
||||
import { notifyNativeStatusReassessment } from "./native-status-reassessment-notifications.js";
|
||||
import { notifyUnsafeWorkspaceExportRecovery } from "./unsafe-workspace-export-notifications.js";
|
||||
@@ -744,6 +745,7 @@ export function accessService(db: Db) {
|
||||
sql`${issues.status} not in ('done', 'cancelled')`,
|
||||
);
|
||||
await notifyNativeStatusReassessment(tx, { companyId });
|
||||
await notifyDeferredExecutionEligibility(tx, { companyId });
|
||||
await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors);
|
||||
await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers);
|
||||
await notifyStatusCardIssueChange(tx, companyId);
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { notifyDeferredExecutionEligibility } from "./deferred-execution-notifications.js";
|
||||
import { notifyRunDispatch } from "./run-dispatch-notifications.js";
|
||||
import { notifyAttentionRetentionWork } from "./retention-work-notifications.js";
|
||||
import { notifySkillVersionPinChange } from "./skill-version-retention-notifications.js";
|
||||
@@ -87,6 +88,7 @@ export async function prepareAgentConfiguration(db: Db, existing: AgentConfigura
|
||||
);
|
||||
}
|
||||
await notifyRunDispatch(db, { companyId: existing.companyId });
|
||||
await notifyDeferredExecutionEligibility(db, { companyId: existing.companyId });
|
||||
if (normalizedPatch.runtimeConfig !== undefined || normalizedPatch.reportsTo !== undefined) {
|
||||
await notifyDeliveryWork(db, DELIVERY_QUEUES.agentTimers);
|
||||
}
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { notifyDeferredExecutionEligibility } from "./deferred-execution-notifications.js";
|
||||
import { notifyRunDispatch } from "./run-dispatch-notifications.js";
|
||||
import { notifyAttentionRetentionWork } from "./retention-work-notifications.js";
|
||||
import { notifyNativeReviewResponse } from "./native-review-response-notifications.js";
|
||||
@@ -43,6 +44,7 @@ export function createAgentLifecycleEffects(_hooks: BudgetServiceHooks = {}): li
|
||||
await notifyAttentionRetentionWork(tx, { companyId });
|
||||
if (before !== after) {
|
||||
await notifyRunDispatch(tx, { companyId });
|
||||
await notifyDeferredExecutionEligibility(tx, { companyId });
|
||||
await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers);
|
||||
}
|
||||
if (!isAgentStatusInvokable(before) && isAgentStatusInvokable(after)) {
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { notifyDeferredExecutionEligibility } from "./deferred-execution-notifications.js";
|
||||
import { notifyRunDispatch } from "./run-dispatch-notifications.js";
|
||||
import { notifyStatusCardIssueChange } from "./status-card-notifications.js";
|
||||
import { notifyExternalObjectRefresh } from "./external-object-refresh-notifications.js";
|
||||
@@ -291,6 +292,7 @@ export function budgetServiceInTransaction(db: Db, publications: ActivityPublica
|
||||
|
||||
if (policy.scopeType === "project") {
|
||||
await notifyRunDispatch(db, { companyId: policy.companyId });
|
||||
await notifyDeferredExecutionEligibility(db, { companyId: policy.companyId });
|
||||
await db
|
||||
.update(projects)
|
||||
.set({
|
||||
@@ -305,6 +307,7 @@ export function budgetServiceInTransaction(db: Db, publications: ActivityPublica
|
||||
const scope = await resolveScopeRecord(db, "company", policy.scopeId);
|
||||
if (scope.paused) return;
|
||||
await notifyRunDispatch(db, { companyId: policy.companyId });
|
||||
await notifyDeferredExecutionEligibility(db, { companyId: policy.companyId });
|
||||
await notifyCompanyRetentionWork(db, policy.companyId);
|
||||
await notifyStatusCardIssueChange(db, policy.companyId);
|
||||
await notifyExternalObjectRefresh(db, policy.companyId);
|
||||
@@ -354,6 +357,7 @@ export function budgetServiceInTransaction(db: Db, publications: ActivityPublica
|
||||
|
||||
if (policy.scopeType === "project") {
|
||||
await notifyRunDispatch(db, { companyId: policy.companyId });
|
||||
await notifyDeferredExecutionEligibility(db, { companyId: policy.companyId });
|
||||
await db
|
||||
.update(projects)
|
||||
.set({
|
||||
@@ -369,6 +373,7 @@ export function budgetServiceInTransaction(db: Db, publications: ActivityPublica
|
||||
const scope = await resolveScopeRecord(db, "company", policy.scopeId);
|
||||
if (scope.pauseReason !== "budget") return;
|
||||
await notifyRunDispatch(db, { companyId: policy.companyId });
|
||||
await notifyDeferredExecutionEligibility(db, { companyId: policy.companyId });
|
||||
await notifyCompanyRetentionWork(db, policy.companyId);
|
||||
await notifyStatusCardIssueChange(db, policy.companyId);
|
||||
await notifyExternalObjectRefresh(db, policy.companyId);
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { notifyDeferredExecutionWork, notifyTaskRetrySettlement } from "../deferred-execution-notifications.js";
|
||||
import { notifyNativeResponsePresentation } from "../native-response-presentation-notifications.js";
|
||||
import { notifyChatInboundWakeNotices } from "../chat-inbound-wake-notice-notifications.js";
|
||||
import { createFailedRetryNotices } from "./failed-retry-notices.js";
|
||||
@@ -1424,6 +1425,7 @@ export function createChatFailedRunRetry(db: Db, options: ChatFailedRunRetryOpti
|
||||
// Source authorization above checked every external receipt and its owner.
|
||||
// Reset only this certified unstarted batch under the task's admission lock.
|
||||
const skippedIds = source.taskRetryInput?.skippedActionIds ?? [];
|
||||
await notifyDeferredExecutionWork(tx, { companyId: input.companyId });
|
||||
await tx.update(agentWakeupRequests).set({ status: "deferred_issue_execution", runId: null, finishedAt: null,
|
||||
payload: sql`${agentWakeupRequests.payload} - 'coalescedIntoWakeupRequestId'`, updatedAt: new Date(),
|
||||
}).where(and(eq(agentWakeupRequests.companyId, input.companyId), eq(agentWakeupRequests.agentId, input.agentId),
|
||||
@@ -1784,6 +1786,9 @@ export function createChatFailedRunRetry(db: Db, options: ChatFailedRunRetryOpti
|
||||
const attemptCount = Number(candidate.result?.attemptCount ?? 0) + 1;
|
||||
const settle = async (status: string, result: Record<string, unknown>) => {
|
||||
const [current] = await db.transaction(async (tx) => {
|
||||
if (typeof claimed.payload.taskRetryIntentId === "string") await notifyTaskRetrySettlement(tx, {
|
||||
companyId: claimed.companyId, requestId: claimed.payload.taskRetryIntentId,
|
||||
});
|
||||
await notifyChatActionWork(tx, "failed_run_retry");
|
||||
if (status === "processed") await notifyNativeResponsePresentation(tx, {
|
||||
companyId: claimed.companyId, issueId: String(claimed.payload.issueId),
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { notifyDeferredExecutionWork } from "../deferred-execution-notifications.js";
|
||||
import { z } from "zod";
|
||||
import { and, asc, eq, gt, inArray, isNull, or, sql } from "drizzle-orm";
|
||||
import { agentWakeupRequests, chatActions, chatConversations, chatEndpoints, heartbeatRuns, issues, type Db } from "@paperclipai/db";
|
||||
@@ -91,6 +92,7 @@ export async function releaseUnadmittedTaskRetryInput(db: Db, input: { companyId
|
||||
)).for("update");
|
||||
if (receipt?.runId) return;
|
||||
await notifyChatInboundWakeNotices(tx, { companyId: input.companyId, issueId: issue.id });
|
||||
await notifyDeferredExecutionWork(tx, { companyId: input.companyId });
|
||||
await tx.update(agentWakeupRequests).set({ status: "deferred_issue_execution", finishedAt: null,
|
||||
payload: sql`${agentWakeupRequests.payload} - 'coalescedIntoWakeupRequestId'`, updatedAt: new Date(),
|
||||
}).where(and(eq(agentWakeupRequests.companyId, input.companyId), eq(agentWakeupRequests.status, "coalesced"),
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { notifyDeferredExecutionEligibility } from "./deferred-execution-notifications.js";
|
||||
import { notifyRunDispatch } from "./run-dispatch-notifications.js";
|
||||
import { notifyStatusCardIssueChange } from "./status-card-notifications.js";
|
||||
import { notifyGitHubCompanyChange } from "./github-connection-notifications.js";
|
||||
@@ -338,6 +339,7 @@ export function companyService(db: Db, _budgetHooks: BudgetServiceHooks = {}) {
|
||||
if (companyPatch.status !== undefined && companyPatch.status !== existing.status) {
|
||||
await notifyRunDispatch(tx, { companyId: id });
|
||||
await notifyGitHubCompanyChange(tx, id);
|
||||
await notifyDeferredExecutionEligibility(tx, { companyId: id });
|
||||
await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers);
|
||||
await notifyDeliveryWork(tx, DELIVERY_QUEUES.skillSourceSync);
|
||||
await notifyExternalObjectRefresh(tx, id);
|
||||
@@ -498,6 +500,7 @@ export function companyService(db: Db, _budgetHooks: BudgetServiceHooks = {}) {
|
||||
if (!wasAlreadyArchived) {
|
||||
await notifyRunDispatch(tx, { companyId: id });
|
||||
await notifyGitHubCompanyChange(tx, id);
|
||||
await notifyDeferredExecutionEligibility(tx, { companyId: id });
|
||||
await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers);
|
||||
await notifyDeliveryWork(tx, DELIVERY_QUEUES.skillSourceSync);
|
||||
await notifyExternalObjectRefresh(tx, id);
|
||||
|
||||
@@ -0,0 +1,34 @@
|
||||
import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js";
|
||||
|
||||
export type DeferredExecutionScope = { companyId: string; requestId?: string };
|
||||
export type DeferredReceiptRecoveryOptions = {
|
||||
companyId?: string;
|
||||
requestId?: string;
|
||||
onError?: (error: unknown) => void;
|
||||
};
|
||||
|
||||
function hint(scope: DeferredExecutionScope) {
|
||||
return JSON.stringify({ companyId: scope.companyId, requestId: scope.requestId });
|
||||
}
|
||||
|
||||
export async function notifyDeferredExecutionWork(tx: object, scope: DeferredExecutionScope) {
|
||||
await notifyDeliveryWork(tx, DELIVERY_QUEUES.deferredExecutionReceipts, hint(scope));
|
||||
}
|
||||
|
||||
export async function notifyNativeStatusWakeWork(tx: object, scope: DeferredExecutionScope) {
|
||||
await notifyDeliveryWork(tx, DELIVERY_QUEUES.nativeStatusWakeIntents, hint(scope));
|
||||
}
|
||||
|
||||
export async function notifyTaskRetrySettlement(tx: object, scope: DeferredExecutionScope) {
|
||||
await notifyDeliveryWork(tx, DELIVERY_QUEUES.taskRetryReceiptSettlement, hint(scope));
|
||||
}
|
||||
|
||||
export async function notifyDeferredExecutionEligibility(tx: object, scope?: { companyId: string }) {
|
||||
if (scope) {
|
||||
await notifyDeferredExecutionWork(tx, scope);
|
||||
await notifyNativeStatusWakeWork(tx, scope);
|
||||
} else {
|
||||
await notifyDeliveryWork(tx, DELIVERY_QUEUES.deferredExecutionReceipts, "*");
|
||||
await notifyDeliveryWork(tx, DELIVERY_QUEUES.nativeStatusWakeIntents, "*");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
import { and, eq, sql } from "drizzle-orm";
|
||||
import { agentWakeupRequests, type Db } from "@paperclipai/db";
|
||||
import { notifyDeferredExecutionWork } from "./deferred-execution-notifications.js";
|
||||
|
||||
export async function writeDeferredExecutionWait(db: Db, scope: { companyId: string; requestId: string },
|
||||
condition: Record<string, unknown>, options: { replace?: boolean } = {}) {
|
||||
return db.transaction(async tx => {
|
||||
const [current] = await tx.select({ payload: agentWakeupRequests.payload }).from(agentWakeupRequests)
|
||||
.where(and(eq(agentWakeupRequests.companyId, scope.companyId), eq(agentWakeupRequests.id, scope.requestId),
|
||||
eq(agentWakeupRequests.status, "deferred_issue_execution"))).for("update");
|
||||
if (!current) return;
|
||||
const previous = current.payload?.executionWait;
|
||||
const previousReason = previous && typeof previous === "object" && !Array.isArray(previous)
|
||||
? (previous as Record<string, unknown>).reason : undefined;
|
||||
if (!previous || previousReason !== condition.reason) await notifyDeferredExecutionWork(tx, scope);
|
||||
await tx.update(agentWakeupRequests).set({
|
||||
payload: sql`jsonb_set(coalesce(${agentWakeupRequests.payload}, '{}'::jsonb), '{executionWait}',
|
||||
${options.replace ? sql`'{}'::jsonb` : sql`coalesce(${agentWakeupRequests.payload}->'executionWait', '{}'::jsonb)`}
|
||||
|| ${JSON.stringify(condition)}::jsonb)`,
|
||||
updatedAt: new Date(),
|
||||
}).where(and(eq(agentWakeupRequests.companyId, scope.companyId), eq(agentWakeupRequests.id, scope.requestId),
|
||||
eq(agentWakeupRequests.status, "deferred_issue_execution")));
|
||||
});
|
||||
}
|
||||
@@ -2,6 +2,9 @@ import { signalDatabaseWork, subscribeDatabaseWork } from "@paperclipai/db";
|
||||
|
||||
/** Topics for existing durable queues; register intent before their writes. */
|
||||
export const DELIVERY_QUEUES = {
|
||||
deferredExecutionReceipts: "deferred-execution-receipts",
|
||||
taskRetryReceiptSettlement: "task-retry-receipt-settlement",
|
||||
nativeStatusWakeIntents: "native-status-wake-intents",
|
||||
orphanBlockers: "orphan-blockers",
|
||||
activeRecoveryActions: "active-recovery-actions",
|
||||
strandedAssignments: "stranded-assignments",
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import { notifyAttentionRetentionWork } from "./retention-work-notifications.js";
|
||||
import { notifyDeferredExecutionEligibility } from "./deferred-execution-notifications.js";
|
||||
import { notifyRunDispatch } from "./run-dispatch-notifications.js";
|
||||
import { notifyStatusCardIssueChange } from "./status-card-notifications.js";
|
||||
import { notifyRoutineExecutionStatus } from "./routine-execution-status-notifications.js";
|
||||
@@ -12,8 +13,9 @@ export async function notifyExecutionRunWork(transaction: object, scope?: Execut
|
||||
await notifyAttentionRetentionWork(transaction, scope);
|
||||
await notifyRunDispatch(transaction, scope);
|
||||
await notifyStatusCardIssueChange(transaction, scope.companyId);
|
||||
await notifyDeferredExecutionEligibility(transaction, scope);
|
||||
await notifyRoutineExecutionStatus(transaction, scope.issueId, scope.companyId);
|
||||
}
|
||||
if (scope) await notifyRoutineExecutionStatus(transaction, scope.issueId, scope.companyId);
|
||||
for (const queue of [DELIVERY_QUEUES.executionStatus, DELIVERY_QUEUES.executionFinalization,
|
||||
DELIVERY_QUEUES.executionReplacement, DELIVERY_QUEUES.executionDisposition, DELIVERY_QUEUES.issueMonitors]) {
|
||||
await notifyDeliveryWork(transaction, queue);
|
||||
@@ -31,6 +33,7 @@ export function writeExecutionControl<T>(
|
||||
if (scope) await notifyStatusCardIssueChange(transaction, scope.companyId);
|
||||
if (scope) await notifyAttentionRetentionWork(transaction, scope);
|
||||
if (scope) await notifyRoutineExecutionStatus(transaction, scope.issueId, scope.companyId);
|
||||
if (scope) await notifyDeferredExecutionEligibility(transaction, scope);
|
||||
await notifyDeliveryWork(transaction, DELIVERY_QUEUES.issueMonitors);
|
||||
for (const topic of typeof queue === "string" ? [queue] : queue) await notifyDeliveryWork(transaction, topic);
|
||||
return write(transaction as unknown as Db);
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
import { registerDeferredReceiptWork } from "./heartbeat/deferred-receipt-work.js";
|
||||
import { reconcileTaskChatRetryReceipts } from "./task-retry.js";
|
||||
import { registerIssueRecoveryWork } from "./recovery/work.js";
|
||||
import { registerRunDispatchWork } from "./heartbeat/dispatch-work.js";
|
||||
import { externalConversationSettlementWork } from "./external-conversation-settlement-work.js";
|
||||
@@ -4243,6 +4245,13 @@ export function heartbeatService(
|
||||
computeTaskDrain,
|
||||
applyTaskDrain,
|
||||
|
||||
registerDeferredReceiptWork: (coordinator: Parameters<typeof registerDeferredReceiptWork>[1], canRun: () => boolean) =>
|
||||
registerDeferredReceiptWork(db, coordinator, {
|
||||
resume: scope => heartbeatQueue.resumeDeferredExecutionReceipts(scope),
|
||||
settleTaskChat: scope => reconcileTaskChatRetryReceipts(db, { ...scope, settleOnly: true }),
|
||||
dispatchNative: dispatchPendingNativeStatusWakeups,
|
||||
schedulingSuppressed: async () => (await getSchedulingSuppression(true)).suppressed,
|
||||
}, () => canRun() && readTaskDrain(new Date()) === null),
|
||||
registerIssueStateRecoveryWork: (coordinator: Parameters<typeof registerIssueRecoveryWork>[1], canRun: () => boolean) =>
|
||||
registerIssueRecoveryWork(db, coordinator, {
|
||||
stranded: scope => reconcileStrandedAssignedIssues({ ...scope, skipExternalConversationSettlement: true, skipRelatedRecovery: true }),
|
||||
|
||||
@@ -0,0 +1,118 @@
|
||||
import { and, asc, eq, gt, inArray, isNull, or, sql } from "drizzle-orm";
|
||||
import { agentWakeupRequests, type Db } from "@paperclipai/db";
|
||||
import type { createDeliveryWorkCoordinator } from "../delivery-work-coordinator.js";
|
||||
import { DELIVERY_QUEUES } from "../delivery-work-notifications.js";
|
||||
import type { DeferredExecutionScope, DeferredReceiptRecoveryOptions } from "../deferred-execution-notifications.js";
|
||||
import { registerScopedDeadlineWork } from "../scoped-deadline-work.js";
|
||||
|
||||
type Entry = { id: string; companyId: string; dueAt: number };
|
||||
type NativeIntentOptions = DeferredExecutionScope & {
|
||||
onDeferredDeadline(at: number): void;
|
||||
onError(error: unknown): void;
|
||||
};
|
||||
|
||||
export function registerDeferredReceiptWork(
|
||||
db: Db,
|
||||
coordinator: ReturnType<typeof createDeliveryWorkCoordinator>,
|
||||
owner: {
|
||||
resume(scope: DeferredReceiptRecoveryOptions): Promise<unknown>;
|
||||
settleTaskChat(scope: DeferredExecutionScope): Promise<unknown>;
|
||||
dispatchNative(scope: NativeIntentOptions): Promise<unknown>;
|
||||
schedulingSuppressed(): Promise<boolean>;
|
||||
},
|
||||
canRun: () => boolean,
|
||||
) {
|
||||
function register(kind: "deferred" | "task_chat" | "native") {
|
||||
const cursors = new Map<string, string>();
|
||||
let enabled = false;
|
||||
const queue = kind === "deferred" ? DELIVERY_QUEUES.deferredExecutionReceipts
|
||||
: kind === "task_chat" ? DELIVERY_QUEUES.taskRetryReceiptSettlement : DELIVERY_QUEUES.nativeStatusWakeIntents;
|
||||
return registerScopedDeadlineWork<Entry, DeferredExecutionScope>(coordinator, {
|
||||
queue,
|
||||
reconstructionIntervalMs: 30_000,
|
||||
beforePass: async () => { enabled = !(await owner.schedulingSuppressed()); },
|
||||
decodeScope: hint => hint === "*" ? undefined : JSON.parse(hint),
|
||||
matches: (entry, scope) => !scope || entry.companyId === scope.companyId && (!scope.requestId || entry.id === scope.requestId),
|
||||
load: async scope => {
|
||||
const key = scope ? JSON.stringify({ companyId: scope.companyId, requestId: scope.requestId }) : "*";
|
||||
if (!enabled) { cursors.delete(key); return []; }
|
||||
const cursor = cursors.get(key);
|
||||
const rows = await db.select({ id: agentWakeupRequests.id, companyId: agentWakeupRequests.companyId,
|
||||
status: agentWakeupRequests.status, updatedAt: agentWakeupRequests.updatedAt,
|
||||
claimedAt: agentWakeupRequests.claimedAt, error: agentWakeupRequests.error,
|
||||
delayed: sql<boolean>`${agentWakeupRequests.payload}->'queuedCommentInterrupt' is not null
|
||||
or ${agentWakeupRequests.payload}->'executionWait' is not null`,
|
||||
interruptedLegacy: sql<boolean>`exists (select 1 from heartbeat_runs r
|
||||
where r.company_id = ${agentWakeupRequests.companyId} and r.agent_id = ${agentWakeupRequests.agentId}
|
||||
and r.runtime_mode = 'legacy' and r.status = 'cancelled'
|
||||
and r.result_json->>'queuedCommentInterruptQueueId' = ${agentWakeupRequests.id}::text)`,
|
||||
}).from(agentWakeupRequests).where(and(
|
||||
scope ? eq(agentWakeupRequests.companyId, scope.companyId) : undefined,
|
||||
scope?.requestId ? eq(agentWakeupRequests.id, scope.requestId) : undefined,
|
||||
cursor ? gt(agentWakeupRequests.id, cursor) : undefined,
|
||||
kind === "native" ? and(
|
||||
eq(agentWakeupRequests.requestedByActorType, "system"),
|
||||
eq(agentWakeupRequests.requestedByActorId, "native-status-committer"),
|
||||
inArray(agentWakeupRequests.status, ["queued", "claimed"]), isNull(agentWakeupRequests.runId),
|
||||
) : and(eq(agentWakeupRequests.status, "deferred_issue_execution"),
|
||||
kind === "task_chat" ? and(eq(agentWakeupRequests.reason, "task_retry"),
|
||||
sql`${agentWakeupRequests.payload}#>>'{taskRetry,chatActionId}' is not null`) : or(
|
||||
eq(agentWakeupRequests.reason, "task_retry"),
|
||||
sql`${agentWakeupRequests.payload}->'queuedCommentInterrupt' is not null`,
|
||||
sql`${agentWakeupRequests.payload}->'executionWait' is not null`,
|
||||
sql`jsonb_typeof(${agentWakeupRequests.payload}#>'{_paperclipWakeContext,wakeCommentIds}') = 'array'
|
||||
and ${agentWakeupRequests.payload}#>'{_paperclipWakeContext,wakeCommentIds}' <> '[]'::jsonb`,
|
||||
sql`${agentWakeupRequests.payload}->>'mutation' = 'interaction'`,
|
||||
sql`exists (select 1 from heartbeat_runs r where r.company_id = ${agentWakeupRequests.companyId}
|
||||
and r.runtime_mode = 'legacy' and r.status = 'cancelled'
|
||||
and r.result_json->>'queuedCommentInterruptQueueId' = ${agentWakeupRequests.id}::text)`,
|
||||
)),
|
||||
)).orderBy(asc(agentWakeupRequests.id)).limit(100);
|
||||
const complete = rows.length < 100;
|
||||
if (complete) cursors.delete(key); else cursors.set(key, rows.at(-1)!.id);
|
||||
return { rows: rows.map(row => ({ id: row.id, companyId: row.companyId,
|
||||
dueAt: kind === "native" && row.status === "claimed" && row.claimedAt ? row.claimedAt.getTime() + 60_000
|
||||
: kind === "native" && row.error ? row.updatedAt.getTime() + 30_000
|
||||
: kind === "deferred" && row.delayed && !row.interruptedLegacy ? row.updatedAt.getTime() + 30_000
|
||||
: Date.now(),
|
||||
})), complete };
|
||||
},
|
||||
check: async entry => {
|
||||
if (!enabled) return null;
|
||||
if (kind !== "task_chat") {
|
||||
const [current] = await db.select({ status: agentWakeupRequests.status, payload: agentWakeupRequests.payload,
|
||||
updatedAt: agentWakeupRequests.updatedAt, claimedAt: agentWakeupRequests.claimedAt, error: agentWakeupRequests.error,
|
||||
interruptedLegacy: sql<boolean>`exists (select 1 from heartbeat_runs r
|
||||
where r.company_id = ${agentWakeupRequests.companyId} and r.agent_id = ${agentWakeupRequests.agentId}
|
||||
and r.runtime_mode = 'legacy' and r.status = 'cancelled'
|
||||
and r.result_json->>'queuedCommentInterruptQueueId' = ${agentWakeupRequests.id}::text)`,
|
||||
}).from(agentWakeupRequests).where(and(eq(agentWakeupRequests.companyId, entry.companyId),
|
||||
eq(agentWakeupRequests.id, entry.id)));
|
||||
if (!current) return null;
|
||||
const due = kind === "native" && current.status === "claimed" && current.claimedAt
|
||||
? current.claimedAt.getTime() + 60_000
|
||||
: kind === "native" && current.status === "queued" && current.error ? current.updatedAt.getTime() + 30_000
|
||||
: kind === "deferred" && current.status === "deferred_issue_execution" && !current.interruptedLegacy
|
||||
&& (current.payload?.queuedCommentInterrupt || current.payload?.executionWait)
|
||||
? current.updatedAt.getTime() + 30_000 : null;
|
||||
if (due !== null && due > Date.now()) return { ...entry, dueAt: due };
|
||||
}
|
||||
const errors: unknown[] = [];
|
||||
let next: number | null = null;
|
||||
const scope = { companyId: entry.companyId, requestId: entry.id };
|
||||
if (kind === "native") await owner.dispatchNative({ ...scope,
|
||||
onDeferredDeadline: at => { next = next === null ? at : Math.min(next, at); },
|
||||
onError: error => { errors.push(error); },
|
||||
});
|
||||
else if (kind === "task_chat") await owner.settleTaskChat(scope);
|
||||
else await owner.resume({ ...scope, onError: error => { errors.push(error); } });
|
||||
if (errors.length) throw new AggregateError(errors, `Deferred receipt ${entry.id} could not settle`);
|
||||
return next === null ? null : { ...entry, dueAt: next };
|
||||
},
|
||||
}, canRun);
|
||||
}
|
||||
const workers = [register("deferred"), register("task_chat"), register("native")];
|
||||
return { ready: Promise.all(workers.map(worker => worker.ready)),
|
||||
wake() { for (const worker of workers) worker.invalidate(); },
|
||||
async stop() { await Promise.all(workers.map(worker => worker.stop())); } };
|
||||
}
|
||||
@@ -1,3 +1,6 @@
|
||||
import { writeDeferredExecutionWait } from "../deferred-execution-wait.js";
|
||||
import { notifyDeferredExecutionWork } from "../deferred-execution-notifications.js";
|
||||
import type { DeferredReceiptRecoveryOptions } from "../deferred-execution-notifications.js";
|
||||
import { notifyRunDispatch } from "../run-dispatch-notifications.js";
|
||||
import { notifyStatusCardIssueChange } from "../status-card-notifications.js";
|
||||
import { notifyRoutineExecutionStatus } from "../routine-execution-status-notifications.js";
|
||||
@@ -224,7 +227,7 @@ export interface HeartbeatQueueDependencies extends Pick<HeartbeatRetryDependenc
|
||||
publishRunLifecyclePluginEvent: (run: HeartbeatRun) => void;
|
||||
cancelRunInternal: (runId: string, reason?: string, options?: { errorCode?: string }) => Promise<unknown>;
|
||||
cancelActiveForAgentInternal: (agentId: string, reason: string) => Promise<unknown>;
|
||||
resumeExecutionWaitComments: () => Promise<unknown>;
|
||||
resumeExecutionWaitComments: (scope?: DeferredReceiptRecoveryOptions) => Promise<unknown>;
|
||||
resumeQueuedCommentInterrupt: (companyId: string, queueId: string) => Promise<unknown>;
|
||||
resumeSavedLegacyComments: (companyId: string, queueId: string) => Promise<unknown>;
|
||||
formatIssueIdentifierLink: (identifier: string | null, fallback: string) => string;
|
||||
@@ -1854,13 +1857,15 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
|
||||
return cancelled;
|
||||
}
|
||||
|
||||
async function resumeQueuedRuns() {
|
||||
async function resumeDeferredExecutionReceipts(scope: DeferredReceiptRecoveryOptions = {}) {
|
||||
if ((await getSchedulingSuppression()).suppressed) return;
|
||||
await resumeExecutionWaitComments();
|
||||
await resumeExecutionWaitComments(scope);
|
||||
const cutoff = await getWorktreeExecutionCutoff();
|
||||
const pendingInterrupts = await db.select({ id: agentWakeupRequests.id, companyId: agentWakeupRequests.companyId })
|
||||
.from(agentWakeupRequests).innerJoin(companies, eq(companies.id, agentWakeupRequests.companyId))
|
||||
.where(and(eq(agentWakeupRequests.status, "deferred_issue_execution"),
|
||||
.where(and(scope.companyId ? eq(agentWakeupRequests.companyId, scope.companyId) : undefined,
|
||||
scope.requestId ? eq(agentWakeupRequests.id, scope.requestId) : undefined,
|
||||
eq(agentWakeupRequests.status, "deferred_issue_execution"),
|
||||
eq(companies.status, "active"),
|
||||
sql`${agentWakeupRequests.payload}->'queuedCommentInterrupt' is not null`,
|
||||
lte(agentWakeupRequests.updatedAt, new Date(Date.now() - 30_000)),
|
||||
@@ -1871,6 +1876,7 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
|
||||
eq(agentWakeupRequests.id, wake.id), eq(agentWakeupRequests.status, "deferred_issue_execution"),
|
||||
));
|
||||
await resumeQueuedCommentInterrupt(wake.companyId, wake.id).catch(err => {
|
||||
scope.onError?.(err);
|
||||
logger.warn({ err, queueId: wake.id }, "failed to resume interrupted comment queue");
|
||||
});
|
||||
}
|
||||
@@ -1884,7 +1890,9 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
|
||||
or(eq(issues.assigneeAgentId, agentWakeupRequests.agentId),
|
||||
sql`${agentWakeupRequests.payload} #>> '{_paperclipWakeContext,wakeReason}' = 'issue_comment_mentioned'`)))
|
||||
.innerJoin(companies, and(eq(companies.id, issues.companyId), eq(companies.status, "active")))
|
||||
.where(and(eq(agentWakeupRequests.status, "deferred_issue_execution"),
|
||||
.where(and(scope.companyId ? eq(agentWakeupRequests.companyId, scope.companyId) : undefined,
|
||||
scope.requestId ? eq(agentWakeupRequests.id, scope.requestId) : undefined,
|
||||
eq(agentWakeupRequests.status, "deferred_issue_execution"),
|
||||
isNull(issues.executionRunId),
|
||||
or(and(
|
||||
sql`jsonb_typeof(${agentWakeupRequests.payload} #> '{_paperclipWakeContext,wakeCommentIds}') = 'array'`,
|
||||
@@ -1923,7 +1931,8 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
|
||||
if (latest.runtimeMode !== "legacy" && latest.runtimeMode !== "native") continue;
|
||||
if (wake.idempotencyKey?.startsWith("chat-inbound:")) {
|
||||
await resumeDeferredChatWake(db, { companyId: wake.companyId, agentId: wake.agentId,
|
||||
issueId: String(wake.payload?.issueId), requestId: wake.id }).catch(() => {
|
||||
issueId: String(wake.payload?.issueId), requestId: wake.id }).catch(err => {
|
||||
scope.onError?.(err);
|
||||
logger.warn({ requestId: wake.id }, "saved chat source could not resume after recovery");
|
||||
});
|
||||
continue;
|
||||
@@ -1935,6 +1944,7 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
|
||||
latest.errorCode === "execution_reconciliation_required";
|
||||
if ((latest.status !== "cancelled" || cancelledAdmission) && await getExecutionBlocker(db, wake.companyId, String(wake.payload?.issueId))) {
|
||||
await resumeSavedLegacyComments(wake.companyId, wake.id).catch(err => {
|
||||
scope.onError?.(err);
|
||||
logger.warn({ err, queueId: wake.id }, "failed to deliver saved legacy comment after recovery stopped");
|
||||
});
|
||||
}
|
||||
@@ -1942,6 +1952,7 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
|
||||
await releaseIssueExecutionAndPromote(latest, { suppressImmediateRecovery: true, resumeStrandedQueue: true }).then(() => {
|
||||
if (latest.runtimeMode === "native") releasedNativeSources.add(releaseSource);
|
||||
}).catch(err => {
|
||||
scope.onError?.(err);
|
||||
logger.warn({ err, queueId: wake.id }, "failed to promote stranded comments");
|
||||
});
|
||||
}
|
||||
@@ -1959,6 +1970,8 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
|
||||
))
|
||||
.innerJoin(companies, eq(companies.id, heartbeatRuns.companyId))
|
||||
.where(and(
|
||||
scope.companyId ? eq(agentWakeupRequests.companyId, scope.companyId) : undefined,
|
||||
scope.requestId ? eq(agentWakeupRequests.id, scope.requestId) : undefined,
|
||||
eq(agentWakeupRequests.status, "deferred_issue_execution"),
|
||||
eq(heartbeatRuns.status, "cancelled"),
|
||||
eq(heartbeatRuns.runtimeMode, "legacy"),
|
||||
@@ -1967,10 +1980,15 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
|
||||
));
|
||||
for (const run of interruptedQueues) {
|
||||
await releaseIssueExecutionAndPromote(run, { suppressImmediateRecovery: true }).catch((err) => {
|
||||
scope.onError?.(err);
|
||||
logger.error({ err, runId: run.id }, "failed to retry interrupted comment queue");
|
||||
});
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
async function resumeQueuedRuns() {
|
||||
await resumeDeferredExecutionReceipts();
|
||||
await resumeQueuedRunsAfterDrain();
|
||||
}
|
||||
|
||||
@@ -2509,17 +2527,11 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
|
||||
// its one durable receipt ID on a skipped wake.
|
||||
if (caseRequest) return { created: false };
|
||||
if (executionWaitRequestId) {
|
||||
await db.update(agentWakeupRequests).set({
|
||||
payload: sql`jsonb_set(coalesce(${agentWakeupRequests.payload}, '{}'::jsonb), '{executionWait}',
|
||||
coalesce(${agentWakeupRequests.payload}->'executionWait', '{}'::jsonb) || ${JSON.stringify({
|
||||
reason: skipReason, message: patch.error ?? (skipReason === "issue_tree_hold_active"
|
||||
? "This task is paused. Resume it to send your saved message."
|
||||
: "Waiting for task execution to be enabled. Your message is saved."),
|
||||
})}::jsonb)`,
|
||||
updatedAt: new Date(),
|
||||
}).where(and(eq(agentWakeupRequests.id, executionWaitRequestId),
|
||||
eq(agentWakeupRequests.companyId, agent.companyId), eq(agentWakeupRequests.agentId, agentId),
|
||||
eq(agentWakeupRequests.status, "deferred_issue_execution")));
|
||||
await writeDeferredExecutionWait(db, { companyId: agent.companyId, requestId: executionWaitRequestId }, {
|
||||
reason: skipReason, message: patch.error ?? (skipReason === "issue_tree_hold_active"
|
||||
? "This task is paused. Resume it to send your saved message."
|
||||
: "Waiting for task execution to be enabled. Your message is saved."),
|
||||
});
|
||||
return { created: false };
|
||||
}
|
||||
const request = {
|
||||
@@ -3407,15 +3419,14 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
|
||||
opts.issueStateGuard?.onExecutionDeferred?.(executionBlocker.cause);
|
||||
const condition = { recoveryActionId: executionBlocker.recoveryActionId, ...continuationWait };
|
||||
if (executionWaitRequestId) {
|
||||
await tx.update(agentWakeupRequests).set({
|
||||
payload: sql`jsonb_set(coalesce(${agentWakeupRequests.payload}, '{}'::jsonb), '{executionWait}', ${JSON.stringify(condition)}::jsonb)`,
|
||||
updatedAt: new Date(),
|
||||
}).where(eq(agentWakeupRequests.id, executionWaitRequestId));
|
||||
await writeDeferredExecutionWait(tx as unknown as Db, { companyId: agent.companyId,
|
||||
requestId: executionWaitRequestId }, condition, { replace: true });
|
||||
return { kind: "deferred" as const, admitted: true };
|
||||
}
|
||||
if (durableRequest || wakeCommentId ||
|
||||
hasInteractionContinuationWakeContext(enrichedContextSnapshot) ||
|
||||
readNonEmptyString(enrichedContextSnapshot.nativeStatusWakeIntentId)) {
|
||||
await notifyDeferredExecutionWork(tx, { companyId: agent.companyId });
|
||||
await tx.insert(agentWakeupRequests).values({
|
||||
...durableReceiptFields,
|
||||
companyId: agent.companyId, agentId, source, triggerDetail, reason,
|
||||
@@ -4035,6 +4046,7 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
|
||||
}));
|
||||
}
|
||||
if (followUp) {
|
||||
if (!followUp.consumed) await notifyDeferredExecutionWork(tx, { companyId: agent.companyId });
|
||||
const assignmentId = followUp.assignmentId;
|
||||
const pending = followUp.consumed ? undefined : pendingFollowUps.find(wake =>
|
||||
wake.payload?.[followUpPayloadKey] === assignmentId &&
|
||||
@@ -4773,6 +4785,9 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
|
||||
companyId?: string;
|
||||
limit?: number;
|
||||
staleClaimMs?: number;
|
||||
requestId?: string;
|
||||
onDeferredDeadline?: (at: number) => void;
|
||||
onError?: (error: unknown) => void;
|
||||
} = {},
|
||||
) {
|
||||
const now = new Date();
|
||||
@@ -4785,6 +4800,7 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
|
||||
input.companyId
|
||||
? eq(agentWakeupRequests.companyId, input.companyId)
|
||||
: undefined,
|
||||
input.requestId ? eq(agentWakeupRequests.id, input.requestId) : undefined,
|
||||
eq(agentWakeupRequests.requestedByActorType, "system"),
|
||||
eq(agentWakeupRequests.requestedByActorId, "native-status-committer"),
|
||||
inArray(agentWakeupRequests.status, ["queued", "claimed"]),
|
||||
@@ -4861,6 +4877,7 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
|
||||
candidate.claimedAt &&
|
||||
now.getTime() - candidate.claimedAt.getTime() < staleClaimMs
|
||||
) {
|
||||
input.onDeferredDeadline?.(candidate.claimedAt.getTime() + staleClaimMs);
|
||||
deferred += 1;
|
||||
continue;
|
||||
}
|
||||
@@ -5035,6 +5052,7 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
|
||||
if (wakeRun) dispatched += 1;
|
||||
else deferred += 1;
|
||||
} catch (error) {
|
||||
input.onError?.(error);
|
||||
await db
|
||||
.update(agentWakeupRequests)
|
||||
.set({
|
||||
@@ -5078,6 +5096,7 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
|
||||
withChatControlRecoveryGate,
|
||||
trackWakeup,
|
||||
resumeQueuedRuns,
|
||||
resumeDeferredExecutionReceipts,
|
||||
resumeQueuedRunsAfterDrain,
|
||||
parseHeartbeatPolicy,
|
||||
nextTimerHeartbeatAt,
|
||||
|
||||
@@ -1979,7 +1979,7 @@ export function createHeartbeatRecovery(db: Db, dependencies: HeartbeatRecoveryD
|
||||
|
||||
let settledProcessesDiscovered = false;
|
||||
|
||||
async function reapOrphanedRuns(opts?: { staleThresholdMs?: number; skipScheduledRetryCancellation?: boolean; skipCapturedInstructionCommit?: boolean; skipInstructionDirectoryCleanup?: boolean; skipStoppedInstructionCollection?: boolean; skipCancelledWorkspacePreservation?: boolean; skipObsoletePolicyReviews?: boolean; skipAutomaticCompletionReviews?: boolean; skipUnsafeWorkspaceExports?: boolean; skipNativePendingEffects?: boolean; skipNativeResponsePresentation?: boolean; skipNativeReviewResponse?: boolean; skipNativeStatusReassessment?: boolean; skipRetainedNativeSessionCleanup?: boolean; skipNativeWorkspaceFinalization?: boolean; skipNativeResultCommit?: boolean; skipNativeRunProjection?: boolean; skipNativeRecoveryNotices?: boolean; skipExternalConversationSettlement?: boolean; skipExternalAgentSettlement?: boolean; skipNativeEnvironmentSettlement?: boolean }) {
|
||||
async function reapOrphanedRuns(opts?: { staleThresholdMs?: number; skipNativeStatusWakeIntents?: boolean; skipScheduledRetryCancellation?: boolean; skipCapturedInstructionCommit?: boolean; skipInstructionDirectoryCleanup?: boolean; skipStoppedInstructionCollection?: boolean; skipCancelledWorkspacePreservation?: boolean; skipObsoletePolicyReviews?: boolean; skipAutomaticCompletionReviews?: boolean; skipUnsafeWorkspaceExports?: boolean; skipNativePendingEffects?: boolean; skipNativeResponsePresentation?: boolean; skipNativeReviewResponse?: boolean; skipNativeStatusReassessment?: boolean; skipRetainedNativeSessionCleanup?: boolean; skipNativeWorkspaceFinalization?: boolean; skipNativeResultCommit?: boolean; skipNativeRunProjection?: boolean; skipNativeRecoveryNotices?: boolean; skipExternalConversationSettlement?: boolean; skipExternalAgentSettlement?: boolean; skipNativeEnvironmentSettlement?: boolean }) {
|
||||
if (!opts?.skipCancelledWorkspacePreservation) await preserveCancelledNativeWorkspaces(db);
|
||||
const staleThresholdMs = opts?.staleThresholdMs ?? 0;
|
||||
const now = new Date();
|
||||
@@ -2023,7 +2023,7 @@ export function createHeartbeatRecovery(db: Db, dependencies: HeartbeatRecoveryD
|
||||
});
|
||||
if (!opts?.skipRetainedNativeSessionCleanup) scheduleRetainedNativeSessionCleanup();
|
||||
await reconcileRetainedWorkspaces(db, environmentRuntime, resumeRemoteStopComments);
|
||||
await dispatchPendingNativeStatusWakeups().catch((error) => {
|
||||
if (!opts?.skipNativeStatusWakeIntents) await dispatchPendingNativeStatusWakeups().catch((error) => {
|
||||
logger.warn(
|
||||
{ err: error },
|
||||
"failed to dispatch persisted native status wake intents before orphan reaping",
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
import { writeDeferredExecutionWait } from "../deferred-execution-wait.js";
|
||||
import type { DeferredReceiptRecoveryOptions } from "../deferred-execution-notifications.js";
|
||||
import { observeStoppedWorkspaceRecovery } from "../workspace-recovery.js";
|
||||
import { notifyScheduledRetryCancellation } from "../scheduled-retry-cancellation-notifications.js";
|
||||
import { notifyChatInboundWakeNotices } from "../chat-inbound-wake-notice-notifications.js";
|
||||
@@ -616,12 +618,7 @@ export function createHeartbeatRunControl(db: Db, dependencies: HeartbeatRunCont
|
||||
onBlocked: (reason, message) => { wait = { reason, message }; },
|
||||
});
|
||||
if (!admitted) {
|
||||
await db.update(agentWakeupRequests).set({
|
||||
payload: sql`jsonb_set(coalesce(${agentWakeupRequests.payload}, '{}'::jsonb), '{executionWait}',
|
||||
coalesce(${agentWakeupRequests.payload}->'executionWait', '{}'::jsonb) || ${JSON.stringify(wait)}::jsonb)`,
|
||||
updatedAt: new Date(),
|
||||
}).where(and(eq(agentWakeupRequests.id, wake.id), eq(agentWakeupRequests.companyId, run.companyId),
|
||||
eq(agentWakeupRequests.status, "deferred_issue_execution")));
|
||||
await writeDeferredExecutionWait(db, { companyId: run.companyId, requestId: wake.id }, wait);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
@@ -777,21 +774,25 @@ export function createHeartbeatRunControl(db: Db, dependencies: HeartbeatRunCont
|
||||
idempotencyKey: `task-retry-dispatch:${wake.id}` }, wake.id);
|
||||
}
|
||||
|
||||
async function resumeExecutionWaitComments() {
|
||||
async function resumeExecutionWaitComments(scope: DeferredReceiptRecoveryOptions = {}) {
|
||||
if ((await getSchedulingSuppression()).suppressed) return;
|
||||
await reconcileTaskChatRetryReceipts(db);
|
||||
await reconcileTaskChatRetryReceipts(db, scope);
|
||||
const retries = await db.select({ id: agentWakeupRequests.id, companyId: agentWakeupRequests.companyId })
|
||||
.from(agentWakeupRequests).innerJoin(issues, and(eq(issues.companyId, agentWakeupRequests.companyId),
|
||||
sql`${issues.id}::text = ${agentWakeupRequests.payload}->>'issueId'`,
|
||||
eq(issues.assigneeAgentId, agentWakeupRequests.agentId)))
|
||||
.innerJoin(companies, and(eq(companies.id, issues.companyId), eq(companies.status, "active")))
|
||||
.where(and(eq(agentWakeupRequests.reason, "task_retry"), eq(agentWakeupRequests.status, "deferred_issue_execution"),
|
||||
.where(and(scope.companyId ? eq(agentWakeupRequests.companyId, scope.companyId) : undefined,
|
||||
scope.requestId ? eq(agentWakeupRequests.id, scope.requestId) : undefined,
|
||||
eq(agentWakeupRequests.reason, "task_retry"), eq(agentWakeupRequests.status, "deferred_issue_execution"),
|
||||
sql`${agentWakeupRequests.payload}->'taskRetry'->>'chatActionId' is null`,
|
||||
notInArray(issues.status, ["done", "cancelled"])))
|
||||
.orderBy(asc(agentWakeupRequests.updatedAt)).limit(50);
|
||||
for (const retry of retries) {
|
||||
await db.update(agentWakeupRequests).set({ updatedAt: new Date() }).where(eq(agentWakeupRequests.id, retry.id));
|
||||
await resumeTaskRetry(retry.companyId, retry.id).catch(err => logger.warn({ err, requestId: retry.id }, "Task retry dispatch deferred"));
|
||||
await resumeTaskRetry(retry.companyId, retry.id).catch(err => {
|
||||
scope.onError?.(err); logger.warn({ err, requestId: retry.id }, "Task retry dispatch deferred");
|
||||
});
|
||||
}
|
||||
const waits = await db.select({ wake: agentWakeupRequests })
|
||||
.from(agentWakeupRequests)
|
||||
@@ -799,7 +800,9 @@ export function createHeartbeatRunControl(db: Db, dependencies: HeartbeatRunCont
|
||||
sql`${issues.id}::text = ${agentWakeupRequests.payload}->>'issueId'`,
|
||||
eq(issues.assigneeAgentId, agentWakeupRequests.agentId)))
|
||||
.innerJoin(companies, and(eq(companies.id, issues.companyId), eq(companies.status, "active")))
|
||||
.where(and(exists(db.select({ id: issueRecoveryActions.id }).from(issueRecoveryActions).where(and(
|
||||
.where(and(scope.companyId ? eq(agentWakeupRequests.companyId, scope.companyId) : undefined,
|
||||
scope.requestId ? eq(agentWakeupRequests.id, scope.requestId) : undefined,
|
||||
exists(db.select({ id: issueRecoveryActions.id }).from(issueRecoveryActions).where(and(
|
||||
eq(issueRecoveryActions.companyId, issues.companyId), eq(issueRecoveryActions.sourceIssueId, issues.id),
|
||||
executionBlockerPredicate(),
|
||||
))), eq(agentWakeupRequests.status, "deferred_issue_execution"),
|
||||
@@ -831,11 +834,13 @@ export function createHeartbeatRunControl(db: Db, dependencies: HeartbeatRunCont
|
||||
if (!run || run.companyId !== wake.companyId || run.agentId !== wake.agentId) continue;
|
||||
if (canContinueCancelledRun(run)) {
|
||||
await resumeSavedComments(wake.companyId, wake.id).catch(err => {
|
||||
scope.onError?.(err);
|
||||
logger.warn({ err, runId: run.id }, "failed to resume saved input after provider cancellation");
|
||||
});
|
||||
continue;
|
||||
}
|
||||
await resumeRemoteStopComments(run, wake.id).catch(err => {
|
||||
scope.onError?.(err);
|
||||
logger.warn({ err, runId: run.id }, "failed to resume saved execution-wait message");
|
||||
});
|
||||
}
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { notifyDeferredExecutionEligibility } from "./deferred-execution-notifications.js";
|
||||
import { notifyRunDispatch } from "./run-dispatch-notifications.js";
|
||||
import { notifyStatusCardEvaluation } from "./status-card-notifications.js";
|
||||
import { notifySkillVersionRetentionReconstruction } from "./skill-version-retention-notifications.js";
|
||||
@@ -636,6 +637,7 @@ export function instanceSettingsService(db: Db, options: InstanceSettingsService
|
||||
if (patch.enableWorktreeRunExecution !== undefined) await notifyScheduledRetryCancellation(tx);
|
||||
if (patch.enableWorktreeRunExecution !== undefined) {
|
||||
await notifyRunDispatch(tx);
|
||||
await notifyDeferredExecutionEligibility(tx);
|
||||
await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers);
|
||||
await notifyExternalAgentSettlement(tx);
|
||||
await notifyInstructionCommit(tx);
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import { notifyStatusCardIssueChange } from "./status-card-notifications.js";
|
||||
import { notifyRoutineExecutionStatus } from "./routine-execution-status-notifications.js";
|
||||
import { notifyDeferredExecutionEligibility } from "./deferred-execution-notifications.js";
|
||||
import { notifyIssueRecovery } from "./issue-recovery-notifications.js";
|
||||
import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js";
|
||||
import { notifyNativeStatusReassessment } from "./native-status-reassessment-notifications.js";
|
||||
@@ -762,6 +763,7 @@ export function issueTreeControlService(db: Db) {
|
||||
|
||||
const { hold: resumeHold } = await db.transaction(async (tx) => {
|
||||
await notifyIssueRecovery(tx, { companyId });
|
||||
await notifyDeferredExecutionEligibility(tx, { companyId });
|
||||
const [createdHold] = await tx
|
||||
.insert(issueTreeHolds)
|
||||
.values({
|
||||
@@ -841,6 +843,7 @@ export function issueTreeControlService(db: Db) {
|
||||
|
||||
const { hold, members } = await db.transaction(async (tx) => {
|
||||
await notifyIssueRecovery(tx, { companyId });
|
||||
await notifyDeferredExecutionEligibility(tx, { companyId });
|
||||
const [createdHold] = await tx
|
||||
.insert(issueTreeHolds)
|
||||
.values({
|
||||
@@ -1005,6 +1008,7 @@ export function issueTreeControlService(db: Db) {
|
||||
const now = new Date();
|
||||
const releasedCancelHoldIds = activeCancelHolds.map((hold) => hold.id);
|
||||
const updatedIssues = await db.transaction(async (tx) => {
|
||||
await notifyDeferredExecutionEligibility(tx, { companyId });
|
||||
await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors);
|
||||
await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers);
|
||||
const restored: TreeStatusUpdateResult["updatedIssues"] = [];
|
||||
@@ -1186,6 +1190,7 @@ export function issueTreeControlService(db: Db) {
|
||||
|
||||
const [updated] = await db.transaction(async tx => {
|
||||
await notifyIssueRecovery(tx, { companyId });
|
||||
await notifyDeferredExecutionEligibility(tx, { companyId });
|
||||
return tx
|
||||
.update(issueTreeHolds)
|
||||
.set({
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { notifyDeferredExecutionEligibility } from "./deferred-execution-notifications.js";
|
||||
import { notifyIssueRecovery } from "./issue-recovery-notifications.js";
|
||||
import { notifyRunDispatch } from "./run-dispatch-notifications.js";
|
||||
import { notifyStatusCardIssueChange } from "./status-card-notifications.js";
|
||||
@@ -3857,6 +3858,10 @@ async function readIssueBlockerAttentionBatch(
|
||||
|
||||
const nodesById = new Map<string, IssueBlockerAttentionNode>();
|
||||
const edgesByIssueId = new Map<string, IssueBlockerAttentionEdge[]>();
|
||||
// Returned rows are already bounded by the residual fetch budget. Keep their
|
||||
// positive live-work evidence even when they cannot enter the traversal graph.
|
||||
const observedNodes = new Map<string, IssueBlockerAttentionNode>();
|
||||
const observedChildrenByParent = new Map<string, Set<string>>();
|
||||
const admittedRoots = roots.slice(0, BLOCKER_ATTENTION_MAX_NODES);
|
||||
for (const root of admittedRoots) nodesById.set(root.id, { ...root });
|
||||
|
||||
@@ -3953,6 +3958,10 @@ async function readIssueBlockerAttentionBatch(
|
||||
);
|
||||
for (const row of [...unresolvedExplicitBlockerRows, ...childRows]) {
|
||||
if (!row.issueId) continue;
|
||||
observedNodes.set(row.blockerIssueId, row);
|
||||
const observed = observedChildrenByParent.get(row.issueId) ?? new Set<string>();
|
||||
observed.add(row.blockerIssueId);
|
||||
observedChildrenByParent.set(row.issueId, observed);
|
||||
if (!nodesById.has(row.blockerIssueId)) {
|
||||
if (nodesById.size >= BLOCKER_ATTENTION_MAX_NODES) { complete = false; continue; }
|
||||
nodesById.set(row.blockerIssueId, {
|
||||
@@ -4000,11 +4009,12 @@ async function readIssueBlockerAttentionBatch(
|
||||
if (traversal.frontier.length > 0) traversal.truncated = true;
|
||||
}
|
||||
|
||||
const nodeIds = [...nodesById.keys()];
|
||||
const evidenceNodes = new Map([...nodesById, ...observedNodes]);
|
||||
const nodeIds = [...evidenceNodes.keys()];
|
||||
const activeIssueIds = new Set<string>();
|
||||
const agentIds = new Set<string>();
|
||||
const issueIdByExecutionRunId = new Map<string, string>();
|
||||
for (const node of nodesById.values()) {
|
||||
for (const node of evidenceNodes.values()) {
|
||||
if (node.assigneeAgentId) agentIds.add(node.assigneeAgentId);
|
||||
if (node.executionRunId)
|
||||
issueIdByExecutionRunId.set(node.executionRunId, node.id);
|
||||
@@ -4062,7 +4072,7 @@ async function readIssueBlockerAttentionBatch(
|
||||
}
|
||||
}
|
||||
|
||||
const explicitWaitCandidateIds = [...nodesById.values()]
|
||||
const explicitWaitCandidateIds = [...evidenceNodes.values()]
|
||||
.filter((node) => node.status !== "done")
|
||||
.map((node) => node.id);
|
||||
const explicitWaitingIssueIds = new Set<string>();
|
||||
@@ -4407,19 +4417,22 @@ async function readIssueBlockerAttentionBatch(
|
||||
};
|
||||
};
|
||||
|
||||
const nodeHasLiveWork = (node: IssueBlockerAttentionNode) =>
|
||||
node.status === "in_progress" || activeIssueIds.has(node.id) ||
|
||||
explicitWaitingIssueIds.has(node.id) || Boolean(node.assigneeUserId);
|
||||
const observedChildHasLiveWork = (parentId: string) =>
|
||||
[...observedChildrenByParent.get(parentId) ?? []].some(id => {
|
||||
const node = observedNodes.get(id);
|
||||
return node?.companyId === companyId && nodeHasLiveWork(node);
|
||||
});
|
||||
const pathHasLiveWork = (nodeId: string, seen: Set<string>, traversal: Traversal): boolean => {
|
||||
if (seen.has(nodeId) || !traversal.nodes.has(nodeId)) return false;
|
||||
const node = nodesById.get(nodeId);
|
||||
if (!node || node.companyId !== companyId) return false;
|
||||
if (
|
||||
node.status === "in_progress" ||
|
||||
activeIssueIds.has(node.id) ||
|
||||
explicitWaitingIssueIds.has(node.id) ||
|
||||
Boolean(node.assigneeUserId)
|
||||
)
|
||||
return true;
|
||||
if (nodeHasLiveWork(node)) return true;
|
||||
|
||||
if (!traversal.expanded.has(nodeId)) return false;
|
||||
if (observedChildHasLiveWork(nodeId)) return true;
|
||||
const nextSeen = new Set(seen);
|
||||
nextSeen.add(nodeId);
|
||||
return (edgesByIssueId.get(node.id) ?? []).some((edge) => {
|
||||
@@ -4546,7 +4559,7 @@ async function readIssueBlockerAttentionBatch(
|
||||
stalledEntry?.result.sampleStalledBlockerIdentifier ??
|
||||
sampleStalledFromChain ??
|
||||
null,
|
||||
blockingTreeLive: topLevelEdges.some((edge) =>
|
||||
blockingTreeLive: observedChildHasLiveWork(root.id) || topLevelEdges.some((edge) =>
|
||||
pathHasLiveWork(edge.blockerIssueId, new Set([root.id]), traversal),
|
||||
),
|
||||
directBlockerIssueId: sampleEntry?.edge.blockerIssueId ?? null,
|
||||
@@ -7653,6 +7666,7 @@ export function issueService(db: Db) {
|
||||
|
||||
await notifyIssueAttentionRetentionWork(dbOrTx, { companyId, issueId });
|
||||
await notifyIssueRecovery(dbOrTx, { companyId });
|
||||
await notifyDeferredExecutionEligibility(dbOrTx, { companyId });
|
||||
await dbOrTx
|
||||
.delete(issueRelations)
|
||||
.where(
|
||||
@@ -10643,6 +10657,7 @@ export function issueService(db: Db) {
|
||||
}
|
||||
if (values.monitorNextCheckAt) await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors);
|
||||
await notifyRunDispatch(tx, { companyId: values.companyId });
|
||||
await notifyDeferredExecutionEligibility(tx, { companyId: values.companyId });
|
||||
await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers);
|
||||
values.id ??= randomUUID();
|
||||
await notifyStatusCardIssueChange(tx, companyId);
|
||||
@@ -11548,6 +11563,7 @@ export function issueService(db: Db) {
|
||||
|| patch.executionState !== undefined || patch.executionRunId !== undefined || patch.checkoutRunId !== undefined
|
||||
|| blockedByIssueIds !== undefined) {
|
||||
await notifyRunDispatch(tx, { companyId: receiptExisting.companyId });
|
||||
await notifyDeferredExecutionEligibility(tx, { companyId: receiptExisting.companyId });
|
||||
await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers);
|
||||
await notifyRoutineExecutionStatus(tx, id);
|
||||
}
|
||||
@@ -12023,6 +12039,7 @@ export function issueService(db: Db) {
|
||||
await notifyDecisionTargetIssueRemoval(tx, id);
|
||||
await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors);
|
||||
await notifyRunDispatch(tx, owner ? { companyId: owner.companyId } : undefined);
|
||||
if (owner) await notifyDeferredExecutionEligibility(tx, owner);
|
||||
await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers);
|
||||
await notifyRoutineExecutionStatus(tx, id);
|
||||
removedIssue = await tx
|
||||
@@ -12155,6 +12172,7 @@ export function issueService(db: Db) {
|
||||
await notifyIssueAttentionRetentionWork(tx, { companyId: issueCompany.companyId, issueId: id });
|
||||
await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors);
|
||||
await notifyRunDispatch(tx, { companyId: issueCompany.companyId });
|
||||
await notifyDeferredExecutionEligibility(tx, { companyId: issueCompany.companyId });
|
||||
await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers);
|
||||
await notifyRoutineExecutionStatus(tx, id);
|
||||
return tx
|
||||
@@ -12291,6 +12309,7 @@ export function issueService(db: Db) {
|
||||
await notifyIssueAttentionRetentionWork(tx, { companyId: issueCompany.companyId, issueId: id });
|
||||
await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors);
|
||||
await notifyRunDispatch(tx, { companyId: issueCompany.companyId });
|
||||
await notifyDeferredExecutionEligibility(tx, { companyId: issueCompany.companyId });
|
||||
await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers);
|
||||
await notifyRoutineExecutionStatus(tx, id);
|
||||
return tx
|
||||
@@ -12549,6 +12568,7 @@ export function issueService(db: Db) {
|
||||
await notifyIssueAttentionRetentionWork(tx, { companyId: existing.companyId, issueId: id });
|
||||
await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors);
|
||||
await notifyRunDispatch(tx, { companyId: existing.companyId });
|
||||
await notifyDeferredExecutionEligibility(tx, { companyId: existing.companyId });
|
||||
await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers);
|
||||
await notifyRoutineExecutionStatus(tx, id);
|
||||
const updated = await tx
|
||||
@@ -12613,6 +12633,7 @@ export function issueService(db: Db) {
|
||||
}
|
||||
await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors);
|
||||
await notifyRunDispatch(tx, { companyId: existing.companyId });
|
||||
await notifyDeferredExecutionEligibility(tx, { companyId: existing.companyId });
|
||||
await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers);
|
||||
await notifyRoutineExecutionStatus(tx, id);
|
||||
const updated = await tx
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { notifyNativeStatusWakeWork } from "../deferred-execution-notifications.js";
|
||||
import { notifyExternalAgentSettlement } from "../../modules/external-agents/index.js";
|
||||
import { lockCurrentNativeWorkAssessmentEvidence } from "./work-assessments.js";
|
||||
import { hasNativeWorkspaceFinalizationOwner } from "./native-workspace-finalization-recovery-state.js";
|
||||
@@ -478,6 +479,7 @@ async function enqueueWake(input: {
|
||||
.limit(1)
|
||||
.then((rows) => rows[0] ?? null);
|
||||
if (existing) return existing.id;
|
||||
await notifyNativeStatusWakeWork(input.tx, { companyId: input.companyId });
|
||||
const inserted = await input.tx
|
||||
.insert(agentWakeupRequests)
|
||||
.values({
|
||||
|
||||
@@ -339,8 +339,10 @@ export async function requestTaskRetry(input: {
|
||||
|
||||
/** Chat owns admission and authorization. The task receipt follows that
|
||||
* durable outcome; it never launches a parallel generic board execution. */
|
||||
export async function reconcileTaskChatRetryReceipts(db: Db) {
|
||||
export async function reconcileTaskChatRetryReceipts(db: Db, scope: { companyId?: string; requestId?: string; settleOnly?: boolean } = {}) {
|
||||
const pending = await db.select().from(agentWakeupRequests).where(and(
|
||||
scope.companyId ? eq(agentWakeupRequests.companyId, scope.companyId) : undefined,
|
||||
scope.requestId ? eq(agentWakeupRequests.id, scope.requestId) : undefined,
|
||||
eq(agentWakeupRequests.reason, "task_retry"), eq(agentWakeupRequests.status, "deferred_issue_execution"),
|
||||
sql`${agentWakeupRequests.payload}#>>'{taskRetry,chatActionId}' is not null`,
|
||||
)).orderBy(agentWakeupRequests.updatedAt).limit(50);
|
||||
@@ -356,7 +358,7 @@ export async function reconcileTaskChatRetryReceipts(db: Db) {
|
||||
// A deferred chat receipt is admitted input, not a completed dispatch.
|
||||
// Re-stage that exact action through chat's owner; it rechecks authority
|
||||
// and normal queue policy before updating the existing receipt.
|
||||
if (chat?.status === "processed" && chat.payload.taskRetryIntentId === wake.id &&
|
||||
if (!scope.settleOnly && chat?.status === "processed" && chat.payload.taskRetryIntentId === wake.id &&
|
||||
chat.payload.failedRunId === intent.sourceRunId && receipt?.status === "deferred_issue_execution" &&
|
||||
!receipt.runId && receipt.payload?.taskRetryRequestId === wake.id) {
|
||||
await db.transaction(async tx => {
|
||||
|
||||
Reference in new issue
Block a user